svn commit: r351705 - head/sys/dev/nvme

[Warner Losh]

Author: imp
Date: Mon Sep  2 17:11:27 2019
New Revision: 351705
URL: https://svnweb.freebsd.org/changeset/base/351705

Log:
  In all the places that we use the polled for completion interface, except crash
  dump support code, move the while loop into an inline function. These aren't
  done in the fast path, so if the compiler choses to not inline, any performance
  hit is tiny.

Modified:
  head/sys/dev/nvme/nvme_ctrlr.c
  head/sys/dev/nvme/nvme_ns.c
  head/sys/dev/nvme/nvme_private.h

Modified: head/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- head/sys/dev/nvme/nvme_ctrlr.c	Mon Sep  2 17:10:46 2019	(r351704)
+++ head/sys/dev/nvme/nvme_ctrlr.c	Mon Sep  2 17:11:27 2019	(r351705)
@@ -394,8 +394,7 @@ nvme_ctrlr_identify(struct nvme_controller *ctrlr)
 	status.done = 0;
 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
 	    nvme_completion_poll_cb, &status);
-	while (!atomic_load_acq_int(&status.done))
-		pause("nvme", 1);
+	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
 		return (ENXIO);
@@ -424,8 +423,7 @@ nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrl
 	status.done = 0;
 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
 	    nvme_completion_poll_cb, &status);
-	while (!atomic_load_acq_int(&status.done))
-		pause("nvme", 1);
+	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
 		return (ENXIO);
@@ -463,8 +461,7 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr
 		status.done = 0;
 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
 		    nvme_completion_poll_cb, &status);
-		while (!atomic_load_acq_int(&status.done))
-			pause("nvme", 1);
+		nvme_completion_poll(&status);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
 			return (ENXIO);
@@ -473,8 +470,7 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr
 		status.done = 0;
 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
 		    nvme_completion_poll_cb, &status);
-		while (!atomic_load_acq_int(&status.done))
-			pause("nvme", 1);
+		nvme_completion_poll(&status);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
 			return (ENXIO);
@@ -496,8 +492,7 @@ nvme_ctrlr_destroy_qpairs(struct nvme_controller *ctrl
 		status.done = 0;
 		nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
 		    nvme_completion_poll_cb, &status);
-		while (!atomic_load_acq_int(&status.done))
-			pause("nvme", 1);
+		nvme_completion_poll(&status);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
 			return (ENXIO);
@@ -506,8 +501,7 @@ nvme_ctrlr_destroy_qpairs(struct nvme_controller *ctrl
 		status.done = 0;
 		nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
 		    nvme_completion_poll_cb, &status);
-		while (!atomic_load_acq_int(&status.done))
-			pause("nvme", 1);
+		nvme_completion_poll(&status);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
 			return (ENXIO);
@@ -789,8 +783,7 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr
 	status.done = 0;
 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
 	    0, NULL, 0, nvme_completion_poll_cb, &status);
-	while (!atomic_load_acq_int(&status.done))
-		pause("nvme", 1);
+	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl) ||
 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
 	    (status.cpl.cdw0 & 0xFFFF) == 0x0000) {

Modified: head/sys/dev/nvme/nvme_ns.c
==============================================================================
--- head/sys/dev/nvme/nvme_ns.c	Mon Sep  2 17:10:46 2019	(r351704)
+++ head/sys/dev/nvme/nvme_ns.c	Mon Sep  2 17:11:27 2019	(r351705)
@@ -530,8 +530,7 @@ nvme_ns_construct(struct nvme_namespace *ns, uint32_t 
 	status.done = 0;
 	nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data,
 	    nvme_completion_poll_cb, &status);
-	while (!atomic_load_acq_int(&status.done))
-		pause("nvme", 1);
+	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_identify_namespace failed\n");
 		return (ENXIO);

Modified: head/sys/dev/nvme/nvme_private.h
==============================================================================
--- head/sys/dev/nvme/nvme_private.h	Mon Sep  2 17:10:46 2019	(r351704)
+++ head/sys/dev/nvme/nvme_private.h	Mon Sep  2 17:11:27 2019	(r351705)
@@ -446,6 +446,14 @@ int	nvme_attach(device_t dev);
 int	nvme_shutdown(device_t dev);
 int	nvme_detach(device_t dev);
 
+static __inline
+void
+nvme_completion_poll(struct nvme_completion_poll_status *status)
+{
+	while (!atomic_load_acq_int(&status->done))
+		pause("nvme", 1);
+}
+
 static __inline void
 nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 {