svn commit: r335149 - stable/11/sys/dev/nvme

[Alexander Motin]

Author: mav
Date: Thu Jun 14 16:19:05 2018
New Revision: 335149
URL: https://svnweb.freebsd.org/changeset/base/335149

Log:
  MFC r330954, r330955 (by imp):
  When tearing down a queue pair, also delete the queue entries.
  
  The NVME standard has required in section 7.2.6, since at least 1.1,
  that a clean shutdown is signalled by deleting the subission and the
  completion queues before setting the shutdown bit in CC. The 1.0
  standard, apparently, did not and many of the early Intel cards didn't
  care. Some newer cards care, at least one whose beta firmware can
  scramble the card on an unclean shutdown. Linux has done this for some
  time. To make it possible to move forward with an evaluation of this
  pre-release card with wonky firmware, delete the queues on the card
  when we delete the qpair structures.

Modified:
  stable/11/sys/dev/nvme/nvme_ctrlr.c
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- stable/11/sys/dev/nvme/nvme_ctrlr.c	Thu Jun 14 16:11:50 2018	(r335148)
+++ stable/11/sys/dev/nvme/nvme_ctrlr.c	Thu Jun 14 16:19:05 2018	(r335149)
@@ -493,6 +493,34 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr
 }
 
 static int
+nvme_ctrlr_destroy_qpair(struct nvme_controller *ctrlr, struct nvme_qpair *qpair)
+{
+	struct nvme_completion_poll_status	status;
+
+	status.done = 0;
+	nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
+	    nvme_completion_poll_cb, &status);
+	while (!atomic_load_acq_int(&status.done))
+		pause("nvme", 1);
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
+		return (ENXIO);
+	}
+
+	status.done = 0;
+	nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
+	    nvme_completion_poll_cb, &status);
+	while (!atomic_load_acq_int(&status.done))
+		pause("nvme", 1);
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static int
 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
 {
 	struct nvme_namespace	*ns;
@@ -1216,6 +1244,7 @@ nvme_ctrlr_destruct(struct nvme_controller *ctrlr, dev
 		destroy_dev(ctrlr->cdev);
 
 	for (i = 0; i < ctrlr->num_io_queues; i++) {
+		nvme_ctrlr_destroy_qpair(ctrlr, &ctrlr->ioq[i]);
 		nvme_io_qpair_destroy(&ctrlr->ioq[i]);
 	}