svn commit: r322994 - head/sys/dev/nvme

[Warner Losh]

Author: imp
Date: Mon Aug 28 23:54:20 2017
New Revision: 322994
URL: https://svnweb.freebsd.org/changeset/base/322994

Log:
  Set the max transactions for NVMe drives better.
  
  Provided a better estimate for the number of transactions that can be
  pending at one time. This will be number of queues * number of
  trackers / 4, as suggested by Jim Harris. This gives a better estimate
  of the number of transactions that CAM should queue before applying
  back pressure. This should be revisted when we have real multi-queue
  support in CAM and the upper layers of the I/O stack.
  
  Sponsored by: Netflix

Modified:
  head/sys/dev/nvme/nvme_ctrlr.c
  head/sys/dev/nvme/nvme_private.h
  head/sys/dev/nvme/nvme_sim.c

Modified: head/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- head/sys/dev/nvme/nvme_ctrlr.c	Mon Aug 28 23:54:16 2017	(r322993)
+++ head/sys/dev/nvme/nvme_ctrlr.c	Mon Aug 28 23:54:20 2017	(r322994)
@@ -146,6 +146,14 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller 
 	num_trackers = min(num_trackers, (num_entries-1));
 
 	/*
+	 * Our best estimate for the maximum number of I/Os that we should
+	 * noramlly have in flight at one time. This should be viewed as a hint,
+	 * not a hard limit and will need to be revisitted when the upper layers
+	 * of the storage system grows multi-queue support.
+	 */
+	ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues / 4;
+
+	/*
 	 * This was calculated previously when setting up interrupts, but
 	 *  a controller could theoretically support fewer I/O queues than
 	 *  MSI-X vectors.  So calculate again here just to be safe.

Modified: head/sys/dev/nvme/nvme_private.h
==============================================================================
--- head/sys/dev/nvme/nvme_private.h	Mon Aug 28 23:54:16 2017	(r322993)
+++ head/sys/dev/nvme/nvme_private.h	Mon Aug 28 23:54:20 2017	(r322994)
@@ -263,6 +263,7 @@ struct nvme_controller {
 
 	uint32_t		num_io_queues;
 	uint32_t		num_cpus_per_ioq;
+	uint32_t		max_hw_pend_io;
 
 	/* Fields for tracking progress during controller initialization. */
 	struct intr_config_hook	config_hook;

Modified: head/sys/dev/nvme/nvme_sim.c
==============================================================================
--- head/sys/dev/nvme/nvme_sim.c	Mon Aug 28 23:54:16 2017	(r322993)
+++ head/sys/dev/nvme/nvme_sim.c	Mon Aug 28 23:54:20 2017	(r322994)
@@ -253,7 +253,7 @@ nvme_sim_new_controller(struct nvme_controller *ctrlr)
 	int unit;
 	struct nvme_sim_softc *sc = NULL;
 
-	max_trans = ctrlr->num_io_queues;
+	max_trans = ctrlr->max_hw_pend_io;
 	unit = device_get_unit(ctrlr->dev);
 	devq = cam_simq_alloc(max_trans);
 	if (devq == NULL)