git: 931dd5feb0a1 - main - nvmf: Add sysctl nodes for each queue pair

From: John Baldwin <jhb_at_FreeBSD.org>
Date: Sat, 02 Nov 2024 13:54:53 UTC
The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=931dd5feb0a13d6f24d3de7055d344732de0e353

commit 931dd5feb0a13d6f24d3de7055d344732de0e353
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2024-11-02 13:54:13 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2024-11-02 13:54:13 +0000

    nvmf: Add sysctl nodes for each queue pair
    
    These report the queue size, queue head, queue tail, and the number of
    commands submitted.
    
    Sponsored by:   Chelsio Communications
---
 sys/dev/nvmf/host/nvmf.c       | 10 ++++++++--
 sys/dev/nvmf/host/nvmf_qpair.c | 42 +++++++++++++++++++++++++++++++++++++++++-
 sys/dev/nvmf/host/nvmf_var.h   |  5 ++++-
 3 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c
index 12023ebf96f9..9d06b501d675 100644
--- a/sys/dev/nvmf/host/nvmf.c
+++ b/sys/dev/nvmf/host/nvmf.c
@@ -266,7 +266,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars)
 
 	/* Setup the admin queue. */
 	sc->admin = nvmf_init_qp(sc, ivars->hh->trtype, &ivars->hh->admin,
-	    "admin queue");
+	    "admin queue", 0);
 	if (sc->admin == NULL) {
 		device_printf(sc->dev, "Failed to setup admin queue\n");
 		return (ENXIO);
@@ -279,7 +279,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars)
 	for (u_int i = 0; i < sc->num_io_queues; i++) {
 		snprintf(name, sizeof(name), "I/O queue %u", i);
 		sc->io[i] = nvmf_init_qp(sc, ivars->hh->trtype,
-		    &ivars->io_params[i], name);
+		    &ivars->io_params[i], name, i);
 		if (sc->io[i] == NULL) {
 			device_printf(sc->dev, "Failed to setup I/O queue %u\n",
 			    i + 1);
@@ -453,6 +453,7 @@ nvmf_attach(device_t dev)
 	struct make_dev_args mda;
 	struct nvmf_softc *sc = device_get_softc(dev);
 	struct nvmf_ivars *ivars = device_get_ivars(dev);
+	struct sysctl_oid *oid;
 	uint64_t val;
 	u_int i;
 	int error;
@@ -467,6 +468,11 @@ nvmf_attach(device_t dev)
 	sx_init(&sc->connection_lock, "nvmf connection");
 	TASK_INIT(&sc->disconnect_task, 0, nvmf_disconnect_task, sc);
 
+	oid = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev),
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "ioq",
+	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queues");
+	sc->ioq_oid_list = SYSCTL_CHILDREN(oid);
+
 	/* Claim the cdata pointer from ivars. */
 	sc->cdata = ivars->cdata;
 	ivars->cdata = NULL;
diff --git a/sys/dev/nvmf/host/nvmf_qpair.c b/sys/dev/nvmf/host/nvmf_qpair.c
index 96cb5a8b0465..1aeb0535eacf 100644
--- a/sys/dev/nvmf/host/nvmf_qpair.c
+++ b/sys/dev/nvmf/host/nvmf_qpair.c
@@ -10,6 +10,7 @@
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
+#include <sys/sysctl.h>
 #include <dev/nvme/nvme.h>
 #include <dev/nvmf/nvmf.h>
 #include <dev/nvmf/nvmf_transport.h>
@@ -31,6 +32,7 @@ struct nvmf_host_qpair {
 	u_int	num_commands;
 	uint16_t sqhd;
 	uint16_t sqtail;
+	uint64_t submitted;
 
 	struct mtx lock;
 
@@ -41,6 +43,7 @@ struct nvmf_host_qpair {
 	struct nvmf_host_command **active_commands;
 
 	char	name[16];
+	struct sysctl_ctx_list sysctl_ctx;
 };
 
 struct nvmf_request *
@@ -212,6 +215,7 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
 	} else {
 		cmd->req = STAILQ_FIRST(&qp->pending_requests);
 		STAILQ_REMOVE_HEAD(&qp->pending_requests, link);
+		qp->submitted++;
 		mtx_unlock(&qp->lock);
 		nvmf_dispatch_command(qp, cmd);
 	}
@@ -221,9 +225,39 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
 	nvmf_free_request(req);
 }
 
+static void
+nvmf_sysctls_qp(struct nvmf_softc *sc, struct nvmf_host_qpair *qp,
+    bool admin, u_int qid)
+{
+	struct sysctl_ctx_list *ctx = &qp->sysctl_ctx;
+	struct sysctl_oid *oid;
+	struct sysctl_oid_list *list;
+	char name[8];
+
+	if (admin) {
+		oid = SYSCTL_ADD_NODE(ctx,
+		    SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
+		    "adminq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Admin Queue");
+	} else {
+		snprintf(name, sizeof(name), "%u", qid);
+		oid = SYSCTL_ADD_NODE(ctx, sc->ioq_oid_list, OID_AUTO, name,
+		    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queue");
+	}
+	list = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "num_entries", CTLFLAG_RD,
+	    NULL, qp->num_commands + 1, "Number of entries in queue");
+	SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_head", CTLFLAG_RD, &qp->sqhd,
+	    0, "Current head of submission queue (as observed by driver)");
+	SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_tail", CTLFLAG_RD, &qp->sqtail,
+	    0, "Current tail of submission queue (as observed by driver)");
+	SYSCTL_ADD_U64(ctx, list, OID_AUTO, "num_cmds", CTLFLAG_RD,
+	    &qp->submitted, 0, "Number of commands submitted");
+}
+
 struct nvmf_host_qpair *
 nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
-    struct nvmf_handoff_qpair_params *handoff, const char *name)
+    struct nvmf_handoff_qpair_params *handoff, const char *name, u_int qid)
 {
 	struct nvmf_host_command *cmd, *ncmd;
 	struct nvmf_host_qpair *qp;
@@ -236,6 +270,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
 	qp->sqtail = handoff->sqtail;
 	strlcpy(qp->name, name, sizeof(qp->name));
 	mtx_init(&qp->lock, "nvmf qp", NULL, MTX_DEF);
+	(void)sysctl_ctx_init(&qp->sysctl_ctx);
 
 	/*
 	 * Allocate a spare command slot for each pending AER command
@@ -258,6 +293,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
 	qp->qp = nvmf_allocate_qpair(trtype, false, handoff, nvmf_qp_error,
 	    qp, nvmf_receive_capsule, qp);
 	if (qp->qp == NULL) {
+		(void)sysctl_ctx_free(&qp->sysctl_ctx);
 		TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {
 			TAILQ_REMOVE(&qp->free_commands, cmd, link);
 			free(cmd, M_NVMF);
@@ -268,6 +304,8 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
 		return (NULL);
 	}
 
+	nvmf_sysctls_qp(sc, qp, handoff->admin, qid);
+
 	return (qp);
 }
 
@@ -339,6 +377,7 @@ nvmf_destroy_qp(struct nvmf_host_qpair *qp)
 	struct nvmf_host_command *cmd, *ncmd;
 
 	nvmf_shutdown_qp(qp);
+	(void)sysctl_ctx_free(&qp->sysctl_ctx);
 
 	TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {
 		TAILQ_REMOVE(&qp->free_commands, cmd, link);
@@ -381,6 +420,7 @@ nvmf_submit_request(struct nvmf_request *req)
 	    ("%s: CID already busy", __func__));
 	qp->active_commands[cmd->cid] = cmd;
 	cmd->req = req;
+	qp->submitted++;
 	mtx_unlock(&qp->lock);
 	nvmf_dispatch_command(qp, cmd);
 }
diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h
index adf6d8bde3d9..0e52f36a80a5 100644
--- a/sys/dev/nvmf/host/nvmf_var.h
+++ b/sys/dev/nvmf/host/nvmf_var.h
@@ -22,6 +22,7 @@ struct nvmf_aer;
 struct nvmf_capsule;
 struct nvmf_host_qpair;
 struct nvmf_namespace;
+struct sysctl_oid_list;
 
 typedef void nvmf_request_complete_t(void *, const struct nvme_completion *);
 
@@ -85,6 +86,8 @@ struct nvmf_softc {
 	u_int num_aer;
 	struct nvmf_aer *aer;
 
+	struct sysctl_oid_list *ioq_oid_list;
+
 	eventhandler_tag shutdown_pre_sync_eh;
 	eventhandler_tag shutdown_post_sync_eh;
 };
@@ -200,7 +203,7 @@ bool	nvmf_update_ns(struct nvmf_namespace *ns,
 /* nvmf_qpair.c */
 struct nvmf_host_qpair *nvmf_init_qp(struct nvmf_softc *sc,
     enum nvmf_trtype trtype, struct nvmf_handoff_qpair_params *handoff,
-    const char *name);
+    const char *name, u_int qid);
 void	nvmf_shutdown_qp(struct nvmf_host_qpair *qp);
 void	nvmf_destroy_qp(struct nvmf_host_qpair *qp);
 struct nvmf_request *nvmf_allocate_request(struct nvmf_host_qpair *qp,