git: 931dd5feb0a1 - main - nvmf: Add sysctl nodes for each queue pair
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sat, 02 Nov 2024 13:54:53 UTC
The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=931dd5feb0a13d6f24d3de7055d344732de0e353 commit 931dd5feb0a13d6f24d3de7055d344732de0e353 Author: John Baldwin <jhb@FreeBSD.org> AuthorDate: 2024-11-02 13:54:13 +0000 Commit: John Baldwin <jhb@FreeBSD.org> CommitDate: 2024-11-02 13:54:13 +0000 nvmf: Add sysctl nodes for each queue pair These report the queue size, queue head, queue tail, and the number of commands submitted. Sponsored by: Chelsio Communications --- sys/dev/nvmf/host/nvmf.c | 10 ++++++++-- sys/dev/nvmf/host/nvmf_qpair.c | 42 +++++++++++++++++++++++++++++++++++++++++- sys/dev/nvmf/host/nvmf_var.h | 5 ++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c index 12023ebf96f9..9d06b501d675 100644 --- a/sys/dev/nvmf/host/nvmf.c +++ b/sys/dev/nvmf/host/nvmf.c @@ -266,7 +266,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars) /* Setup the admin queue. */ sc->admin = nvmf_init_qp(sc, ivars->hh->trtype, &ivars->hh->admin, - "admin queue"); + "admin queue", 0); if (sc->admin == NULL) { device_printf(sc->dev, "Failed to setup admin queue\n"); return (ENXIO); @@ -279,7 +279,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars) for (u_int i = 0; i < sc->num_io_queues; i++) { snprintf(name, sizeof(name), "I/O queue %u", i); sc->io[i] = nvmf_init_qp(sc, ivars->hh->trtype, - &ivars->io_params[i], name); + &ivars->io_params[i], name, i); if (sc->io[i] == NULL) { device_printf(sc->dev, "Failed to setup I/O queue %u\n", i + 1); @@ -453,6 +453,7 @@ nvmf_attach(device_t dev) struct make_dev_args mda; struct nvmf_softc *sc = device_get_softc(dev); struct nvmf_ivars *ivars = device_get_ivars(dev); + struct sysctl_oid *oid; uint64_t val; u_int i; int error; @@ -467,6 +468,11 @@ nvmf_attach(device_t dev) sx_init(&sc->connection_lock, "nvmf connection"); TASK_INIT(&sc->disconnect_task, 0, nvmf_disconnect_task, sc); + oid = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "ioq", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queues"); + sc->ioq_oid_list = SYSCTL_CHILDREN(oid); + /* Claim the cdata pointer from ivars. */ sc->cdata = ivars->cdata; ivars->cdata = NULL; diff --git a/sys/dev/nvmf/host/nvmf_qpair.c b/sys/dev/nvmf/host/nvmf_qpair.c index 96cb5a8b0465..1aeb0535eacf 100644 --- a/sys/dev/nvmf/host/nvmf_qpair.c +++ b/sys/dev/nvmf/host/nvmf_qpair.c @@ -10,6 +10,7 @@ #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> +#include <sys/sysctl.h> #include <dev/nvme/nvme.h> #include <dev/nvmf/nvmf.h> #include <dev/nvmf/nvmf_transport.h> @@ -31,6 +32,7 @@ struct nvmf_host_qpair { u_int num_commands; uint16_t sqhd; uint16_t sqtail; + uint64_t submitted; struct mtx lock; @@ -41,6 +43,7 @@ struct nvmf_host_qpair { struct nvmf_host_command **active_commands; char name[16]; + struct sysctl_ctx_list sysctl_ctx; }; struct nvmf_request * @@ -212,6 +215,7 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc) } else { cmd->req = STAILQ_FIRST(&qp->pending_requests); STAILQ_REMOVE_HEAD(&qp->pending_requests, link); + qp->submitted++; mtx_unlock(&qp->lock); nvmf_dispatch_command(qp, cmd); } @@ -221,9 +225,39 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc) nvmf_free_request(req); } +static void +nvmf_sysctls_qp(struct nvmf_softc *sc, struct nvmf_host_qpair *qp, + bool admin, u_int qid) +{ + struct sysctl_ctx_list *ctx = &qp->sysctl_ctx; + struct sysctl_oid *oid; + struct sysctl_oid_list *list; + char name[8]; + + if (admin) { + oid = SYSCTL_ADD_NODE(ctx, + SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, + "adminq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Admin Queue"); + } else { + snprintf(name, sizeof(name), "%u", qid); + oid = SYSCTL_ADD_NODE(ctx, sc->ioq_oid_list, OID_AUTO, name, + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queue"); + } + list = SYSCTL_CHILDREN(oid); + + SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "num_entries", CTLFLAG_RD, + NULL, qp->num_commands + 1, "Number of entries in queue"); + SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_head", CTLFLAG_RD, &qp->sqhd, + 0, "Current head of submission queue (as observed by driver)"); + SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_tail", CTLFLAG_RD, &qp->sqtail, + 0, "Current tail of submission queue (as observed by driver)"); + SYSCTL_ADD_U64(ctx, list, OID_AUTO, "num_cmds", CTLFLAG_RD, + &qp->submitted, 0, "Number of commands submitted"); +} + struct nvmf_host_qpair * nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype, - struct nvmf_handoff_qpair_params *handoff, const char *name) + struct nvmf_handoff_qpair_params *handoff, const char *name, u_int qid) { struct nvmf_host_command *cmd, *ncmd; struct nvmf_host_qpair *qp; @@ -236,6 +270,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype, qp->sqtail = handoff->sqtail; strlcpy(qp->name, name, sizeof(qp->name)); mtx_init(&qp->lock, "nvmf qp", NULL, MTX_DEF); + (void)sysctl_ctx_init(&qp->sysctl_ctx); /* * Allocate a spare command slot for each pending AER command @@ -258,6 +293,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype, qp->qp = nvmf_allocate_qpair(trtype, false, handoff, nvmf_qp_error, qp, nvmf_receive_capsule, qp); if (qp->qp == NULL) { + (void)sysctl_ctx_free(&qp->sysctl_ctx); TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) { TAILQ_REMOVE(&qp->free_commands, cmd, link); free(cmd, M_NVMF); @@ -268,6 +304,8 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype, return (NULL); } + nvmf_sysctls_qp(sc, qp, handoff->admin, qid); + return (qp); } @@ -339,6 +377,7 @@ nvmf_destroy_qp(struct nvmf_host_qpair *qp) struct nvmf_host_command *cmd, *ncmd; nvmf_shutdown_qp(qp); + (void)sysctl_ctx_free(&qp->sysctl_ctx); TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) { TAILQ_REMOVE(&qp->free_commands, cmd, link); @@ -381,6 +420,7 @@ nvmf_submit_request(struct nvmf_request *req) ("%s: CID already busy", __func__)); qp->active_commands[cmd->cid] = cmd; cmd->req = req; + qp->submitted++; mtx_unlock(&qp->lock); nvmf_dispatch_command(qp, cmd); } diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h index adf6d8bde3d9..0e52f36a80a5 100644 --- a/sys/dev/nvmf/host/nvmf_var.h +++ b/sys/dev/nvmf/host/nvmf_var.h @@ -22,6 +22,7 @@ struct nvmf_aer; struct nvmf_capsule; struct nvmf_host_qpair; struct nvmf_namespace; +struct sysctl_oid_list; typedef void nvmf_request_complete_t(void *, const struct nvme_completion *); @@ -85,6 +86,8 @@ struct nvmf_softc { u_int num_aer; struct nvmf_aer *aer; + struct sysctl_oid_list *ioq_oid_list; + eventhandler_tag shutdown_pre_sync_eh; eventhandler_tag shutdown_post_sync_eh; }; @@ -200,7 +203,7 @@ bool nvmf_update_ns(struct nvmf_namespace *ns, /* nvmf_qpair.c */ struct nvmf_host_qpair *nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype, struct nvmf_handoff_qpair_params *handoff, - const char *name); + const char *name, u_int qid); void nvmf_shutdown_qp(struct nvmf_host_qpair *qp); void nvmf_destroy_qp(struct nvmf_host_qpair *qp); struct nvmf_request *nvmf_allocate_request(struct nvmf_host_qpair *qp,