git: 1a7f22d9c211 - main - ctlstat: add prometheus output
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 20 Apr 2022 21:19:06 UTC
The branch main has been updated by asomers:
URL: https://cgit.FreeBSD.org/src/commit/?id=1a7f22d9c211f504f6c48a86401469181a67ec34
commit 1a7f22d9c211f504f6c48a86401469181a67ec34
Author: Alan Somers <asomers@FreeBSD.org>
AuthorDate: 2021-04-21 22:56:48 +0000
Commit: Alan Somers <asomers@FreeBSD.org>
CommitDate: 2022-04-20 21:18:41 +0000
ctlstat: add prometheus output
When invoked by inetd, ctlstat -P will now produce output suitable for
ingestion into Prometheus.
It's a drop-in replacement for https://github.com/Gandi/ctld_exporter,
except that it doesn't report the number of initiators per target, and
it does report time and dma_time.
MFC after: 2 weeks
Sponsored by: Axcient
Relnotes: yes
Reviewed by: bapt, bcr
Differential Revision: https://reviews.freebsd.org/D29901
---
usr.bin/ctlstat/Makefile | 2 +
usr.bin/ctlstat/ctlstat.8 | 24 +++-
usr.bin/ctlstat/ctlstat.c | 241 ++++++++++++++++++++++++++++++++++++++--
usr.sbin/inetd/inetd.conf | 3 +
usr.sbin/services_mkdb/services | 1 +
5 files changed, 261 insertions(+), 10 deletions(-)
diff --git a/usr.bin/ctlstat/Makefile b/usr.bin/ctlstat/Makefile
index 31182374f381..53b7a59fae23 100644
--- a/usr.bin/ctlstat/Makefile
+++ b/usr.bin/ctlstat/Makefile
@@ -5,4 +5,6 @@ MAN= ctlstat.8
SDIR= ${SRCTOP}/sys
CFLAGS+= -I${SDIR}
+LIBADD= sbuf bsdxml
+
.include <bsd.prog.mk>
diff --git a/usr.bin/ctlstat/ctlstat.8 b/usr.bin/ctlstat/ctlstat.8
index 28e6d6fd1462..2512803f798a 100644
--- a/usr.bin/ctlstat/ctlstat.8
+++ b/usr.bin/ctlstat/ctlstat.8
@@ -34,7 +34,7 @@
.\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
.\" $FreeBSD$
.\"
-.Dd January 9, 2017
+.Dd April 22, 2021
.Dt CTLSTAT 8
.Os
.Sh NAME
@@ -48,6 +48,7 @@
.Op Fl d
.Op Fl D
.Op Fl j
+.Op Fl P
.Op Fl l Ar lun
.Op Fl n Ar numdevs
.Op Fl p Ar port
@@ -83,6 +84,19 @@ Suppress display of the header.
JSON dump mode.
Dump statistics every 30 seconds in JavaScript Object Notation (JSON) format.
No statistics are computed in this mode, only raw numbers are displayed.
+.It Fl P
+Prometheus dump mode.
+Dump statistics in a format suitable for ingestion into Prometheus.
+When invoked with this option,
+.Nm
+dumps once, regardless of the
+.Fl t
+option.
+This option is especially useful when invoked by
+.Xr inetd 8 .
+See the comments in
+.Pa /etc/inetd.conf
+for an example configuration.
.It Fl l Ar lun
Request statistics for the specified LUN.
.It Fl n Ar numdevs
@@ -116,7 +130,13 @@ every 10 seconds.
.Xr camcontrol 8 ,
.Xr ctladm 8 ,
.Xr ctld 8 ,
-.Xr iostat 8
+.Xr iostat 8 ,
+.Lk
+Prometheus project:
+.Pa https://prometheus.io/ .
+.Pp
+Prometheus exposition formats:
+.Lk https://prometheus.io/docs/instrumenting/exposition_formats/ .
.Sh AUTHORS
.An Ken Merry Aq Mt ken@FreeBSD.org
.An Will Andrews Aq Mt will@FreeBSD.org
diff --git a/usr.bin/ctlstat/ctlstat.c b/usr.bin/ctlstat/ctlstat.c
index 6586430172ce..50b9e3b1445b 100644
--- a/usr.bin/ctlstat/ctlstat.c
+++ b/usr.bin/ctlstat/ctlstat.c
@@ -41,19 +41,23 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
-#include <sys/ioctl.h>
-#include <sys/types.h>
#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysctl.h>
-#include <sys/resource.h>
-#include <sys/queue.h>
#include <sys/callout.h>
+#include <sys/ioctl.h>
+#include <sys/queue.h>
+#include <sys/resource.h>
+#include <sys/sbuf.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include <bsdxml.h>
+#include <malloc_np.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
+#include <inttypes.h>
#include <getopt.h>
#include <string.h>
#include <errno.h>
@@ -76,8 +80,8 @@ __FBSDID("$FreeBSD$");
static int ctl_stat_bits;
-static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
-static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]"
+static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:";
+static const char *ctlstat_usage = "Usage: ctlstat [-CDPdjht] [-l lunnum]"
"[-c count] [-n numdevs] [-w wait]\n";
struct ctl_cpu_stats {
@@ -92,6 +96,7 @@ typedef enum {
CTLSTAT_MODE_STANDARD,
CTLSTAT_MODE_DUMP,
CTLSTAT_MODE_JSON,
+ CTLSTAT_MODE_PROMETHEUS,
} ctlstat_mode_types;
#define CTLSTAT_FLAG_CPU (1 << 0)
@@ -129,6 +134,15 @@ struct ctlstat_context {
int header_interval;
};
+struct cctl_portlist_data {
+ int level;
+ struct sbuf *cur_sb[32];
+ int lun;
+ int ntargets;
+ char *target;
+ char **targets;
+};
+
#ifndef min
#define min(x,y) (((x) < (y)) ? (x) : (y))
#endif
@@ -381,6 +395,200 @@ ctlstat_json(struct ctlstat_context *ctx) {
printf("]}");
}
+#define CTLSTAT_PROMETHEUS_LOOP(field) \
+ for (i = n = 0; i < ctx->cur_items; i++) { \
+ if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+ (int)stats[i].item) == 0) \
+ continue; \
+ for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+ int lun = stats[i].item; \
+ if (lun >= targdata.ntargets) \
+ errx(1, "LUN %u out of range", lun); \
+ printf("iscsi_target_" #field "{" \
+ "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+ "\n", \
+ lun, targdata.targets[lun], iotypes[iotype], \
+ stats[i].field[iotype]); \
+ } \
+ } \
+
+#define CTLSTAT_PROMETHEUS_TIMELOOP(field) \
+ for (i = n = 0; i < ctx->cur_items; i++) { \
+ if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+ (int)stats[i].item) == 0) \
+ continue; \
+ for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+ uint64_t us; \
+ struct timespec ts; \
+ int lun = stats[i].item; \
+ if (lun >= targdata.ntargets) \
+ errx(1, "LUN %u out of range", lun); \
+ bintime2timespec(&stats[i].field[iotype], &ts); \
+ us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \
+ printf("iscsi_target_" #field "{" \
+ "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+ "\n", \
+ lun, targdata.targets[lun], iotypes[iotype], us); \
+ } \
+ } \
+
+static void
+cctl_start_pelement(void *user_data, const char *name, const char **attr __unused)
+{
+ struct cctl_portlist_data* targdata = user_data;
+
+ targdata->level++;
+ if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) /
+ sizeof(targdata->cur_sb[0])))
+ errx(1, "%s: too many nesting levels, %zd max", __func__,
+ sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0]));
+
+ targdata->cur_sb[targdata->level] = sbuf_new_auto();
+ if (targdata->cur_sb[targdata->level] == NULL)
+ err(1, "%s: Unable to allocate sbuf", __func__);
+
+ if (strcmp(name, "targ_port") == 0) {
+ targdata->lun = -1;
+ free(targdata->target);
+ targdata->target = NULL;
+ }
+}
+
+static void
+cctl_char_phandler(void *user_data, const XML_Char *str, int len)
+{
+ struct cctl_portlist_data *targdata = user_data;
+
+ sbuf_bcat(targdata->cur_sb[targdata->level], str, len);
+}
+
+static void
+cctl_end_pelement(void *user_data, const char *name)
+{
+ struct cctl_portlist_data* targdata = user_data;
+ char *str;
+
+ if (targdata->cur_sb[targdata->level] == NULL)
+ errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
+ targdata->level, name);
+
+ if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0)
+ err(1, "%s: sbuf_finish", __func__);
+ str = strdup(sbuf_data(targdata->cur_sb[targdata->level]));
+ if (str == NULL)
+ err(1, "%s can't allocate %zd bytes for string", __func__,
+ sbuf_len(targdata->cur_sb[targdata->level]));
+
+ sbuf_delete(targdata->cur_sb[targdata->level]);
+ targdata->cur_sb[targdata->level] = NULL;
+ targdata->level--;
+
+ if (strcmp(name, "target") == 0) {
+ free(targdata->target);
+ targdata->target = str;
+ } else if (strcmp(name, "lun") == 0) {
+ targdata->lun = atoi(str);
+ free(str);
+ } else if (strcmp(name, "targ_port") == 0) {
+ if (targdata->lun >= 0 && targdata->target != NULL) {
+ if (targdata->lun >= targdata->ntargets) {
+ /*
+ * This can happen for example if there are
+ * holes in CTL's lunlist.
+ */
+ targdata->ntargets = MAX(targdata->ntargets * 2,
+ targdata->lun + 1);
+ size_t newsize = targdata->ntargets *
+ sizeof(char*);
+ targdata->targets = rallocx(targdata->targets,
+ newsize, MALLOCX_ZERO);
+ }
+ free(targdata->targets[targdata->lun]);
+ targdata->targets[targdata->lun] = targdata->target;
+ targdata->target = NULL;
+ }
+ free(str);
+ } else {
+ free(str);
+ }
+}
+
+static void
+ctlstat_prometheus(int fd, struct ctlstat_context *ctx) {
+ struct ctl_io_stats *stats = ctx->cur_stats;
+ struct ctl_lun_list list;
+ struct cctl_portlist_data targdata;
+ XML_Parser parser;
+ char *port_str = NULL;
+ int iotype, i, n, retval;
+ int port_len = 4096;
+
+ bzero(&targdata, sizeof(targdata));
+ targdata.ntargets = ctx->cur_items;
+ targdata.targets = calloc(targdata.ntargets, sizeof(char*));
+retry:
+ port_str = (char *)realloc(port_str, port_len);
+ bzero(&list, sizeof(list));
+ list.alloc_len = port_len;
+ list.status = CTL_LUN_LIST_NONE;
+ list.lun_xml = port_str;
+ if (ioctl(fd, CTL_PORT_LIST, &list) == -1)
+ err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__);
+ if (list.status == CTL_LUN_LIST_ERROR) {
+ warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s",
+ __func__, list.error_str);
+ } else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
+ port_len <<= 1;
+ goto retry;
+ }
+
+ parser = XML_ParserCreate(NULL);
+ if (parser == NULL)
+ err(1, "%s: Unable to create XML parser", __func__);
+ XML_SetUserData(parser, &targdata);
+ XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
+ XML_SetCharacterDataHandler(parser, cctl_char_phandler);
+
+ retval = XML_Parse(parser, port_str, strlen(port_str), 1);
+ if (retval != 1) {
+ errx(1, "%s: Unable to parse XML: Error %d", __func__,
+ XML_GetErrorCode(parser));
+ }
+ XML_ParserFree(parser);
+
+ /*
+ * NB: Some clients will print a warning if we don't set Content-Length,
+ * but they still work. And the data still gets into Prometheus.
+ */
+ printf("HTTP/1.1 200 OK\r\n"
+ "Connection: close\r\n"
+ "Content-Type: text/plain; version=0.0.4\r\n"
+ "\r\n");
+
+ printf("# HELP iscsi_target_bytes Number of bytes\n"
+ "# TYPE iscsi_target_bytes counter\n");
+ CTLSTAT_PROMETHEUS_LOOP(bytes);
+ printf("# HELP iscsi_target_dmas Number of DMA\n"
+ "# TYPE iscsi_target_dmas counter\n");
+ CTLSTAT_PROMETHEUS_LOOP(dmas);
+ printf("# HELP iscsi_target_operations Number of operations\n"
+ "# TYPE iscsi_target_operations counter\n");
+ CTLSTAT_PROMETHEUS_LOOP(operations);
+ printf("# HELP iscsi_target_time Cumulative operation time in us\n"
+ "# TYPE iscsi_target_time counter\n");
+ CTLSTAT_PROMETHEUS_TIMELOOP(time);
+ printf("# HELP iscsi_target_dma_time Cumulative DMA time in us\n"
+ "# TYPE iscsi_target_dma_time counter\n");
+ CTLSTAT_PROMETHEUS_TIMELOOP(dma_time);
+
+ for (i = 0; i < targdata.ntargets; i++)
+ free(targdata.targets[i]);
+ free(targdata.target);
+ free(targdata.targets);
+
+ fflush(stdout);
+}
+
static void
ctlstat_standard(struct ctlstat_context *ctx) {
long double etime;
@@ -659,6 +867,9 @@ main(int argc, char **argv)
ctx.flags |= CTLSTAT_FLAG_PORTS;
break;
}
+ case 'P':
+ ctx.mode = CTLSTAT_MODE_PROMETHEUS;
+ break;
case 't':
ctx.flags |= CTLSTAT_FLAG_TOTALS;
break;
@@ -676,6 +887,17 @@ main(int argc, char **argv)
if (F_LUNS(&ctx) && F_PORTS(&ctx))
errx(1, "Options -p and -l are exclusive.");
+ if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) {
+ if ((count != -1) ||
+ (waittime != 1) ||
+ /* NB: -P could be compatible with -t in the future */
+ (ctx.flags & CTLSTAT_FLAG_TOTALS))
+ {
+ errx(1, "Option -P is exclusive with -c, -w, and -t");
+ }
+ count = 1;
+ }
+
if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) {
if (F_TOTALS(&ctx))
ctx.flags |= CTLSTAT_FLAG_PORTS;
@@ -712,6 +934,9 @@ main(int argc, char **argv)
case CTLSTAT_MODE_JSON:
ctlstat_json(&ctx);
break;
+ case CTLSTAT_MODE_PROMETHEUS:
+ ctlstat_prometheus(fd, &ctx);
+ break;
default:
break;
}
diff --git a/usr.sbin/inetd/inetd.conf b/usr.sbin/inetd/inetd.conf
index 82f68be7ab5d..09dfbab66496 100644
--- a/usr.sbin/inetd/inetd.conf
+++ b/usr.sbin/inetd/inetd.conf
@@ -120,6 +120,9 @@
#
#prom-sysctl stream tcp nowait nobody /usr/sbin/prometheus_sysctl_exporter prometheus_sysctl_exporter -dgh
#
+# Example entry for the CTL exporter
+#prom-ctl stream tcp nowait root /usr/bin/ctlstat ctlstat -P
+#
# Example entry for insecure rsync server
# This is best combined with encrypted virtual tunnel interfaces, which can be
# found with: apropos if_ | grep tunnel
diff --git a/usr.sbin/services_mkdb/services b/usr.sbin/services_mkdb/services
index d1f0596c958c..71dda5b8d80f 100644
--- a/usr.sbin/services_mkdb/services
+++ b/usr.sbin/services_mkdb/services
@@ -2000,6 +2000,7 @@ bacula-sd 9103/udp #Bacula Storage Daemon
prom-sysctl 9124/tcp #prometheus_sysctl_exporter(8)
git 9418/tcp #git pack transfer service
git 9418/udp #git pack transfer service
+prom-ctl 9572/tcp #CTL prometheus
odbcpathway 9628/tcp #ODBC Pathway Service
odbcpathway 9628/udp #ODBC Pathway Service
davsrc 9800/tcp #WebDav Source Port