git: 7b02c1e8c6a1 - main - iscsi: Fetch limits based on a socket rather than assuming global limits.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 18 Apr 2022 19:53:45 UTC
The branch main has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=7b02c1e8c6a17d11b69988f3e1b449b783785415
commit 7b02c1e8c6a17d11b69988f3e1b449b783785415
Author: John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2022-04-18 19:48:42 +0000
Commit: John Baldwin <jhb@FreeBSD.org>
CommitDate: 2022-04-18 19:53:28 +0000
iscsi: Fetch limits based on a socket rather than assuming global limits.
cxgbei needs the ability to return different limits based on the
connection (e.g. if the connection is over a T5 adapter or a T6
adapter as well as factoring in the MTU).
This change plumbs through the changes in the ioctls without changing
any of the backends. The limits callback passed to icl_register now
accepts a second socket argument which holds the integer file
descriptor. To support ABI compatiblity for old binaries, the
callback should return "global" values if the socket fd is zero.
The CTL_ISCSI_LIMITS argument used with CTL_ISCSI by ctld(8) now
accepts the socket fd in a field that was previously part of a
reserved spare field. Old binaries zero this request which results in
passing a socket fd of 0 to the limits callback.
The ISCSIDREQUEST ioctl no longer returns limits. Instead, iscsid(8)
invokes a new ISCSIDLIMITS ioctl after establishing the connection via
connect(2). For ABI compat, if the old ISCSIDREQUEST is invoked, the
global limits are still fetched (with a socket fd of 0) and returned.
Reviewed by: mav
Sponsored by: Chelsio Communications
Differential Revision: https://reviews.freebsd.org/D34928
---
sys/cam/ctl/ctl_frontend_iscsi.c | 2 +-
sys/cam/ctl/ctl_ioctl.h | 5 +-
sys/dev/cxgbe/cxgbei/icl_cxgbei.c | 2 +-
sys/dev/iscsi/icl.c | 10 ++--
sys/dev/iscsi/icl.h | 4 +-
sys/dev/iscsi/icl_soft.c | 2 +-
sys/dev/iscsi/iscsi.c | 110 +++++++++++++++++++++++++++++++-------
sys/dev/iscsi/iscsi_ioctl.h | 8 ++-
sys/dev/iser/icl_iser.c | 2 +-
usr.sbin/ctld/ctld.h | 2 +-
usr.sbin/ctld/kernel.c | 3 +-
usr.sbin/ctld/login.c | 1 +
usr.sbin/iscsid/iscsid.c | 87 ++++++++++++++++++------------
13 files changed, 170 insertions(+), 68 deletions(-)
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index cccb3fd90317..b78c6a7d8441 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -1896,7 +1896,7 @@ cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
- error = icl_limits(cilp->offload, false, &idl);
+ error = icl_limits(cilp->offload, false, cilp->socket, &idl);
if (error != 0) {
ci->status = CTL_ISCSI_ERROR;
snprintf(ci->error_str, sizeof(ci->error_str),
diff --git a/sys/cam/ctl/ctl_ioctl.h b/sys/cam/ctl/ctl_ioctl.h
index 49c48afbd766..5cb7f29d82a7 100644
--- a/sys/cam/ctl/ctl_ioctl.h
+++ b/sys/cam/ctl/ctl_ioctl.h
@@ -669,9 +669,12 @@ struct ctl_iscsi_terminate_params {
struct ctl_iscsi_limits_params {
/* passed to kernel */
char offload[CTL_ISCSI_OFFLOAD_LEN];
+ int socket;
/* passed to userland */
- size_t spare;
+#ifdef __LP64__
+ int spare;
+#endif
int max_recv_data_segment_length;
int max_send_data_segment_length;
int max_burst_length;
diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
index a3dbf4cb88a2..f0189372bd03 100644
--- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
@@ -1773,7 +1773,7 @@ cxgbei_limits(struct adapter *sc, void *arg)
}
static int
-icl_cxgbei_limits(struct icl_drv_limits *idl)
+icl_cxgbei_limits(struct icl_drv_limits *idl, int socket)
{
/* Maximum allowed by the RFC. cxgbei_limits will clip them. */
diff --git a/sys/dev/iscsi/icl.c b/sys/dev/iscsi/icl.c
index 1e1f1bef91bb..1a86474a5033 100644
--- a/sys/dev/iscsi/icl.c
+++ b/sys/dev/iscsi/icl.c
@@ -60,7 +60,8 @@ struct icl_module {
char *im_name;
bool im_iser;
int im_priority;
- int (*im_limits)(struct icl_drv_limits *idl);
+ int (*im_limits)(struct icl_drv_limits *idl,
+ int socket);
struct icl_conn *(*im_new_conn)(const char *name,
struct mtx *lock);
};
@@ -184,7 +185,8 @@ icl_new_conn(const char *offload, bool iser, const char *name, struct mtx *lock)
}
int
-icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
+icl_limits(const char *offload, bool iser, int socket,
+ struct icl_drv_limits *idl)
{
struct icl_module *im;
int error;
@@ -197,7 +199,7 @@ icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
return (ENXIO);
}
- error = im->im_limits(idl);
+ error = im->im_limits(idl, socket);
sx_sunlock(&sc->sc_lock);
/*
@@ -232,7 +234,7 @@ icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
int
icl_register(const char *offload, bool iser, int priority,
- int (*limits)(struct icl_drv_limits *),
+ int (*limits)(struct icl_drv_limits *, int),
struct icl_conn *(*new_conn)(const char *, struct mtx *))
{
struct icl_module *im;
diff --git a/sys/dev/iscsi/icl.h b/sys/dev/iscsi/icl.h
index edd43a45ba2e..59b1e2aacc96 100644
--- a/sys/dev/iscsi/icl.h
+++ b/sys/dev/iscsi/icl.h
@@ -137,10 +137,10 @@ typedef void (*icl_pdu_cb)(struct icl_pdu *, int error);
struct icl_conn *icl_new_conn(const char *offload, bool iser, const char *name,
struct mtx *lock);
-int icl_limits(const char *offload, bool iser,
+int icl_limits(const char *offload, bool iser, int socket,
struct icl_drv_limits *idl);
int icl_register(const char *offload, bool iser, int priority,
- int (*limits)(struct icl_drv_limits *),
+ int (*limits)(struct icl_drv_limits *, int),
struct icl_conn *(*new_conn)(const char *, struct mtx *));
int icl_unregister(const char *offload, bool rdma);
diff --git a/sys/dev/iscsi/icl_soft.c b/sys/dev/iscsi/icl_soft.c
index c8adad7e9d03..669cb9618d3a 100644
--- a/sys/dev/iscsi/icl_soft.c
+++ b/sys/dev/iscsi/icl_soft.c
@@ -1652,7 +1652,7 @@ icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv)
}
static int
-icl_soft_limits(struct icl_drv_limits *idl)
+icl_soft_limits(struct icl_drv_limits *idl, int socket)
{
idl->idl_max_recv_data_segment_length = max_data_segment_length;
diff --git a/sys/dev/iscsi/iscsi.c b/sys/dev/iscsi/iscsi.c
index 9535aa9b9e2a..f7a585c02545 100644
--- a/sys/dev/iscsi/iscsi.c
+++ b/sys/dev/iscsi/iscsi.c
@@ -78,6 +78,20 @@ __FBSDID("$FreeBSD$");
FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
#endif
+#ifdef COMPAT_FREEBSD13
+struct iscsi_daemon_request13 {
+ unsigned int idr_session_id;
+ struct iscsi_session_conf idr_conf;
+ uint8_t idr_isid[6];
+ uint16_t idr_tsih;
+ uint16_t idr_spare_cid;
+ struct iscsi_session_limits idr_limits;
+ int idr_spare[4];
+};
+
+#define ISCSIDWAIT13 _IOR('I', 0x01, struct iscsi_daemon_request13)
+#endif
+
/*
* XXX: This is global so the iscsi_unload() can access it.
* Think about how to do this properly.
@@ -1423,10 +1437,9 @@ iscsi_pdu_handle_reject(struct icl_pdu *response)
static int
iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
- struct iscsi_daemon_request *request)
+ struct iscsi_daemon_request *request, bool freebsd13)
{
struct iscsi_session *is;
- struct icl_drv_limits idl;
int error;
sx_slock(&sc->sc_lock);
@@ -1472,29 +1485,78 @@ iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
memcpy(&request->idr_conf, &is->is_conf,
sizeof(request->idr_conf));
- error = icl_limits(is->is_conf.isc_offload,
- is->is_conf.isc_iser, &idl);
- if (error != 0) {
- ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
- "failed with error %d", is->is_conf.isc_offload,
- error);
- sx_sunlock(&sc->sc_lock);
- return (error);
- }
- request->idr_limits.isl_max_recv_data_segment_length =
- idl.idl_max_recv_data_segment_length;
- request->idr_limits.isl_max_send_data_segment_length =
- idl.idl_max_send_data_segment_length;
- request->idr_limits.isl_max_burst_length =
- idl.idl_max_burst_length;
- request->idr_limits.isl_first_burst_length =
- idl.idl_first_burst_length;
+#ifdef COMPAT_FREEBSD13
+ if (freebsd13) {
+ struct icl_drv_limits idl;
+ struct iscsi_daemon_request13 *request13;
+ error = icl_limits(is->is_conf.isc_offload,
+ is->is_conf.isc_iser, 0, &idl);
+ if (error != 0) {
+ ISCSI_SESSION_WARN(is, "icl_limits for "
+ "offload \"%s\" failed with error %d",
+ is->is_conf.isc_offload, error);
+ sx_sunlock(&sc->sc_lock);
+ return (error);
+ }
+ request13 = (struct iscsi_daemon_request13 *)request;
+ request13->idr_limits.isl_max_recv_data_segment_length =
+ idl.idl_max_recv_data_segment_length;
+ request13->idr_limits.isl_max_send_data_segment_length =
+ idl.idl_max_send_data_segment_length;
+ request13->idr_limits.isl_max_burst_length =
+ idl.idl_max_burst_length;
+ request13->idr_limits.isl_first_burst_length =
+ idl.idl_first_burst_length;
+ }
+#endif
sx_sunlock(&sc->sc_lock);
return (0);
}
}
+static int
+iscsi_ioctl_daemon_limits(struct iscsi_softc *sc,
+ struct iscsi_daemon_limits *limits)
+{
+ struct icl_drv_limits idl;
+ struct iscsi_session *is;
+ int error;
+
+ sx_slock(&sc->sc_lock);
+
+ /*
+ * Find the session to fetch limits for.
+ */
+ TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
+ if (is->is_id == limits->idl_session_id)
+ break;
+ }
+ if (is == NULL) {
+ sx_sunlock(&sc->sc_lock);
+ return (ESRCH);
+ }
+
+ error = icl_limits(is->is_conf.isc_offload, is->is_conf.isc_iser,
+ limits->idl_socket, &idl);
+ sx_sunlock(&sc->sc_lock);
+ if (error != 0) {
+ ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
+ "failed with error %d", is->is_conf.isc_offload, error);
+ return (error);
+ }
+ limits->idl_limits.isl_max_recv_data_segment_length =
+ idl.idl_max_recv_data_segment_length;
+ limits->idl_limits.isl_max_send_data_segment_length =
+ idl.idl_max_send_data_segment_length;
+ limits->idl_limits.isl_max_burst_length =
+ idl.idl_max_burst_length;
+ limits->idl_limits.isl_first_burst_length =
+ idl.idl_first_burst_length;
+
+ return (0);
+}
+
static int
iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
struct iscsi_daemon_handoff *handoff)
@@ -2179,7 +2241,15 @@ iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
switch (cmd) {
case ISCSIDWAIT:
return (iscsi_ioctl_daemon_wait(sc,
- (struct iscsi_daemon_request *)arg));
+ (struct iscsi_daemon_request *)arg, false));
+#ifdef COMPAT_FREEBSD13
+ case ISCSIDWAIT13:
+ return (iscsi_ioctl_daemon_wait(sc,
+ (struct iscsi_daemon_request *)arg, true));
+#endif
+ case ISCSIDLIMITS:
+ return (iscsi_ioctl_daemon_limits(sc,
+ (struct iscsi_daemon_limits *)arg));
case ISCSIDHANDOFF:
return (iscsi_ioctl_daemon_handoff(sc,
(struct iscsi_daemon_handoff *)arg));
diff --git a/sys/dev/iscsi/iscsi_ioctl.h b/sys/dev/iscsi/iscsi_ioctl.h
index c1de089c9d3f..caf403b55970 100644
--- a/sys/dev/iscsi/iscsi_ioctl.h
+++ b/sys/dev/iscsi/iscsi_ioctl.h
@@ -117,10 +117,15 @@ struct iscsi_daemon_request {
uint8_t idr_isid[6];
uint16_t idr_tsih;
uint16_t idr_spare_cid;
- struct iscsi_session_limits idr_limits;
int idr_spare[4];
};
+struct iscsi_daemon_limits {
+ unsigned int idl_session_id;
+ int idl_socket;
+ struct iscsi_session_limits idl_limits;
+};
+
struct iscsi_daemon_handoff {
unsigned int idh_session_id;
int idh_socket;
@@ -150,6 +155,7 @@ struct iscsi_daemon_fail {
#define ISCSIDWAIT _IOR('I', 0x01, struct iscsi_daemon_request)
#define ISCSIDHANDOFF _IOW('I', 0x02, struct iscsi_daemon_handoff)
#define ISCSIDFAIL _IOW('I', 0x03, struct iscsi_daemon_fail)
+#define ISCSIDLIMITS _IOWR('I', 0x07, struct iscsi_daemon_limits)
#ifdef ICL_KERNEL_PROXY
diff --git a/sys/dev/iser/icl_iser.c b/sys/dev/iser/icl_iser.c
index 140b5622385d..d43c1dc3b885 100644
--- a/sys/dev/iser/icl_iser.c
+++ b/sys/dev/iser/icl_iser.c
@@ -511,7 +511,7 @@ iser_conn_task_done(struct icl_conn *ic, void *prv)
}
static int
-iser_limits(struct icl_drv_limits *idl)
+iser_limits(struct icl_drv_limits *idl, int socket)
{
idl->idl_max_recv_data_segment_length = 128 * 1024;
diff --git a/usr.sbin/ctld/ctld.h b/usr.sbin/ctld/ctld.h
index 293f5378592f..de2f480af30b 100644
--- a/usr.sbin/ctld/ctld.h
+++ b/usr.sbin/ctld/ctld.h
@@ -356,7 +356,7 @@ int kernel_lun_add(struct lun *lun);
int kernel_lun_modify(struct lun *lun);
int kernel_lun_remove(struct lun *lun);
void kernel_handoff(struct ctld_connection *conn);
-void kernel_limits(const char *offload,
+void kernel_limits(const char *offload, int s,
int *max_recv_data_segment_length,
int *max_send_data_segment_length,
int *max_burst_length,
diff --git a/usr.sbin/ctld/kernel.c b/usr.sbin/ctld/kernel.c
index dbbd7c35f2d9..7e5d2a386878 100644
--- a/usr.sbin/ctld/kernel.c
+++ b/usr.sbin/ctld/kernel.c
@@ -954,7 +954,7 @@ kernel_handoff(struct ctld_connection *conn)
}
void
-kernel_limits(const char *offload, int *max_recv_dsl, int *max_send_dsl,
+kernel_limits(const char *offload, int s, int *max_recv_dsl, int *max_send_dsl,
int *max_burst_length, int *first_burst_length)
{
struct ctl_iscsi req;
@@ -967,6 +967,7 @@ kernel_limits(const char *offload, int *max_recv_dsl, int *max_send_dsl,
if (offload != NULL) {
strlcpy(cilp->offload, offload, sizeof(cilp->offload));
}
+ cilp->socket = s;
if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
log_err(1, "error issuing CTL_ISCSI ioctl; "
diff --git a/usr.sbin/ctld/login.c b/usr.sbin/ctld/login.c
index 19fab3bc494c..c52733f7894a 100644
--- a/usr.sbin/ctld/login.c
+++ b/usr.sbin/ctld/login.c
@@ -698,6 +698,7 @@ login_negotiate(struct ctld_connection *conn, struct pdu *request)
conn->conn_max_burst_limit = (1 << 24) - 1;
conn->conn_first_burst_limit = (1 << 24) - 1;
kernel_limits(conn->conn_portal->p_portal_group->pg_offload,
+ conn->conn.conn_socket,
&conn->conn_max_recv_data_segment_limit,
&conn->conn_max_send_data_segment_limit,
&conn->conn_max_burst_limit,
diff --git a/usr.sbin/iscsid/iscsid.c b/usr.sbin/iscsid/iscsid.c
index 5555ffccbf7c..c81692948c1a 100644
--- a/usr.sbin/iscsid/iscsid.c
+++ b/usr.sbin/iscsid/iscsid.c
@@ -225,7 +225,6 @@ static struct iscsid_connection *
connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
{
struct iscsid_connection *conn;
- struct iscsi_session_limits *isl;
struct addrinfo *from_ai, *to_ai;
const char *from_addr, *to_addr;
#ifdef ICL_KERNEL_PROXY
@@ -249,38 +248,6 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
sizeof(conn->conn.conn_isid));
conn->conn.conn_tsih = request->idr_tsih;
- /*
- * Read the driver limits and provide reasonable defaults for the ones
- * the driver doesn't care about. If a max_snd_dsl is not explicitly
- * provided by the driver then we'll make sure both conn->max_snd_dsl
- * and isl->max_snd_dsl are set to the rcv_dsl. This preserves historic
- * behavior.
- */
- isl = &conn->conn_limits;
- memcpy(isl, &request->idr_limits, sizeof(*isl));
- if (isl->isl_max_recv_data_segment_length == 0)
- isl->isl_max_recv_data_segment_length = (1 << 24) - 1;
- if (isl->isl_max_send_data_segment_length == 0)
- isl->isl_max_send_data_segment_length =
- isl->isl_max_recv_data_segment_length;
- if (isl->isl_max_burst_length == 0)
- isl->isl_max_burst_length = (1 << 24) - 1;
- if (isl->isl_first_burst_length == 0)
- isl->isl_first_burst_length = (1 << 24) - 1;
- if (isl->isl_first_burst_length > isl->isl_max_burst_length)
- isl->isl_first_burst_length = isl->isl_max_burst_length;
-
- /*
- * Limit default send length in case it won't be negotiated.
- * We can't do it for other limits, since they may affect both
- * sender and receiver operation, and we must obey defaults.
- */
- if (conn->conn.conn_max_send_data_segment_length >
- isl->isl_max_send_data_segment_length) {
- conn->conn.conn_max_send_data_segment_length =
- isl->isl_max_send_data_segment_length;
- }
-
from_addr = conn->conn_conf.isc_initiator_addr;
to_addr = conn->conn_conf.isc_target_addr;
@@ -436,6 +403,56 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
return (conn);
}
+static void
+limits(struct iscsid_connection *conn)
+{
+ struct iscsi_daemon_limits idl;
+ struct iscsi_session_limits *isl;
+ int error;
+
+ log_debugx("fetching limits from the kernel");
+
+ memset(&idl, 0, sizeof(idl));
+ idl.idl_session_id = conn->conn_session_id;
+ idl.idl_socket = conn->conn.conn_socket;
+
+ error = ioctl(conn->conn_iscsi_fd, ISCSIDLIMITS, &idl);
+ if (error != 0)
+ log_err(1, "ISCSIDLIMITS");
+
+ /*
+ * Read the driver limits and provide reasonable defaults for the ones
+ * the driver doesn't care about. If a max_snd_dsl is not explicitly
+ * provided by the driver then we'll make sure both conn->max_snd_dsl
+ * and isl->max_snd_dsl are set to the rcv_dsl. This preserves historic
+ * behavior.
+ */
+ isl = &conn->conn_limits;
+ memcpy(isl, &idl.idl_limits, sizeof(*isl));
+ if (isl->isl_max_recv_data_segment_length == 0)
+ isl->isl_max_recv_data_segment_length = (1 << 24) - 1;
+ if (isl->isl_max_send_data_segment_length == 0)
+ isl->isl_max_send_data_segment_length =
+ isl->isl_max_recv_data_segment_length;
+ if (isl->isl_max_burst_length == 0)
+ isl->isl_max_burst_length = (1 << 24) - 1;
+ if (isl->isl_first_burst_length == 0)
+ isl->isl_first_burst_length = (1 << 24) - 1;
+ if (isl->isl_first_burst_length > isl->isl_max_burst_length)
+ isl->isl_first_burst_length = isl->isl_max_burst_length;
+
+ /*
+ * Limit default send length in case it won't be negotiated.
+ * We can't do it for other limits, since they may affect both
+ * sender and receiver operation, and we must obey defaults.
+ */
+ if (conn->conn.conn_max_send_data_segment_length >
+ isl->isl_max_send_data_segment_length) {
+ conn->conn.conn_max_send_data_segment_length =
+ isl->isl_max_send_data_segment_length;
+ }
+}
+
static void
handoff(struct iscsid_connection *conn)
{
@@ -502,6 +519,7 @@ capsicate(struct iscsid_connection *conn)
ISCSIDSEND,
ISCSIDRECEIVE,
#endif
+ ISCSIDLIMITS,
ISCSIDHANDOFF,
ISCSIDFAIL,
ISCSISADD,
@@ -626,8 +644,9 @@ handle_request(int iscsi_fd, const struct iscsi_daemon_request *request, int tim
}
conn = connection_new(iscsi_fd, request);
- set_timeout(timeout);
capsicate(conn);
+ limits(conn);
+ set_timeout(timeout);
login(conn);
if (conn->conn_conf.isc_discovery != 0)
discovery(conn);