git: c7ec839e301d - stable/13 - mlx5/mlx5en: Add SQ remap support
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 08 Feb 2022 15:14:01 UTC
The branch stable/13 has been updated by hselasky:
URL: https://cgit.FreeBSD.org/src/commit/?id=c7ec839e301dc6db99c157dfff9d8ad582f2ea0d
commit c7ec839e301dc6db99c157dfff9d8ad582f2ea0d
Author: Hans Petter Selasky <hselasky@FreeBSD.org>
AuthorDate: 2022-02-08 15:08:53 +0000
Commit: Hans Petter Selasky <hselasky@FreeBSD.org>
CommitDate: 2022-02-08 15:08:53 +0000
mlx5/mlx5en: Add SQ remap support
Add support to map an SQ to a specific schedule queue using a
special WQE as performance enhancement.
SQ remap operation is handled by a privileged internal queue, IQ,
and the mapping is enabled from one rate to another.
The transition from paced to non-paced should however always go
through FW.
Sponsored by: NVIDIA Networking
(cherry picked from commit 266c81aae38adec45effaa67dfec5cd06996f9d3)
---
sys/dev/mlx5/device.h | 1 +
sys/dev/mlx5/driver.h | 12 +++++
sys/dev/mlx5/mlx5_core/mlx5_rl.c | 29 ++++++++++
sys/dev/mlx5/mlx5_en/en.h | 6 +++
sys/dev/mlx5/mlx5_en/mlx5_en_iq.c | 10 ++++
sys/dev/mlx5/mlx5_en/mlx5_en_rl.c | 109 ++++++++++++++++++++++++++++++++++++--
sys/dev/mlx5/mlx5_ifc.h | 61 ++++++++++++++++++---
sys/dev/mlx5/qp.h | 7 +++
8 files changed, 225 insertions(+), 10 deletions(-)
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index e59fb6771d83..c35ef6f2ff0c 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -390,6 +390,7 @@ enum {
MLX5_OPCODE_RCHECK_PSV = 0x27,
MLX5_OPCODE_UMR = 0x25,
+ MLX5_OPCODE_QOS_REMAP = 0x2a,
MLX5_OPCODE_SIGNATURE_CANCELED = (1 << 15),
};
diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
index 0bb2621fec6b..6d3a3be6562e 100644
--- a/sys/dev/mlx5/driver.h
+++ b/sys/dev/mlx5/driver.h
@@ -50,6 +50,8 @@
#define MLX5_QCOUNTER_SETS_NETDEV 64
#define MLX5_MAX_NUMBER_OF_VFS 128
+#define MLX5_INVALID_QUEUE_HANDLE 0xffffffff
+
enum {
MLX5_BOARD_ID_LEN = 64,
MLX5_MAX_NAME_LEN = 16,
@@ -549,6 +551,7 @@ struct mlx5_rl_entry {
u32 rate;
u16 burst;
u16 index;
+ u32 qos_handle; /* schedule queue handle */
u32 refcount;
};
@@ -1193,6 +1196,15 @@ void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev);
int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u32 burst, u16 *index);
void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, u32 rate, u32 burst);
bool mlx5_rl_is_in_range(const struct mlx5_core_dev *dev, u32 rate, u32 burst);
+int mlx5e_query_rate_limit_cmd(struct mlx5_core_dev *dev, u16 index, u32 *scq_handle);
+
+static inline u32 mlx5_rl_get_scq_handle(struct mlx5_core_dev *dev, uint16_t index)
+{
+ KASSERT(index > 0,
+ ("invalid rate index for sq remap, failed retrieving SCQ handle"));
+
+ return (dev->priv.rl_table.rl_entry[index - 1].qos_handle);
+}
static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev)
{
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_rl.c b/sys/dev/mlx5/mlx5_core/mlx5_rl.c
index aa8f351e0fc6..d522e72d0442 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_rl.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_rl.c
@@ -73,6 +73,25 @@ static int mlx5_set_rate_limit_cmd(struct mlx5_core_dev *dev,
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
+int mlx5e_query_rate_limit_cmd(struct mlx5_core_dev *dev,
+ u16 index, u32 *scq_handle)
+{
+ int err;
+ u32 in[MLX5_ST_SZ_DW(query_pp_rate_limit_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(query_pp_rate_limit_out)] = {};
+
+ MLX5_SET(query_pp_rate_limit_in, in, opcode, MLX5_CMD_OP_QUERY_RATE_LIMIT);
+ MLX5_SET(query_pp_rate_limit_in, in, rate_limit_index, index);
+
+ err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ if (err)
+ return err;
+
+ *scq_handle = MLX5_GET(query_pp_rate_limit_out, out, pp_context.qos_handle);
+
+ return 0;
+}
+
bool mlx5_rl_is_in_range(const struct mlx5_core_dev *dev, u32 rate, u32 burst)
{
const struct mlx5_rl_table *table = &dev->priv.rl_table;
@@ -122,6 +141,16 @@ int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u32 burst, u16 *index)
entry->rate = rate;
entry->burst = burst;
entry->refcount = 1;
+
+ if (MLX5_CAP_QOS(dev, qos_remap_pp)) {
+ err = mlx5e_query_rate_limit_cmd(dev, entry->index, &entry->qos_handle);
+ if (err) {
+ mlx5_core_err(dev, "Failed retrieving schedule queue handle for"
+ "SQ remap: rate: %u error:(%d)\n", rate, err);
+ entry->qos_handle = MLX5_INVALID_QUEUE_HANDLE;
+ }
+ } else
+ entry->qos_handle = MLX5_INVALID_QUEUE_HANDLE;
}
*index = entry->index;
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index bc3bd11085a3..8267835bf7dd 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -880,6 +880,7 @@ struct mlx5e_sq {
u16 max_inline;
u8 min_inline_mode;
u8 min_insert_caps;
+ u32 queue_handle; /* SQ remap support */
#define MLX5E_INSERT_VLAN 1
#define MLX5E_INSERT_NON_VLAN 2
@@ -1145,6 +1146,11 @@ struct mlx5e_tx_psv_wqe {
struct mlx5_seg_set_psv psv;
};
+struct mlx5e_tx_qos_remap_wqe {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_qos_remap_seg qos_remap;
+};
+
struct mlx5e_rx_wqe {
struct mlx5_wqe_srq_next_seg next;
struct mlx5_wqe_data_seg data[];
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_iq.c b/sys/dev/mlx5/mlx5_en/mlx5_en_iq.c
index 3bc4959e046f..e375864dc85f 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_iq.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_iq.c
@@ -262,6 +262,16 @@ mlx5e_iq_enable(struct mlx5e_iq *iq, struct mlx5e_sq_param *param,
MLX5_SET(sqc, sqc, flush_in_error_en, 1);
MLX5_SET(sqc, sqc, allow_swp, 1);
+ /* SQ remap support requires reg_umr privileges level */
+ if (MLX5_CAP_QOS(iq->priv->mdev, qos_remap_pp)) {
+ MLX5_SET(sqc, sqc, qos_remap_en, 1);
+ if (MLX5_CAP_ETH(iq->priv->mdev, reg_umr_sq))
+ MLX5_SET(sqc, sqc, reg_umr, 1);
+ else
+ mlx5_en_err(iq->priv->ifp,
+ "No reg umr SQ capability, SQ remap disabled\n");
+ }
+
MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
MLX5_SET(wq, wq, uar_page, bfreg->index);
MLX5_SET(wq, wq, log_wq_pg_sz, iq->wq_ctrl.buf.page_shift -
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
index a5c44dee5fad..8ab118c21249 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
@@ -153,6 +153,29 @@ mlx5e_rl_destroy_sq(struct mlx5e_sq *sq)
bus_dma_tag_destroy(sq->dma_tag);
}
+static int
+mlx5e_rl_query_sq(struct mlx5e_sq *sq)
+{
+ void *out;
+ int inlen;
+ int err;
+
+ inlen = MLX5_ST_SZ_BYTES(query_sq_out);
+ out = mlx5_vzalloc(inlen);
+ if (!out)
+ return -ENOMEM;
+
+ err = mlx5_core_query_sq(sq->priv->mdev, sq->sqn, out);
+ if (err)
+ goto out;
+
+ sq->queue_handle = MLX5_GET(query_sq_out, out, sq_context.queue_handle);
+
+out:
+ kvfree(out);
+ return err;
+}
+
static int
mlx5e_rl_open_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
struct mlx5e_sq_param *param, int ix)
@@ -171,6 +194,16 @@ mlx5e_rl_open_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
if (err)
goto err_disable_sq;
+ if (MLX5_CAP_QOS(priv->mdev, qos_remap_pp)) {
+ err = mlx5e_rl_query_sq(sq);
+ if (err) {
+ mlx5_en_err(priv->ifp, "Failed retrieving send queue handle for"
+ "SQ remap - sqn=%u, err=(%d)\n", sq->sqn, err);
+ sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;
+ }
+ } else
+ sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;
+
WRITE_ONCE(sq->running, 1);
return (0);
@@ -380,6 +413,68 @@ mlx5e_rl_find_best_rate_locked(struct mlx5e_rl_priv_data *rl, uint64_t user_rate
return (retval);
}
+static int
+mlx5e_rl_post_sq_remap_wqe(struct mlx5e_iq *iq, u32 scq_handle, u32 sq_handle)
+{
+ const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_qos_remap_wqe),
+ MLX5_SEND_WQE_DS);
+ struct mlx5e_tx_qos_remap_wqe *wqe;
+ int pi;
+
+ mtx_lock(&iq->lock);
+ pi = mlx5e_iq_get_producer_index(iq);
+ if (pi < 0) {
+ mtx_unlock(&iq->lock);
+ return (-ENOMEM);
+ }
+ wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
+
+ memset(wqe, 0, sizeof(*wqe));
+
+ wqe->qos_remap.qos_handle = cpu_to_be32(scq_handle);
+ wqe->qos_remap.queue_handle = cpu_to_be32(sq_handle);
+
+ wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
+ MLX5_OPCODE_QOS_REMAP);
+ wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
+ wqe->ctrl.imm = cpu_to_be32(iq->priv->tisn[0] << 8);
+ wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
+
+ /* copy data for doorbell */
+ memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
+
+ iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+ iq->pc += iq->data[pi].num_wqebbs;
+
+ mlx5e_iq_notify_hw(iq);
+
+ mtx_unlock(&iq->lock);
+
+ return (0); /* success */
+}
+
+static int
+mlx5e_rl_remap_sq(struct mlx5e_sq *sq, uint16_t index)
+{
+ struct mlx5e_channel *iq_channel;
+ u32 scq_handle;
+ u32 sq_handle;
+ int error;
+
+ /* Specific SQ remap operations should be handled by same IQ */
+ iq_channel = &sq->priv->channel[sq->sqn % sq->priv->params.num_channels];
+
+ sq_handle = sq->queue_handle;
+ scq_handle = mlx5_rl_get_scq_handle(sq->priv->mdev, index);
+
+ if (sq_handle == -1U || scq_handle == -1U)
+ error = -1;
+ else
+ error = mlx5e_rl_post_sq_remap_wqe(&iq_channel->iq, scq_handle, sq_handle);
+
+ return (error);
+}
+
/*
* This function sets the requested rate for a rate limit channel, in
* bits per second. The requested rate will be filtered through the
@@ -395,6 +490,7 @@ mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw,
uint16_t index;
uint16_t burst;
int error;
+ bool use_sq_remap;
if (rate != 0) {
MLX5E_RL_WORKER_UNLOCK(rlw);
@@ -438,6 +534,10 @@ mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw,
burst = 0; /* default */
}
+ /* paced <--> non-paced transitions must go via FW */
+ use_sq_remap = MLX5_CAP_QOS(rlw->priv->mdev, qos_remap_pp) &&
+ channel->last_rate != 0 && rate != 0;
+
/* atomically swap rates */
temp = channel->last_rate;
channel->last_rate = rate;
@@ -458,11 +558,14 @@ mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw,
/* set new rate, if SQ is running */
sq = channel->sq;
if (sq != NULL && READ_ONCE(sq->running) != 0) {
- error = mlx5e_rl_modify_sq(sq, index);
- if (error != 0)
- atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
+ if (!use_sq_remap || mlx5e_rl_remap_sq(sq, index)) {
+ error = mlx5e_rl_modify_sq(sq, index);
+ if (error != 0)
+ atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
+ }
} else
error = 0;
+
MLX5E_RL_WORKER_LOCK(rlw);
return (-error);
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
index f2edbc406138..dc288e350d05 100644
--- a/sys/dev/mlx5/mlx5_ifc.h
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -693,7 +693,9 @@ struct mlx5_ifc_qos_cap_bits {
u8 packet_pacing_typical_size[0x1];
u8 reserved_at_7[0x19];
- u8 reserved_at_20[0x20];
+ u8 reserved_at_20[0xA];
+ u8 qos_remap_pp[0x1];
+ u8 reserved_at_2b[0x15];
u8 packet_pacing_max_rate[0x20];
@@ -883,9 +885,9 @@ struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
u8 multi_pkt_send_wqe[0x2];
u8 wqe_inline_mode[0x2];
u8 rss_ind_tbl_cap[0x4];
- u8 reg_umr_sq[0x1];
+ u8 reg_umr_sq[0x1];
u8 scatter_fcs[0x1];
- u8 enhanced_multi_pkt_send_wqe[0x1];
+ u8 enhanced_multi_pkt_send_wqe[0x1];
u8 tunnel_lso_const_out_ip_id[0x1];
u8 tunnel_lro_gre[0x1];
u8 tunnel_lro_vxlan[0x1];
@@ -2423,7 +2425,9 @@ struct mlx5_ifc_sqc_bits {
u8 state[0x4];
u8 reg_umr[0x1];
u8 allow_swp[0x1];
- u8 reserved_at_e[0xc];
+ u8 reserved_at_e[0x4];
+ u8 qos_remap_en[0x1];
+ u8 reserved_at_d[0x7];
u8 ts_format[0x2];
u8 reserved_at_1c[0x4];
@@ -2439,9 +2443,12 @@ struct mlx5_ifc_sqc_bits {
u8 packet_pacing_rate_limit_index[0x10];
u8 tis_lst_sz[0x10];
- u8 reserved_4[0x10];
+ u8 qos_queue_group_id[0x10];
- u8 reserved_5[0x40];
+ u8 reserved_4[0x8];
+ u8 queue_handle[0x18];
+
+ u8 reserved_5[0x20];
u8 reserved_6[0x8];
u8 tis_num_0[0x18];
@@ -2449,6 +2456,45 @@ struct mlx5_ifc_sqc_bits {
struct mlx5_ifc_wq_bits wq;
};
+struct mlx5_ifc_query_pp_rate_limit_in_bits {
+ u8 opcode[0x10];
+ u8 uid[0x10];
+
+ u8 reserved1[0x10];
+ u8 op_mod[0x10];
+
+ u8 reserved2[0x10];
+ u8 rate_limit_index[0x10];
+
+ u8 reserved_3[0x20];
+};
+
+struct mlx5_ifc_pp_context_bits {
+ u8 rate_limit[0x20];
+
+ u8 burst_upper_bound[0x20];
+
+ u8 reserved_1[0xc];
+ u8 rate_mode[0x4];
+ u8 typical_packet_size[0x10];
+
+ u8 reserved_2[0x8];
+ u8 qos_handle[0x18];
+
+ u8 reserved_3[0x40];
+};
+
+struct mlx5_ifc_query_pp_rate_limit_out_bits {
+ u8 status[0x8];
+ u8 reserved_1[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_2[0x40];
+
+ struct mlx5_ifc_pp_context_bits pp_context;
+};
+
enum {
MLX5_TSAR_TYPE_DWRR = 0,
MLX5_TSAR_TYPE_ROUND_ROUBIN = 1,
@@ -5528,12 +5574,13 @@ struct mlx5_ifc_modify_rqt_out_bits {
};
struct mlx5_ifc_rqt_bitmask_bits {
- u8 reserved_at_0[0x20];
+ u8 reserved_at_0[0x20];
u8 reserved_at_20[0x1f];
u8 rqn_list[0x1];
};
+
struct mlx5_ifc_modify_rqt_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h
index bf9fef73667d..4acfa4820b12 100644
--- a/sys/dev/mlx5/qp.h
+++ b/sys/dev/mlx5/qp.h
@@ -322,6 +322,13 @@ struct mlx5_seg_set_psv {
__be32 ref_tag;
};
+struct mlx5_wqe_qos_remap_seg {
+ u8 rsvd0[4];
+ u8 rsvd1[4];
+ __be32 qos_handle;
+ __be32 queue_handle;
+};
+
struct mlx5_seg_get_psv {
u8 rsvd[19];
u8 num_psv;