svn commit: r291938 - in head/sys: dev/mlx5/mlx5_core dev/mlx5/mlx5_en modules/mlx5 modules/mlx5en
Hans Petter Selasky
hselasky at FreeBSD.org
Mon Dec 7 12:38:53 UTC 2015
Author: hselasky
Date: Mon Dec 7 12:38:51 2015
New Revision: 291938
URL: https://svnweb.freebsd.org/changeset/base/291938
Log:
Add full support for Receive Side Scaling, RSS, to the mlx5en
driver. This includes binding all interrupt and worker threads
according to the RSS configuration, setting up correct Toeplitz
hashing keys as given by RSS and setting the correct mbuf
hashtype for all received traffic.
MFC after: 1 week
Sponsored by: Mellanox Technologies
Differential Revision: https://reviews.freebsd.org/D4410
Modified:
head/sys/dev/mlx5/mlx5_core/mlx5_eq.c
head/sys/dev/mlx5/mlx5_en/en.h
head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
head/sys/modules/mlx5/Makefile
head/sys/modules/mlx5en/Makefile
Modified: head/sys/dev/mlx5/mlx5_core/mlx5_eq.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_eq.c Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_eq.c Mon Dec 7 12:38:51 2015 (r291938)
@@ -31,6 +31,13 @@
#include <dev/mlx5/mlx5_ifc.h>
#include "mlx5_core.h"
+#include "opt_rss.h"
+
+#ifdef RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
enum {
MLX5_EQE_SIZE = sizeof(struct mlx5_eqe),
MLX5_EQE_OWNER_INIT_VAL = 0x1,
@@ -389,6 +396,18 @@ int mlx5_create_map_eq(struct mlx5_core_
priv->irq_info[vecidx].name, eq);
if (err)
goto err_eq;
+#ifdef RSS
+ if (vecidx >= MLX5_EQ_VEC_COMP_BASE) {
+ u8 bucket = vecidx - MLX5_EQ_VEC_COMP_BASE;
+ err = bind_irq_to_cpu(priv->msix_arr[vecidx].vector,
+ rss_getcpu(bucket % rss_getnumbuckets()));
+ if (err)
+ goto err_irq;
+ }
+#else
+ if (0)
+ goto err_irq;
+#endif
/* EQs are created in ARMED state
@@ -398,6 +417,8 @@ int mlx5_create_map_eq(struct mlx5_core_
kvfree(in);
return 0;
+err_irq:
+ free_irq(priv->msix_arr[vecidx].vector, eq);
err_eq:
mlx5_cmd_destroy_eq(dev, eq->eqn);
Modified: head/sys/dev/mlx5/mlx5_en/en.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/en.h Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/dev/mlx5/mlx5_en/en.h Mon Dec 7 12:38:51 2015 (r291938)
@@ -50,6 +50,13 @@
#include <net/ethernet.h>
#include <sys/buf_ring.h>
+#include "opt_rss.h"
+
+#ifdef RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
#include <machine/bus.h>
#ifdef HAVE_TURBO_LRO
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Mon Dec 7 12:38:51 2015 (r291938)
@@ -931,6 +931,10 @@ mlx5e_create_sq(struct mlx5e_channel *c,
void *sqc = param->sqc;
void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
+#ifdef RSS
+ cpuset_t cpu_mask;
+ int cpu_id;
+#endif
int err;
/* Create DMA descriptor TAG */
@@ -991,9 +995,15 @@ mlx5e_create_sq(struct mlx5e_channel *c,
}
TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq);
- taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s tx sq",
- c->ifp->if_xname);
-
+#ifdef RSS
+ cpu_id = rss_getcpu(c->ix % rss_getnumbuckets());
+ CPU_SETOF(cpu_id, &cpu_mask);
+ taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask,
+ "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id);
+#else
+ taskqueue_start_threads(&sq->sq_tq, 1, PI_NET,
+ "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc);
+#endif
snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
@@ -1768,8 +1778,14 @@ mlx5e_open_rqt(struct mlx5e_priv *priv)
MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
for (i = 0; i < sz; i++) {
- int ix = i % priv->params.num_channels;
-
+ int ix;
+#ifdef RSS
+ ix = rss_get_indirection_to_bucket(i);
+#else
+ ix = i;
+#endif
+ /* ensure we don't overflow */
+ ix %= priv->params.num_channels;
MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn);
}
@@ -1834,6 +1850,8 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
MLX5_CAP_ETH(priv->mdev,
lro_timer_supported_periods[2]));
}
+
+ /* setup parameters for hashing TIR type, if any */
switch (tt) {
case MLX5E_TT_ANY:
MLX5_SET(tirc, tirc, disp_type,
@@ -1848,8 +1866,16 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
priv->rqtn);
MLX5_SET(tirc, tirc, rx_hash_fn,
MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
- MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
+#ifdef RSS
+ /*
+ * The FreeBSD RSS implementation does currently not
+ * support symmetric Toeplitz hashes:
+ */
+ MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
+ rss_getkey((uint8_t *)hkey);
+#else
+ MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
hkey[0] = cpu_to_be32(0xD181C62C);
hkey[1] = cpu_to_be32(0xF7F4DB5B);
hkey[2] = cpu_to_be32(0x1983A2FC);
@@ -1860,6 +1886,7 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
hkey[7] = cpu_to_be32(0x593D56D9);
hkey[8] = cpu_to_be32(0xF3253C06);
hkey[9] = cpu_to_be32(0x2ADC1FFC);
+#endif
break;
}
@@ -1869,6 +1896,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
MLX5_L3_PROT_TYPE_IPV4);
MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
MLX5_L4_PROT_TYPE_TCP);
+#ifdef RSS
+ if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+ MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+ MLX5_HASH_IP);
+ } else
+#endif
MLX5_SET(rx_hash_field_select, hfso, selected_fields,
MLX5_HASH_ALL);
break;
@@ -1878,6 +1911,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
MLX5_L3_PROT_TYPE_IPV6);
MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
MLX5_L4_PROT_TYPE_TCP);
+#ifdef RSS
+ if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
+ MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+ MLX5_HASH_IP);
+ } else
+#endif
MLX5_SET(rx_hash_field_select, hfso, selected_fields,
MLX5_HASH_ALL);
break;
@@ -1887,6 +1926,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
MLX5_L3_PROT_TYPE_IPV4);
MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
MLX5_L4_PROT_TYPE_UDP);
+#ifdef RSS
+ if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+ MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+ MLX5_HASH_IP);
+ } else
+#endif
MLX5_SET(rx_hash_field_select, hfso, selected_fields,
MLX5_HASH_ALL);
break;
@@ -1896,6 +1941,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *p
MLX5_L3_PROT_TYPE_IPV6);
MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
MLX5_L4_PROT_TYPE_UDP);
+#ifdef RSS
+ if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
+ MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+ MLX5_HASH_IP);
+ } else
+#endif
MLX5_SET(rx_hash_field_select, hfso, selected_fields,
MLX5_HASH_ALL);
break;
@@ -2052,6 +2103,13 @@ mlx5e_open_locked(struct ifnet *ifp)
if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
return (0);
+#ifdef RSS
+ if (rss_getnumbuckets() > priv->params.num_channels) {
+ if_printf(ifp, "NOTE: There are more RSS buckets(%u) than "
+ "channels(%u) available\n", rss_getnumbuckets(),
+ priv->params.num_channels);
+ }
+#endif
err = mlx5e_open_tises(priv);
if (err) {
if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Mon Dec 7 12:38:51 2015 (r291938)
@@ -192,12 +192,43 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *c
mb->m_pkthdr.len = mb->m_len = cqe_bcnt;
/* check if a Toeplitz hash was computed */
- if (cqe->rss_hash_type != 0)
+ if (cqe->rss_hash_type != 0) {
mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
- else
+#ifdef RSS
+ /* decode the RSS hash type */
+ switch (cqe->rss_hash_type &
+ (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
+ /* IPv4 */
+ case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
+ break;
+ case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
+ break;
+ case CQE_RSS_DST_HTYPE_IPV4:
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
+ break;
+ /* IPv6 */
+ case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
+ break;
+ case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
+ break;
+ case CQE_RSS_DST_HTYPE_IPV6:
+ M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
+ break;
+ default: /* Other */
+ M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+ break;
+ }
+#else
+ M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+#endif
+ } else {
mb->m_pkthdr.flowid = rq->ix;
-
- M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+ M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+ }
mb->m_pkthdr.rcvif = ifp;
if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) &&
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Mon Dec 7 12:38:51 2015 (r291938)
@@ -85,7 +85,15 @@ mlx5e_select_queue(struct ifnet *ifp, st
/* check if flowid is set */
if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
- ch = (mb->m_pkthdr.flowid % 128) % ch;
+#ifdef RSS
+ u32 temp;
+
+ if (rss_hash2bucket(mb->m_pkthdr.flowid,
+ M_HASHTYPE_GET(mb), &temp) == 0)
+ ch = temp % ch;
+ else
+#endif
+ ch = (mb->m_pkthdr.flowid % 128) % ch;
} else {
#if (__FreeBSD_version >= 1100000)
ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |
Modified: head/sys/modules/mlx5/Makefile
==============================================================================
--- head/sys/modules/mlx5/Makefile Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/modules/mlx5/Makefile Mon Dec 7 12:38:51 2015 (r291938)
@@ -24,7 +24,7 @@ mlx5_uar.c \
mlx5_vport.c \
mlx5_wq.c \
device_if.h bus_if.h vnode_if.h pci_if.h \
- opt_inet.h opt_inet6.h opt_random.h
+ opt_inet.h opt_inet6.h opt_random.h opt_rss.h
CFLAGS+= -I${.CURDIR}/../../ofed/include
CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include
Modified: head/sys/modules/mlx5en/Makefile
==============================================================================
--- head/sys/modules/mlx5en/Makefile Mon Dec 7 12:20:26 2015 (r291937)
+++ head/sys/modules/mlx5en/Makefile Mon Dec 7 12:38:51 2015 (r291938)
@@ -10,7 +10,7 @@ mlx5_en_flow_table.c \
mlx5_en_rx.c \
mlx5_en_txrx.c \
device_if.h bus_if.h vnode_if.h pci_if.h \
- opt_inet.h opt_inet6.h
+ opt_inet.h opt_inet6.h opt_rss.h
.if defined(HAVE_TURBO_LRO)
CFLAGS+= -DHAVE_TURBO_LRO
More information about the svn-src-head
mailing list