svn commit: r346677 - in head/sys: dev/cxgbe dev/mlx5/mlx5_en kern netinet netinet6

[Andrew Gallatin]

Author: gallatin
Date: Thu Apr 25 15:37:28 2019
New Revision: 346677
URL: https://svnweb.freebsd.org/changeset/base/346677

Log:
  Track TCP connection's NUMA domain in the inpcb
  
  Drivers can now pass up numa domain information via the
  mbuf numa domain field.  This information is then used
  by TCP syncache_socket() to associate that information
  with the inpcb. The domain information is then fed back
  into transmitted mbufs in ip{6}_output(). This mechanism
  is nearly identical to what is done to track RSS hash values
  in the inp_flowid.
  
  Follow on changes will use this information for lacp egress
  port selection, binding TCP pacers to the appropriate NUMA
  domain, etc.
  
  Reviewed by:	markj, kib, slavash, bz, scottl, jtl, tuexen
  Sponsored by:	Netflix
  Differential Revision:	https://reviews.freebsd.org/D20028

Modified:
  head/sys/dev/cxgbe/t4_sge.c
  head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
  head/sys/kern/uipc_mbuf.c
  head/sys/netinet/in_pcb.c
  head/sys/netinet/in_pcb.h
  head/sys/netinet/ip_output.c
  head/sys/netinet/tcp_syncache.c
  head/sys/netinet6/ip6_output.c

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/dev/cxgbe/t4_sge.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -2046,6 +2046,9 @@ t4_eth_rx(struct sge_iq *iq, const struct rss_header *
 		rxq->vlan_extraction++;
 	}
 
+#ifdef NUMA
+	m0->m_pkthdr.numa_domain = ifp->if_numa_domain;
+#endif
 #if defined(INET) || defined(INET6)
 	if (iq->flags & IQ_LRO_ENABLED) {
 		if (sort_before_lro(lro)) {

Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -520,6 +520,9 @@ rx_common:
 		mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
 		rq->stats.bytes += byte_cnt;
 		rq->stats.packets++;
+#ifdef NUMA
+		mb->m_pkthdr.numa_domain = rq->ifp->if_numa_domain;
+#endif
 
 #if !defined(HAVE_TCP_LRO_RX)
 		tcp_lro_queue_mbuf(&rq->lro, mb);

Modified: head/sys/kern/uipc_mbuf.c
==============================================================================
--- head/sys/kern/uipc_mbuf.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/kern/uipc_mbuf.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -341,6 +341,9 @@ m_pkthdr_init(struct mbuf *m, int how)
 #endif
 	m->m_data = m->m_pktdat;
 	bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
+#ifdef NUMA
+	m->m_pkthdr.numa_domain = M_NODOM;
+#endif
 #ifdef MAC
 	/* If the label init fails, fail the alloc */
 	error = mac_mbuf_init(m, how);

Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/netinet/in_pcb.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -510,6 +510,9 @@ in_pcballoc(struct socket *so, struct inpcbinfo *pcbin
 	if (inp == NULL)
 		return (ENOBUFS);
 	bzero(&inp->inp_start_zero, inp_zero_size);
+#ifdef NUMA
+	inp->inp_numa_domain = M_NODOM;
+#endif
 	inp->inp_pcbinfo = pcbinfo;
 	inp->inp_socket = so;
 	inp->inp_cred = crhold(so->so_cred);

Modified: head/sys/netinet/in_pcb.h
==============================================================================
--- head/sys/netinet/in_pcb.h	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/netinet/in_pcb.h	Thu Apr 25 15:37:28 2019	(r346677)
@@ -272,7 +272,7 @@ struct inpcb {
 			 inp_hpts_calls :1,	/* (i) from output hpts */
 			 inp_input_calls :1,	/* (i) from input hpts */
 			 inp_spare_bits2 : 4;
-	uint8_t inp_spare_byte;		/* Compiler hole */
+	uint8_t inp_numa_domain;	/* numa domain */
 	void	*inp_ppcb;		/* (i) pointer to per-protocol pcb */
 	struct	socket *inp_socket;	/* (i) back pointer to socket */
 	uint32_t 	 inp_hptsslot;	/* Hpts wheel slot this tcb is Lock(i&b) */

Modified: head/sys/netinet/ip_output.c
==============================================================================
--- head/sys/netinet/ip_output.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/netinet/ip_output.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -247,6 +247,9 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct rou
 			m->m_pkthdr.flowid = inp->inp_flowid;
 			M_HASHTYPE_SET(m, inp->inp_flowtype);
 		}
+#ifdef NUMA
+		m->m_pkthdr.numa_domain = inp->inp_numa_domain;
+#endif
 	}
 
 	if (ro == NULL) {

Modified: head/sys/netinet/tcp_syncache.c
==============================================================================
--- head/sys/netinet/tcp_syncache.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/netinet/tcp_syncache.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -777,6 +777,9 @@ syncache_socket(struct syncache *sc, struct socket *ls
 	if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
 		inp->inp_flowid = m->m_pkthdr.flowid;
 		inp->inp_flowtype = M_HASHTYPE_GET(m);
+#ifdef NUMA
+		inp->inp_numa_domain = m->m_pkthdr.numa_domain;
+#endif
 	}
 
 	/*

Modified: head/sys/netinet6/ip6_output.c
==============================================================================
--- head/sys/netinet6/ip6_output.c	Thu Apr 25 15:31:35 2019	(r346676)
+++ head/sys/netinet6/ip6_output.c	Thu Apr 25 15:37:28 2019	(r346677)
@@ -322,6 +322,9 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
 			m->m_pkthdr.flowid = inp->inp_flowid;
 			M_HASHTYPE_SET(m, inp->inp_flowtype);
 		}
+#ifdef NUMA
+		m->m_pkthdr.numa_domain = inp->inp_numa_domain;
+#endif
 	}
 
 #if defined(IPSEC) || defined(IPSEC_SUPPORT)