svn commit: r260070 - head/sys/net

[Scott Long]

Author: scottl
Date: Mon Dec 30 01:32:17 2013
New Revision: 260070
URL: http://svnweb.freebsd.org/changeset/base/260070

Log:
  Multi-queue NIC drivers and multi-port lagg tend to use the same lower
  bits of the flowid as each other, resulting in a poor distribution of
  packets among queues in certain cases.  Work around this by adding a
  set of sysctls for controlling a bit-shift on the flowid when doing
  multi-port aggrigation in lagg and lacp.  By default, lagg/lacp will
  now use bits 16 and higher instead of 0 and higher.
  
  Reviewed by:	max
  Obtained from:	Netflix
  MFC after:	3 days

Modified:
  head/sys/net/ieee8023ad_lacp.c
  head/sys/net/if_lagg.c
  head/sys/net/if_lagg.h

Modified: head/sys/net/ieee8023ad_lacp.c
==============================================================================
--- head/sys/net/ieee8023ad_lacp.c	Mon Dec 30 01:17:05 2013	(r260069)
+++ head/sys/net/ieee8023ad_lacp.c	Mon Dec 30 01:32:17 2013	(r260070)
@@ -877,7 +877,7 @@ lacp_select_tx_port(struct lagg_softc *s
 	}
 
 	if (sc->use_flowid && (m->m_flags & M_FLOWID))
-		hash = m->m_pkthdr.flowid;
+		hash = m->m_pkthdr.flowid >> sc->flowid_shift;
 	else
 		hash = lagg_hashmbuf(sc, m, lsc->lsc_hashkey);
 	hash %= pm->pm_count;

Modified: head/sys/net/if_lagg.c
==============================================================================
--- head/sys/net/if_lagg.c	Mon Dec 30 01:17:05 2013	(r260069)
+++ head/sys/net/if_lagg.c	Mon Dec 30 01:32:17 2013	(r260070)
@@ -184,6 +184,11 @@ TUNABLE_INT("net.link.lagg.default_use_f
 SYSCTL_INT(_net_link_lagg, OID_AUTO, default_use_flowid, CTLFLAG_RW,
     &def_use_flowid, 0,
     "Default setting for using flow id for load sharing");
+static int def_flowid_shift = 16; /* Default value for using M_FLOWID */
+TUNABLE_INT("net.link.lagg.default_flowid_shift", &def_flowid_shift);
+SYSCTL_INT(_net_link_lagg, OID_AUTO, default_flowid_shift, CTLFLAG_RW,
+    &def_flowid_shift, 0,
+    "Default setting for flowid shift for load sharing");
 
 static int
 lagg_modevent(module_t mod, int type, void *data)
@@ -293,12 +298,17 @@ lagg_clone_create(struct if_clone *ifc, 
 	sysctl_ctx_init(&sc->ctx);
 	snprintf(num, sizeof(num), "%u", unit);
 	sc->use_flowid = def_use_flowid;
+	sc->flowid_shift = def_flowid_shift;
 	sc->sc_oid = oid = SYSCTL_ADD_NODE(&sc->ctx,
 		&SYSCTL_NODE_CHILDREN(_net_link, lagg),
 		OID_AUTO, num, CTLFLAG_RD, NULL, "");
 	SYSCTL_ADD_INT(&sc->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
-		"use_flowid", CTLTYPE_INT|CTLFLAG_RW, &sc->use_flowid, sc->use_flowid,
-		"Use flow id for load sharing");
+		"use_flowid", CTLTYPE_INT|CTLFLAG_RW, &sc->use_flowid,
+		sc->use_flowid, "Use flow id for load sharing");
+	SYSCTL_ADD_INT(&sc->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+		"flowid_shift", CTLTYPE_INT|CTLFLAG_RW, &sc->flowid_shift,
+		sc->flowid_shift,
+		"Shift flowid bits to prevent multiqueue collisions");
 	SYSCTL_ADD_INT(&sc->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
 		"count", CTLTYPE_INT|CTLFLAG_RD, &sc->sc_count, sc->sc_count,
 		"Total number of ports");
@@ -1850,7 +1860,7 @@ lagg_lb_start(struct lagg_softc *sc, str
 	uint32_t p = 0;
 
 	if (sc->use_flowid && (m->m_flags & M_FLOWID))
-		p = m->m_pkthdr.flowid;
+		p = m->m_pkthdr.flowid >> sc->flowid_shift;
 	else
 		p = lagg_hashmbuf(sc, m, lb->lb_key);
 	p %= sc->sc_count;

Modified: head/sys/net/if_lagg.h
==============================================================================
--- head/sys/net/if_lagg.h	Mon Dec 30 01:17:05 2013	(r260069)
+++ head/sys/net/if_lagg.h	Mon Dec 30 01:32:17 2013	(r260070)
@@ -231,6 +231,7 @@ struct lagg_softc {
 	struct sysctl_ctx_list		ctx;		/* sysctl variables */
 	struct sysctl_oid		*sc_oid;	/* sysctl tree oid */
 	int				use_flowid;	/* use M_FLOWID */
+	int				flowid_shift;	/* shift the flowid */
 };
 
 struct lagg_port {