svn commit: r277331 - in head/sys: conf dev/e1000 dev/ixgbe dev/ixl net netinet netinet6
Adrian Chadd
adrian at FreeBSD.org
Sun Jan 18 18:06:44 UTC 2015
Author: adrian
Date: Sun Jan 18 18:06:40 2015
New Revision: 277331
URL: https://svnweb.freebsd.org/changeset/base/277331
Log:
Refactor / restructure the RSS code into generic, IPv4 and IPv6 specific
bits.
The motivation here is to eventually teach netisr and potentially
other networking subsystems a bit more about how RSS work queues / buckets
are configured so things have a hope of auto-configuring in the future.
* net/rss_config.[ch] takes care of the generic bits for doing
configuration, hash function selection, etc;
* topelitz.[ch] is now in net/ rather than netinet/;
* (and would be in libkern if it didn't directly include RSS_KEYSIZE;
that's a later thing to fix up.)
* netinet/in_rss.[ch] now just contains the IPv4 specific methods;
* and netinet/in6_rss.[ch] now just contains the IPv6 specific methods.
This should have no functional impact on anyone currently using
the RSS support.
Differential Revision: D1383
Reviewed by: gnn, jfv (intel driver bits)
Added:
head/sys/net/rss_config.c
- copied, changed from r276951, head/sys/netinet/in_rss.c
head/sys/net/rss_config.h
- copied, changed from r276951, head/sys/netinet/in_rss.h
head/sys/net/toeplitz.c
- copied, changed from r277328, head/sys/netinet/toeplitz.c
head/sys/net/toeplitz.h
- copied unchanged from r276951, head/sys/netinet/toeplitz.h
head/sys/netinet6/in6_rss.c
- copied, changed from r276951, head/sys/netinet/in_rss.c
head/sys/netinet6/in6_rss.h
- copied, changed from r276951, head/sys/netinet/in_rss.h
Deleted:
head/sys/netinet/toeplitz.c
head/sys/netinet/toeplitz.h
Modified:
head/sys/conf/files
head/sys/dev/e1000/if_igb.c
head/sys/dev/ixgbe/ixgbe.c
head/sys/dev/ixl/ixl_txrx.c
head/sys/net/if_ethersubr.c
head/sys/netinet/in_pcb.c
head/sys/netinet/in_pcbgroup.c
head/sys/netinet/in_rss.c
head/sys/netinet/in_rss.h
head/sys/netinet/ip_input.c
head/sys/netinet/ip_output.c
head/sys/netinet/tcp_timer.c
head/sys/netinet/udp_usrreq.c
head/sys/netinet6/in6_pcbgroup.c
head/sys/netinet6/ip6_output.c
head/sys/netinet6/udp6_usrreq.c
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/conf/files Sun Jan 18 18:06:40 2015 (r277331)
@@ -3270,9 +3270,11 @@ net/radix_mpath.c standard
net/raw_cb.c standard
net/raw_usrreq.c standard
net/route.c standard
+net/rss_config.c optional inet rss | inet6 rss
net/rtsock.c standard
net/slcompress.c optional netgraph_vjc | sppp | \
netgraph_sppp
+net/toeplitz.c optional inet rss | inet6 rss
net/vnet.c optional vimage
net/zlib.c optional crypto | geom_uzip | ipsec | \
mxge | netgraph_deflate | \
@@ -3422,7 +3424,7 @@ netinet/in_pcb.c optional inet | inet6
netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup
netinet/in_proto.c optional inet | inet6
netinet/in_rmx.c optional inet
-netinet/in_rss.c optional inet rss | inet6 rss
+netinet/in_rss.c optional inet rss
netinet/ip_divert.c optional inet ipdivert ipfirewall
netinet/ip_ecn.c optional inet | inet6
netinet/ip_encap.c optional inet | inet6
@@ -3465,7 +3467,6 @@ netinet/tcp_syncache.c optional inet |
netinet/tcp_timer.c optional inet | inet6
netinet/tcp_timewait.c optional inet | inet6
netinet/tcp_usrreq.c optional inet | inet6
-netinet/toeplitz.c optional inet rss | inet6 rss
netinet/udp_usrreq.c optional inet | inet6
netinet/libalias/alias.c optional libalias inet | netgraph_nat inet
netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet
@@ -3485,6 +3486,7 @@ netinet6/in6_pcb.c optional inet6
netinet6/in6_pcbgroup.c optional inet6 pcbgroup
netinet6/in6_proto.c optional inet6
netinet6/in6_rmx.c optional inet6
+netinet6/in6_rss.c optional inet6 rss
netinet6/in6_src.c optional inet6
netinet6/ip6_forward.c optional inet6
netinet6/ip6_gre.c optional gre inet6
Modified: head/sys/dev/e1000/if_igb.c
==============================================================================
--- head/sys/dev/e1000/if_igb.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/dev/e1000/if_igb.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -73,6 +73,9 @@
#include <net/if_arp.h>
#include <net/if_dl.h>
#include <net/if_media.h>
+#ifdef RSS
+#include <net/rss_config.h>
+#endif
#include <net/if_types.h>
#include <net/if_vlan_var.h>
@@ -85,9 +88,6 @@
#include <netinet/tcp.h>
#include <netinet/tcp_lro.h>
#include <netinet/udp.h>
-#ifdef RSS
-#include <netinet/in_rss.h>
-#endif
#include <machine/in_cksum.h>
#include <dev/led/led.h>
Modified: head/sys/dev/ixgbe/ixgbe.c
==============================================================================
--- head/sys/dev/ixgbe/ixgbe.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/dev/ixgbe/ixgbe.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -39,7 +39,7 @@
#include "ixgbe.h"
#ifdef RSS
-#include <netinet/in_rss.h>
+#include <net/rss_config.h>
#endif
/*********************************************************************
Modified: head/sys/dev/ixl/ixl_txrx.c
==============================================================================
--- head/sys/dev/ixl/ixl_txrx.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/dev/ixl/ixl_txrx.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -1378,7 +1378,7 @@ static inline int
ixl_ptype_to_hash(u8 ptype)
{
struct i40e_rx_ptype_decoded decoded;
- u8 ex = 0
+ u8 ex = 0;
decoded = decode_rx_desc_ptype(ptype);
ex = decoded.outer_frag;
Modified: head/sys/net/if_ethersubr.c
==============================================================================
--- head/sys/net/if_ethersubr.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/net/if_ethersubr.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -63,6 +63,7 @@
#include <net/if_vlan_var.h>
#include <net/if_llatbl.h>
#include <net/pfil.h>
+#include <net/rss_config.h>
#include <net/vnet.h>
#include <netpfil/pf/pf_mtag.h>
@@ -71,7 +72,6 @@
#include <netinet/in.h>
#include <netinet/in_var.h>
#include <netinet/if_ether.h>
-#include <netinet/in_rss.h>
#include <netinet/ip_carp.h>
#include <netinet/ip_var.h>
#endif
Copied and modified: head/sys/net/rss_config.c (from r276951, head/sys/netinet/in_rss.c)
==============================================================================
--- head/sys/netinet/in_rss.c Sat Jan 10 23:43:39 2015 (r276951, copy source)
+++ head/sys/net/rss_config.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -50,17 +50,20 @@ __FBSDID("$FreeBSD$");
#include <net/if.h>
#include <net/if_var.h>
#include <net/netisr.h>
+#include <net/rss_config.h>
+#include <net/toeplitz.h>
+#if 0
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/in_rss.h>
#include <netinet/in_var.h>
-#include <netinet/toeplitz.h>
/* for software rss hash support */
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
+#endif
/*-
* Operating system parts of receiver-side scaling (RSS), which allows
@@ -89,6 +92,7 @@ __FBSDID("$FreeBSD$");
* placement and pcbgroup expectations.
*/
+SYSCTL_DECL(_net_inet);
SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
/*
@@ -175,8 +179,6 @@ struct rss_table_entry {
};
static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
-static inline u_int rss_gethashconfig_local(void);
-
static void
rss_init(__unused void *arg)
{
@@ -282,7 +284,7 @@ rss_naive_hash(u_int keylen, const uint8
return (v);
}
-static uint32_t
+uint32_t
rss_hash(u_int datalen, const uint8_t *data)
{
@@ -302,88 +304,6 @@ rss_hash(u_int datalen, const uint8_t *d
}
/*
- * Hash an IPv4 2-tuple.
- */
-uint32_t
-rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst)
-{
- uint8_t data[sizeof(src) + sizeof(dst)];
- u_int datalen;
-
- datalen = 0;
- bcopy(&src, &data[datalen], sizeof(src));
- datalen += sizeof(src);
- bcopy(&dst, &data[datalen], sizeof(dst));
- datalen += sizeof(dst);
- return (rss_hash(datalen, data));
-}
-
-/*
- * Hash an IPv4 4-tuple.
- */
-uint32_t
-rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
- u_short dstport)
-{
- uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
- sizeof(dstport)];
- u_int datalen;
-
- datalen = 0;
- bcopy(&src, &data[datalen], sizeof(src));
- datalen += sizeof(src);
- bcopy(&dst, &data[datalen], sizeof(dst));
- datalen += sizeof(dst);
- bcopy(&srcport, &data[datalen], sizeof(srcport));
- datalen += sizeof(srcport);
- bcopy(&dstport, &data[datalen], sizeof(dstport));
- datalen += sizeof(dstport);
- return (rss_hash(datalen, data));
-}
-
-#ifdef INET6
-/*
- * Hash an IPv6 2-tuple.
- */
-uint32_t
-rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
-{
- uint8_t data[sizeof(*src) + sizeof(*dst)];
- u_int datalen;
-
- datalen = 0;
- bcopy(src, &data[datalen], sizeof(*src));
- datalen += sizeof(*src);
- bcopy(dst, &data[datalen], sizeof(*dst));
- datalen += sizeof(*dst);
- return (rss_hash(datalen, data));
-}
-
-/*
- * Hash an IPv6 4-tuple.
- */
-uint32_t
-rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
- const struct in6_addr *dst, u_short dstport)
-{
- uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
- sizeof(dstport)];
- u_int datalen;
-
- datalen = 0;
- bcopy(src, &data[datalen], sizeof(*src));
- datalen += sizeof(*src);
- bcopy(dst, &data[datalen], sizeof(*dst));
- datalen += sizeof(*dst);
- bcopy(&srcport, &data[datalen], sizeof(srcport));
- datalen += sizeof(srcport);
- bcopy(&dstport, &data[datalen], sizeof(dstport));
- datalen += sizeof(dstport);
- return (rss_hash(datalen, data));
-}
-#endif /* INET6 */
-
-/*
* Query the number of RSS bits in use.
*/
u_int
@@ -498,260 +418,6 @@ rss_m2bucket(struct mbuf *m, uint32_t *b
}
/*
- * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
- * IPv4 source/destination address, UDP or TCP source/destination ports
- * and the protocol type.
- *
- * The protocol code may wish to do a software hash of the given
- * tuple. This depends upon the currently configured RSS hash types.
- *
- * This assumes that the packet in question isn't a fragment.
- *
- * It also assumes the packet source/destination address
- * are in "incoming" packet order (ie, source is "far" address.)
- */
-int
-rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
- u_short sp, u_short dp, int proto,
- uint32_t *hashval, uint32_t *hashtype)
-{
- uint32_t hash;
-
- /*
- * Next, choose the hash type depending upon the protocol
- * identifier.
- */
- if ((proto == IPPROTO_TCP) &&
- (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
- hash = rss_hash_ip4_4tuple(s, sp, d, dp);
- *hashval = hash;
- *hashtype = M_HASHTYPE_RSS_TCP_IPV4;
- return (0);
- } else if ((proto == IPPROTO_UDP) &&
- (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
- hash = rss_hash_ip4_4tuple(s, sp, d, dp);
- *hashval = hash;
- *hashtype = M_HASHTYPE_RSS_UDP_IPV4;
- return (0);
- } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
- /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
- hash = rss_hash_ip4_2tuple(s, d);
- *hashval = hash;
- *hashtype = M_HASHTYPE_RSS_IPV4;
- return (0);
- }
-
- /* No configured available hashtypes! */
- printf("%s: no available hashtypes!\n", __func__);
- return (-1);
-}
-
-/*
- * Do a software calculation of the RSS for the given mbuf.
- *
- * This is typically used by the input path to recalculate the RSS after
- * some form of packet processing (eg de-capsulation, IP fragment reassembly.)
- *
- * dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and
- * RSS_HASH_PKT_EGRESS for outgoing.
- *
- * Returns 0 if a hash was done, -1 if no hash was done, +1 if
- * the mbuf already had a valid RSS flowid.
- *
- * This function doesn't modify the mbuf. It's up to the caller to
- * assign flowid/flowtype as appropriate.
- */
-int
-rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
- uint32_t *hashtype)
-{
- const struct ip *ip;
- const struct tcphdr *th;
- const struct udphdr *uh;
- uint32_t flowid;
- uint32_t flowtype;
- uint8_t proto;
- int iphlen;
- int is_frag = 0;
-
- /*
- * XXX For now this only handles hashing on incoming mbufs.
- */
- if (dir != RSS_HASH_PKT_INGRESS) {
- printf("%s: called on EGRESS packet!\n", __func__);
- return (-1);
- }
-
- /*
- * First, validate that the mbuf we have is long enough
- * to have an IPv4 header in it.
- */
- if (m->m_pkthdr.len < (sizeof(struct ip))) {
- printf("%s: short mbuf pkthdr\n", __func__);
- return (-1);
- }
- if (m->m_len < (sizeof(struct ip))) {
- printf("%s: short mbuf len\n", __func__);
- return (-1);
- }
-
- /* Ok, let's dereference that */
- ip = mtod(m, struct ip *);
- proto = ip->ip_p;
- iphlen = ip->ip_hl << 2;
-
- /*
- * If this is a fragment then it shouldn't be four-tuple
- * hashed just yet. Once it's reassembled into a full
- * frame it should be re-hashed.
- */
- if (ip->ip_off & htons(IP_MF | IP_OFFMASK))
- is_frag = 1;
-
- /*
- * If the mbuf flowid/flowtype matches the packet type,
- * and we don't support the 4-tuple version of the given protocol,
- * then signal to the owner that it can trust the flowid/flowtype
- * details.
- *
- * This is a little picky - eg, if TCPv4 / UDPv4 hashing
- * is supported but we got a TCP/UDP frame only 2-tuple hashed,
- * then we shouldn't just "trust" the 2-tuple hash. We need
- * a 4-tuple hash.
- */
- flowid = m->m_pkthdr.flowid;
- flowtype = M_HASHTYPE_GET(m);
-
- if (flowtype != M_HASHTYPE_NONE) {
- switch (proto) {
- case IPPROTO_UDP:
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
- (flowtype == M_HASHTYPE_RSS_UDP_IPV4) &&
- (is_frag == 0)) {
- return (1);
- }
- /*
- * Only allow 2-tuple for UDP frames if we don't also
- * support 4-tuple for UDP.
- */
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
- ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
- flowtype == M_HASHTYPE_RSS_IPV4) {
- return (1);
- }
- break;
- case IPPROTO_TCP:
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
- (flowtype == M_HASHTYPE_RSS_TCP_IPV4) &&
- (is_frag == 0)) {
- return (1);
- }
- /*
- * Only allow 2-tuple for TCP frames if we don't also
- * support 2-tuple for TCP.
- */
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
- ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
- flowtype == M_HASHTYPE_RSS_IPV4) {
- return (1);
- }
- break;
- default:
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
- flowtype == M_HASHTYPE_RSS_IPV4) {
- return (1);
- }
- break;
- }
- }
-
- /*
- * Decode enough information to make a hash decision.
- *
- * XXX TODO: does the hardware hash on 4-tuple if IP
- * options are present?
- */
- if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
- (proto == IPPROTO_TCP) &&
- (is_frag == 0)) {
- if (m->m_len < iphlen + sizeof(struct tcphdr)) {
- printf("%s: short TCP frame?\n", __func__);
- return (-1);
- }
- th = (struct tcphdr *)((caddr_t)ip + iphlen);
- return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
- th->th_sport,
- th->th_dport,
- proto,
- hashval,
- hashtype);
- } else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
- (proto == IPPROTO_UDP) &&
- (is_frag == 0)) {
- uh = (struct udphdr *)((caddr_t)ip + iphlen);
- if (m->m_len < iphlen + sizeof(struct udphdr)) {
- printf("%s: short UDP frame?\n", __func__);
- return (-1);
- }
- return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
- uh->uh_sport,
- uh->uh_dport,
- proto,
- hashval,
- hashtype);
- } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
- /* Default to 2-tuple hash */
- return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
- 0, /* source port */
- 0, /* destination port */
- 0, /* IPPROTO_IP */
- hashval,
- hashtype);
- } else {
- printf("%s: no available hashtypes!\n", __func__);
- return (-1);
- }
-}
-
-/*
- * Similar to rss_m2cpuid, but designed to be used by the IP NETISR
- * on incoming frames.
- *
- * If an existing RSS hash exists and it matches what the configured
- * hashing is, then use it.
- *
- * If there's an existing RSS hash but the desired hash is different,
- * or if there's no useful RSS hash, then calculate it via
- * the software path.
- *
- * XXX TODO: definitely want statistics here!
- */
-struct mbuf *
-rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
-{
- uint32_t hash_val, hash_type;
- int ret;
-
- M_ASSERTPKTHDR(m);
-
- ret = rss_mbuf_software_hash_v4(m, RSS_HASH_PKT_INGRESS,
- &hash_val, &hash_type);
- if (ret > 0) {
- /* mbuf has a valid hash already; don't need to modify it */
- *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
- } else if (ret == 0) {
- /* hash was done; update */
- m->m_pkthdr.flowid = hash_val;
- M_HASHTYPE_SET(m, hash_type);
- *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
- } else { /* ret < 0 */
- /* no hash was done */
- *cpuid = NETISR_CPUID_NONE;
- }
- return (m);
-}
-
-/*
* Query the RSS hash algorithm.
*/
u_int
@@ -799,8 +465,14 @@ rss_getnumcpus(void)
return (rss_ncpus);
}
-static inline u_int
-rss_gethashconfig_local(void)
+/*
+ * Return the supported RSS hash configuration.
+ *
+ * NICs should query this to determine what to configure in their redirection
+ * matching table.
+ */
+inline u_int
+rss_gethashconfig(void)
{
/* Return 4-tuple for TCP; 2-tuple for others */
@@ -829,19 +501,6 @@ rss_gethashconfig_local(void)
}
/*
- * Return the supported RSS hash configuration.
- *
- * NICs should query this to determine what to configure in their redirection
- * matching table.
- */
-u_int
-rss_gethashconfig(void)
-{
-
- return (rss_gethashconfig_local());
-}
-
-/*
* XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
* it appearing in debugging output unnecessarily.
*/
Copied and modified: head/sys/net/rss_config.h (from r276951, head/sys/netinet/in_rss.h)
==============================================================================
--- head/sys/netinet/in_rss.h Sat Jan 10 23:43:39 2015 (r276951, copy source)
+++ head/sys/net/rss_config.h Sun Jan 18 18:06:40 2015 (r277331)
@@ -29,8 +29,8 @@
* $FreeBSD$
*/
-#ifndef _NETINET_IN_RSS_H_
-#define _NETINET_IN_RSS_H_
+#ifndef _NET_RSS_CONFIG_H_
+#define _NET_RSS_CONFIG_H_
#include <netinet/in.h> /* in_addr_t */
@@ -107,36 +107,17 @@ u_int rss_getnumcpus(void);
u_int rss_gethashconfig(void);
/*
- * Network stack interface to generate a hash for a protocol tuple.
+ * Hash calculation functions.
*/
-uint32_t rss_hash_ip4_4tuple(struct in_addr src, u_short srcport,
- struct in_addr dst, u_short dstport);
-uint32_t rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst);
-uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
- const struct in6_addr *dst, u_short dstport);
-uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src,
- const struct in6_addr *dst);
+uint32_t rss_hash(u_int datalen, const uint8_t *data);
/*
* Network stack interface to query desired CPU affinity of a packet.
*/
-struct mbuf *rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
-u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
-int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
- uint32_t *bucket_id);
-int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
-
-/*
- * Functions to calculate a software RSS hash for a given mbuf or
- * packet detail.
- */
-int rss_mbuf_software_hash_v4(const struct mbuf *m, int dir,
- uint32_t *hashval, uint32_t *hashtype);
-int rss_proto_software_hash_v4(struct in_addr src,
- struct in_addr dst, u_short src_port, u_short dst_port,
- int proto, uint32_t *hashval,
- uint32_t *hashtype);
-struct mbuf * rss_soft_m2cpuid(struct mbuf *m, uintptr_t source,
- u_int *cpuid);
+struct mbuf * rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
+u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
+int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
+ uint32_t *bucket_id);
+int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
-#endif /* !_NETINET_IN_RSS_H_ */
+#endif /* !_NET_RSS_CONFIG_H_ */
Copied and modified: head/sys/net/toeplitz.c (from r277328, head/sys/netinet/toeplitz.c)
==============================================================================
--- head/sys/netinet/toeplitz.c Sun Jan 18 17:25:41 2015 (r277328, copy source)
+++ head/sys/net/toeplitz.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -29,8 +29,8 @@ __FBSDID("$FreeBSD$");
#include <sys/types.h>
-#include <netinet/in_rss.h>
-#include <netinet/toeplitz.h>
+#include <net/rss_config.h>
+#include <net/toeplitz.h>
#include <sys/systm.h>
Copied: head/sys/net/toeplitz.h (from r276951, head/sys/netinet/toeplitz.h)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/net/toeplitz.h Sun Jan 18 18:06:40 2015 (r277331, copy of r276951, head/sys/netinet/toeplitz.h)
@@ -0,0 +1,40 @@
+/*-
+ * Copyright (c) 2010 David Malone <dwmalone at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NETINET_TOEPLITZ_H_
+#define _NETINET_TOEPLITZ_H_
+
+/*
+ * Toeplitz (RSS) hash algorithm; possibly we should cache intermediate
+ * results between runs, in which case we'll need explicit init/destroy and
+ * state management.
+ */
+uint32_t toeplitz_hash(u_int keylen, const uint8_t *key,
+ u_int datalen, const uint8_t *data);
+
+#endif /* !_NETINET_TOEPLITZ_H_ */
Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/netinet/in_pcb.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -71,12 +71,12 @@ __FBSDID("$FreeBSD$");
#include <net/if_var.h>
#include <net/if_types.h>
#include <net/route.h>
+#include <net/rss_config.h>
#include <net/vnet.h>
#if defined(INET) || defined(INET6)
#include <netinet/in.h>
#include <netinet/in_pcb.h>
-#include <netinet/in_rss.h>
#include <netinet/ip_var.h>
#include <netinet/tcp_var.h>
#include <netinet/udp.h>
Modified: head/sys/netinet/in_pcbgroup.c
==============================================================================
--- head/sys/netinet/in_pcbgroup.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/netinet/in_pcbgroup.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -42,7 +42,10 @@ __FBSDID("$FreeBSD$");
#include <sys/smp.h>
#include <sys/socketvar.h>
+#include <net/rss_config.h>
+
#include <netinet/in.h>
+
#include <netinet/in_pcb.h>
#include <netinet/in_rss.h>
#ifdef INET6
Modified: head/sys/netinet/in_rss.c
==============================================================================
--- head/sys/netinet/in_rss.c Sun Jan 18 18:02:12 2015 (r277330)
+++ head/sys/netinet/in_rss.c Sun Jan 18 18:06:40 2015 (r277331)
@@ -50,257 +50,18 @@ __FBSDID("$FreeBSD$");
#include <net/if.h>
#include <net/if_var.h>
#include <net/netisr.h>
+#include <net/rss_config.h>
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/in_rss.h>
#include <netinet/in_var.h>
-#include <netinet/toeplitz.h>
/* for software rss hash support */
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
-/*-
- * Operating system parts of receiver-side scaling (RSS), which allows
- * network cards to direct flows to particular receive queues based on hashes
- * of header tuples. This implementation aligns RSS buckets with connection
- * groups at the TCP/IP layer, so each bucket is associated with exactly one
- * group. As a result, the group lookup structures (and lock) should have an
- * effective affinity with exactly one CPU.
- *
- * Network device drivers needing to configure RSS will query this framework
- * for parameters, such as the current RSS key, hashing policies, number of
- * bits, and indirection table mapping hashes to buckets and CPUs. They may
- * provide their own supplementary information, such as queue<->CPU bindings.
- * It is the responsibility of the network device driver to inject packets
- * into the stack on as close to the right CPU as possible, if playing by RSS
- * rules.
- *
- * TODO:
- *
- * - Synchronization for rss_key and other future-configurable parameters.
- * - Event handler drivers can register to pick up RSS configuration changes.
- * - Should we allow rss_basecpu to be configured?
- * - Randomize key on boot.
- * - IPv6 support.
- * - Statistics on how often there's a misalignment between hardware
- * placement and pcbgroup expectations.
- */
-
-SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
-
-/*
- * Toeplitz is the only required hash function in the RSS spec, so use it by
- * default.
- */
-static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
- "RSS hash algorithm");
-
-/*
- * Size of the indirection table; at most 128 entries per the RSS spec. We
- * size it to at least 2 times the number of CPUs by default to allow useful
- * rebalancing. If not set explicitly with a loader tunable, we tune based
- * on the number of CPUs present.
- *
- * XXXRW: buckets might be better to use for the tunable than bits.
- */
-static u_int rss_bits;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
- "RSS bits");
-
-static u_int rss_mask;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
- "RSS mask");
-
-static const u_int rss_maxbits = RSS_MAXBITS;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
- __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
-
-/*
- * RSS's own count of the number of CPUs it could be using for processing.
- * Bounded to 64 by RSS constants.
- */
-static u_int rss_ncpus;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
- "Number of CPUs available to RSS");
-
-#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
-static const u_int rss_maxcpus = RSS_MAXCPUS;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
- __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
-
-/*
- * Variable exists just for reporting rss_bits in a user-friendly way.
- */
-static u_int rss_buckets;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
- "RSS buckets");
-
-/*
- * Base CPU number; devices will add this to all CPU numbers returned by the
- * RSS indirection table. Currently unmodifable in FreeBSD.
- */
-static const u_int rss_basecpu;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
- __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
-
-/*
- * RSS secret key, intended to prevent attacks on load-balancing. Its
- * effectiveness may be limited by algorithm choice and available entropy
- * during the boot.
- *
- * XXXRW: And that we don't randomize it yet!
- *
- * This is the default Microsoft RSS specification key which is also
- * the Chelsio T5 firmware default key.
- */
-static uint8_t rss_key[RSS_KEYSIZE] = {
- 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
- 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
- 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
- 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
- 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
-};
-
-/*
- * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
- * Drivers may supplement this table with a seperate CPU<->queue table when
- * programming devices.
- */
-struct rss_table_entry {
- uint8_t rte_cpu; /* CPU affinity of bucket. */
-};
-static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
-
-static inline u_int rss_gethashconfig_local(void);
-
-static void
-rss_init(__unused void *arg)
-{
- u_int i;
- u_int cpuid;
-
- /*
- * Validate tunables, coerce to sensible values.
- */
- switch (rss_hashalgo) {
- case RSS_HASH_TOEPLITZ:
- case RSS_HASH_NAIVE:
- break;
-
- default:
- printf("%s: invalid RSS hashalgo %u, coercing to %u",
- __func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
- rss_hashalgo = RSS_HASH_TOEPLITZ;
- }
-
- /*
- * Count available CPUs.
- *
- * XXXRW: Note incorrect assumptions regarding contiguity of this set
- * elsewhere.
- */
- rss_ncpus = 0;
- for (i = 0; i <= mp_maxid; i++) {
- if (CPU_ABSENT(i))
- continue;
- rss_ncpus++;
- }
- if (rss_ncpus > RSS_MAXCPUS)
- rss_ncpus = RSS_MAXCPUS;
-
- /*
- * Tune RSS table entries to be no less than 2x the number of CPUs
- * -- unless we're running uniprocessor, in which case there's not
- * much point in having buckets to rearrange for load-balancing!
- */
- if (rss_ncpus > 1) {
- if (rss_bits == 0)
- rss_bits = fls(rss_ncpus - 1) + 1;
-
- /*
- * Microsoft limits RSS table entries to 128, so apply that
- * limit to both auto-detected CPU counts and user-configured
- * ones.
- */
- if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
- printf("%s: RSS bits %u not valid, coercing to %u",
- __func__, rss_bits, RSS_MAXBITS);
- rss_bits = RSS_MAXBITS;
- }
-
- /*
- * Figure out how many buckets to use; warn if less than the
- * number of configured CPUs, although this is not a fatal
- * problem.
- */
- rss_buckets = (1 << rss_bits);
- if (rss_buckets < rss_ncpus)
- printf("%s: WARNING: rss_buckets (%u) less than "
- "rss_ncpus (%u)\n", __func__, rss_buckets,
- rss_ncpus);
- rss_mask = rss_buckets - 1;
- } else {
- rss_bits = 0;
- rss_buckets = 1;
- rss_mask = 0;
- }
-
- /*
- * Set up initial CPU assignments: round-robin by default.
- */
- cpuid = CPU_FIRST();
- for (i = 0; i < rss_buckets; i++) {
- rss_table[i].rte_cpu = cpuid;
- cpuid = CPU_NEXT(cpuid);
- }
-
- /*
- * Randomize rrs_key.
- *
- * XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
- * loop to check for "bad" RSS keys.
- */
-}
-SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
-
-static uint32_t
-rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
- const uint8_t *data)
-{
- uint32_t v;
- u_int i;
-
- v = 0;
- for (i = 0; i < keylen; i++)
- v += key[i];
- for (i = 0; i < datalen; i++)
- v += data[i];
- return (v);
-}
-
-static uint32_t
-rss_hash(u_int datalen, const uint8_t *data)
-{
-
- switch (rss_hashalgo) {
- case RSS_HASH_TOEPLITZ:
- return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
- data));
-
- case RSS_HASH_NAIVE:
- return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
- data));
-
- default:
- panic("%s: unsupported/unknown hashalgo %d", __func__,
- rss_hashalgo);
- }
-}
-
/*
* Hash an IPv4 2-tuple.
*/
@@ -341,162 +102,6 @@ rss_hash_ip4_4tuple(struct in_addr src,
return (rss_hash(datalen, data));
}
-#ifdef INET6
-/*
- * Hash an IPv6 2-tuple.
- */
-uint32_t
-rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
-{
- uint8_t data[sizeof(*src) + sizeof(*dst)];
- u_int datalen;
-
- datalen = 0;
- bcopy(src, &data[datalen], sizeof(*src));
- datalen += sizeof(*src);
- bcopy(dst, &data[datalen], sizeof(*dst));
- datalen += sizeof(*dst);
- return (rss_hash(datalen, data));
-}
-
-/*
- * Hash an IPv6 4-tuple.
- */
-uint32_t
-rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
- const struct in6_addr *dst, u_short dstport)
-{
- uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
- sizeof(dstport)];
- u_int datalen;
-
- datalen = 0;
- bcopy(src, &data[datalen], sizeof(*src));
- datalen += sizeof(*src);
- bcopy(dst, &data[datalen], sizeof(*dst));
- datalen += sizeof(*dst);
- bcopy(&srcport, &data[datalen], sizeof(srcport));
- datalen += sizeof(srcport);
- bcopy(&dstport, &data[datalen], sizeof(dstport));
- datalen += sizeof(dstport);
- return (rss_hash(datalen, data));
-}
-#endif /* INET6 */
-
-/*
- * Query the number of RSS bits in use.
- */
-u_int
-rss_getbits(void)
-{
-
- return (rss_bits);
-}
-
-/*
- * Query the RSS bucket associated with an RSS hash.
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list