svn commit: r262388 - in projects/vmxnet: share/man/man4 sys/dev/vmware/vmxnet3
Bryan Venteicher
bryanv at FreeBSD.org
Sun Feb 23 20:16:52 UTC 2014
Author: bryanv
Date: Sun Feb 23 20:16:51 2014
New Revision: 262388
URL: http://svnweb.freebsd.org/changeset/base/262388
Log:
Misc. vmx(4) cleanup of the multiple Rx/Tx queue support
Modified:
projects/vmxnet/share/man/man4/vmx.4
projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmx.c
projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmxvar.h
Modified: projects/vmxnet/share/man/man4/vmx.4
==============================================================================
--- projects/vmxnet/share/man/man4/vmx.4 Sun Feb 23 20:14:38 2014 (r262387)
+++ projects/vmxnet/share/man/man4/vmx.4 Sun Feb 23 20:16:51 2014 (r262388)
@@ -81,6 +81,49 @@ VMware Fusion 2.0 and newer
.Pp
For more information on configuring this device, see
.Xr ifconfig 8 .
+.Sh MULTIPLE QUEUES
+The
+.Nm
+driver supports multiple transmit and receive queues.
+Multiple queues are only supported by certain VMware products, such as ESXi.
+The number of queues allocated depends on the presence of MSI-X,
+the number of configured CPUs,
+and the tunables listed below.
+.Fx
+does not enable MSI-X support on VMware by default.
+The
+.Va hw.pci.honor_msi_blacklist
+tunable must be disabled to enable MSI-X support.
+.Sh LOADER TUNABLES
+Tunables can be set at the
+.Xr loader 8
+prompt before booting the kernel or stored in
+.Xr loader.conf 5 .
+.Bl -tag -width indent
+.It Va hw.vmx.txnqueue
+.It Va hw.vmx. Ns Ar X Ns Va .txnqueue
+Maximum number of transmit queues allocated by default by the driver.
+The default value is 8.
+The maximum supported by the VMXNET3 virtual NIC is 8.
+.It Va hw.vmx.rxnqueue
+.It Va hw.vmx. Ns Ar X Ns Va .rxnqueue
+Maximum number of receive queues allocated by default by the driver.
+The default value is 8.
+The maximum supported by the VMXNET3 virtual NIC is 16.
+.It Va hw.vmx.txndesc
+.It Va hw.vmx. Ns Ar X Ns Va .txndesc
+.Pp
+Number of transmit descriptors allocated by the driver.
+The default value is 512.
+The value must be a multiple of 32, and the maximum is 4096.
+.It Va hw.vmx.rxndesc
+.It Va hw.vmx. Ns Ar X Ns Va .rxndesc
+.Pp
+Number of receive descriptors per ring allocated by the driver.
+The default value is 256.
+The value must be a multiple of 32, and the maximum is 2048.
+There are two rings so the actual usage is doubled.
+.El
.Sh EXAMPLES
The following entry must be added to the VMware configuration file
to provide the
@@ -104,7 +147,7 @@ The
.Nm
driver was ported from
.Ox
-by
+and significantly rewritten by
.An Bryan Venteicher Aq bryanv at freebsd.org .
The
.Ox
Modified: projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmx.c
==============================================================================
--- projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmx.c Sun Feb 23 20:14:38 2014 (r262387)
+++ projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmx.c Sun Feb 23 20:16:51 2014 (r262388)
@@ -175,6 +175,7 @@ static int vmxnet3_txq_mq_start(struct i
static void vmxnet3_txq_tq_deferred(void *, int);
#endif
static void vmxnet3_txq_start(struct vmxnet3_txqueue *);
+static void vmxnet3_tx_start_all(struct vmxnet3_softc *);
static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
uint16_t);
@@ -663,10 +664,6 @@ vmxnet3_alloc_intr_resources(struct vmxn
return (0);
}
-/*
- * NOTE: We only support the simple case of each Rx and Tx queue on its
- * own MSIX vector. This is good enough until we support mulitqueue.
- */
static int
vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
{
@@ -861,29 +858,37 @@ static int
vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
{
#ifndef VMXNET3_LEGACY_TX
- device_t dev = sc->vmx_dev;
+ device_t dev;
+
+ dev = sc->vmx_dev;
sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
taskqueue_thread_enqueue, &sc->vmx_tq);
-
- return (sc->vmx_tq != NULL ? 0 : ENOMEM);
-#else
- return (0);
+ if (sc->vmx_tq == NULL)
+ return (ENOMEM);
#endif
+ return (0);
}
static void
vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
{
#ifndef VMXNET3_LEGACY_TX
- device_t dev = sc->vmx_dev;
+ device_t dev;
int nthreads, error;
- nthreads = sc->vmx_ntxqueues;
+ dev = sc->vmx_dev;
+
+ /*
+ * The taskqueue is typically not frequently used, so a dedicated
+ * thread for each queue is unnecessary.
+ */
+ nthreads = MAX(1, sc->vmx_ntxqueues / 2);
/*
* Most drivers just ignore the return value - it only fails
- * with ENOMEM so an error is not likely.
+ * with ENOMEM so an error is not likely. It is hard for us
+ * to recover from an error here.
*/
error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
"%s taskq", device_get_nameunit(dev));
@@ -992,7 +997,11 @@ vmxnet3_alloc_rxtx_queues(struct vmxnet3
int i, error;
/*
- * If we don't have MSIX available, there's no point in multiqueue.
+ * Only attempt to create multiple queues if MSIX is available. MSIX is
+ * disabled by default because its apparently broken for devices passed
+ * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
+ * must be set to zero for MSIX. This check prevents us from allocating
+ * queue structures that we will not use.
*/
if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
sc->vmx_max_nrxqueues = 1;
@@ -1128,7 +1137,7 @@ vmxnet3_alloc_shared_data(struct vmxnet3
if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
size = sizeof(struct vmxnet3_rss_shared);
- error = vmxnet3_dma_malloc(sc, size, 512, &sc->vmx_rss_dma);
+ error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
if (error) {
device_printf(dev, "cannot alloc rss shared memory\n");
return (error);
@@ -1576,9 +1585,9 @@ vmxnet3_reinit_interface(struct vmxnet3_
if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
if (ifp->if_capenable & IFCAP_TSO4)
- ifp->if_hwassist |= CSUM_TSO;
+ ifp->if_hwassist |= CSUM_IP_TSO;
if (ifp->if_capenable & IFCAP_TSO6)
- ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */
+ ifp->if_hwassist |= CSUM_IP6_TSO;
}
static void
@@ -1758,8 +1767,11 @@ vmxnet3_evintr(struct vmxnet3_softc *sc)
event = sc->vmx_ds->event;
vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
- if (event & VMXNET3_EVENT_LINK)
+ if (event & VMXNET3_EVENT_LINK) {
vmxnet3_link_status(sc);
+ if (sc->vmx_link_active != 0)
+ vmxnet3_tx_start_all(sc);
+ }
if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
reset = 1;
@@ -2317,6 +2329,7 @@ vmxnet3_rxstop(struct vmxnet3_softc *sc,
if (rxb->vrxb_m == NULL)
continue;
+
bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
@@ -2604,7 +2617,7 @@ vmxnet3_txq_offload_ctx(struct vmxnet3_t
(caddr_t) &iphdr);
ip = &iphdr;
} else
- ip = (struct ip *)(m->m_data + offset);
+ ip = mtodo(m, offset);
*proto = ip->ip_p;
*start = offset + (ip->ip_hl << 2);
break;
@@ -2634,17 +2647,16 @@ vmxnet3_txq_offload_ctx(struct vmxnet3_t
txq->vxtxq_stats.vmtxs_tso++;
+ /*
+ * For TSO, the size of the protocol header is also
+ * included in the descriptor header size.
+ */
if (m->m_len < *start + sizeof(struct tcphdr)) {
m_copydata(m, offset, sizeof(struct tcphdr),
(caddr_t) &tcphdr);
tcp = &tcphdr;
} else
- tcp = (struct tcphdr *)(m->m_data + *start);
-
- /*
- * For TSO, the size of the protocol header is also
- * included in the descriptor header size.
- */
+ tcp = mtodo(m, *start);
*start += (tcp->th_off << 2);
} else
txq->vxtxq_stats.vmtxs_csum++;
@@ -2855,7 +2867,7 @@ vmxnet3_start_locked(struct ifnet *ifp)
ETHER_BPF_MTAP(ifp, m_head);
}
- if (tx != 0) {
+ if (tx > 0) {
vmxnet3_txq_update_pending(txq);
txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
}
@@ -2934,7 +2946,7 @@ vmxnet3_txq_mq_start_locked(struct vmxne
ETHER_BPF_MTAP(ifp, m);
}
- if (tx != 0) {
+ if (tx > 0) {
vmxnet3_txq_update_pending(txq);
txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
}
@@ -3006,6 +3018,23 @@ vmxnet3_txq_start(struct vmxnet3_txqueue
}
static void
+vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_txqueue *txq;
+ int i;
+
+ VMXNET3_CORE_LOCK_ASSERT(sc);
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++) {
+ txq = &sc->vmx_txq[i];
+
+ VMXNET3_TXQ_LOCK(txq);
+ vmxnet3_txq_start(txq);
+ VMXNET3_TXQ_UNLOCK(txq);
+ }
+}
+
+static void
vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
{
struct ifnet *ifp;
@@ -3064,9 +3093,7 @@ vmxnet3_set_rxfilter(struct vmxnet3_soft
ifp = sc->vmx_ifp;
ds = sc->vmx_ds;
- mode = VMXNET3_RXMODE_UCAST;
- if (ifp->if_flags & IFF_BROADCAST)
- mode |= VMXNET3_RXMODE_BCAST;
+ mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
if (ifp->if_flags & IFF_PROMISC)
mode |= VMXNET3_RXMODE_PROMISC;
if (ifp->if_flags & IFF_ALLMULTI)
@@ -3340,7 +3367,7 @@ vmxnet3_accumulate_stats(struct vmxnet3_
ifp->if_iqdrops = rxaccum.vmrxs_iqdrops;
ifp->if_ierrors = rxaccum.vmrxs_ierrors;
ifp->if_opackets = txaccum.vmtxs_opackets;
-#ifndef VTNET_LEGACY_TX
+#ifndef VMXNET3_LEGACY_TX
ifp->if_obytes = txaccum.vmtxs_obytes;
ifp->if_omcasts = txaccum.vmtxs_omcasts;
#endif
Modified: projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmxvar.h
==============================================================================
--- projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmxvar.h Sun Feb 23 20:14:38 2014 (r262387)
+++ projects/vmxnet/sys/dev/vmware/vmxnet3/if_vmxvar.h Sun Feb 23 20:16:51 2014 (r262388)
@@ -234,7 +234,7 @@ struct vmxnet3_softc {
struct vmxnet3_interrupt vmx_intrs[VMXNET3_MAX_INTRS];
struct mtx vmx_mtx;
-#ifndef VMXNET_LEGACY_TX
+#ifndef VMXNET3_LEGACY_TX
struct taskqueue *vmx_tq;
#endif
uint8_t *vmx_mcast;
More information about the svn-src-projects
mailing list