svn commit: r307019 - in stable/10/sys/dev/hyperv: include netvsc vmbus
Sepherosa Ziehau
sephe at FreeBSD.org
Tue Oct 11 06:46:26 UTC 2016
Author: sephe
Date: Tue Oct 11 06:46:24 2016
New Revision: 307019
URL: https://svnweb.freebsd.org/changeset/base/307019
Log:
MFC 302632-302634
302632
hyperv/vmbus: More verbose for GPADL_connect/chan_{rescind,offer}
Reviewed by: Dexuan Cui <decui microsoft com>, Hongjiang Zhang <honzhan microsoft com>
Sponsored by: Microsoft OSTC
Differential Revision: https://reviews.freebsd.org/D6976
302633
hyperv/vmbus: Free sysctl properly upon channel close.
Prepare for sub-channel re-open.
Sponsored by: Microsoft OSTC
Differential Revision: https://reviews.freebsd.org/D6977
302634
hyperv/vmbus: Fix sub-channel re-open support.
For multi-channel devices, once the primary channel is closed,
a set of 'rescind' messages for sub-channels will be delivered
by Hypervisor. Sub-channel MUST be freed according to these
'rescind' messages; directly re-openning sub-channels in the
same fashion as the primary channel's re-opening does NOT work
at all.
After the primary channel is re-opened, requested # of sub-
channels will be delivered though 'channel offer' messages, and
this set of newly offered channels can be opened along side with
the primary channel.
This unbreaks the MTU setting for hn(4), which requires re-
openning all existsing channels upon MTU change.
Sponsored by: Microsoft OSTC
Differential Revision: https://reviews.freebsd.org/D6978
Modified:
stable/10/sys/dev/hyperv/include/hyperv.h
stable/10/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
stable/10/sys/dev/hyperv/vmbus/hv_channel.c
stable/10/sys/dev/hyperv/vmbus/hv_channel_mgmt.c
stable/10/sys/dev/hyperv/vmbus/vmbus_reg.h
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/dev/hyperv/include/hyperv.h
==============================================================================
--- stable/10/sys/dev/hyperv/include/hyperv.h Tue Oct 11 06:35:29 2016 (r307018)
+++ stable/10/sys/dev/hyperv/include/hyperv.h Tue Oct 11 06:46:24 2016 (r307019)
@@ -49,6 +49,7 @@
#include <sys/smp.h>
#include <sys/mutex.h>
#include <sys/bus.h>
+#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
@@ -627,6 +628,8 @@ typedef struct hv_vmbus_channel {
struct task ch_detach_task;
TAILQ_ENTRY(hv_vmbus_channel) ch_link;
+
+ struct sysctl_ctx_list ch_sysctl_ctx;
} hv_vmbus_channel;
#define HV_VMBUS_CHAN_ISPRIMARY(chan) ((chan)->primary_channel == NULL)
@@ -714,6 +717,7 @@ void vmbus_channel_cpu_rr(struct hv_vmb
struct hv_vmbus_channel **
vmbus_get_subchan(struct hv_vmbus_channel *pri_chan, int subchan_cnt);
void vmbus_rel_subchan(struct hv_vmbus_channel **subchan, int subchan_cnt);
+void vmbus_drain_subchan(struct hv_vmbus_channel *pri_chan);
/**
* @brief Get physical address from virtual
Modified: stable/10/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
==============================================================================
--- stable/10/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Tue Oct 11 06:35:29 2016 (r307018)
+++ stable/10/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Tue Oct 11 06:46:24 2016 (r307019)
@@ -349,6 +349,7 @@ static void hn_destroy_rx_data(struct hn
static void hn_set_tx_chimney_size(struct hn_softc *, int);
static void hn_channel_attach(struct hn_softc *, struct hv_vmbus_channel *);
static void hn_subchan_attach(struct hn_softc *, struct hv_vmbus_channel *);
+static void hn_subchan_setup(struct hn_softc *);
static int hn_transmit(struct ifnet *, struct mbuf *);
static void hn_xmit_qflush(struct ifnet *);
@@ -591,25 +592,8 @@ netvsc_attach(device_t dev)
device_printf(dev, "%d TX ring, %d RX ring\n",
sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse);
- if (sc->net_dev->num_channel > 1) {
- struct hv_vmbus_channel **subchan;
- int subchan_cnt = sc->net_dev->num_channel - 1;
- int i;
-
- /* Wait for sub-channels setup to complete. */
- subchan = vmbus_get_subchan(pri_chan, subchan_cnt);
-
- /* Attach the sub-channels. */
- for (i = 0; i < subchan_cnt; ++i) {
- /* NOTE: Calling order is critical. */
- hn_subchan_attach(sc, subchan[i]);
- hv_nv_subchan_attach(subchan[i]);
- }
-
- /* Release the sub-channels */
- vmbus_rel_subchan(subchan, subchan_cnt);
- device_printf(dev, "%d sub-channels setup done\n", subchan_cnt);
- }
+ if (sc->net_dev->num_channel > 1)
+ hn_subchan_setup(sc);
#if __FreeBSD_version >= 1100099
if (sc->hn_rx_ring_inuse > 1) {
@@ -1637,6 +1621,10 @@ hn_ioctl(struct ifnet *ifp, u_long cmd,
NV_UNLOCK(sc);
break;
}
+
+ /* Wait for subchannels to be destroyed */
+ vmbus_drain_subchan(hn_dev->channel);
+
error = hv_rf_on_device_add(hn_dev, &device_info,
sc->hn_rx_ring_inuse);
if (error) {
@@ -1645,6 +1633,26 @@ hn_ioctl(struct ifnet *ifp, u_long cmd,
NV_UNLOCK(sc);
break;
}
+ KASSERT(sc->hn_rx_ring_cnt == sc->net_dev->num_channel,
+ ("RX ring count %d and channel count %u mismatch",
+ sc->hn_rx_ring_cnt, sc->net_dev->num_channel));
+ if (sc->net_dev->num_channel > 1) {
+ int r;
+
+ /*
+ * Skip the rings on primary channel; they are
+ * handled by the hv_rf_on_device_add() above.
+ */
+ for (r = 1; r < sc->hn_rx_ring_cnt; ++r) {
+ sc->hn_rx_ring[r].hn_rx_flags &=
+ ~HN_RX_FLAG_ATTACHED;
+ }
+ for (r = 1; r < sc->hn_tx_ring_cnt; ++r) {
+ sc->hn_tx_ring[r].hn_tx_flags &=
+ ~HN_TX_FLAG_ATTACHED;
+ }
+ hn_subchan_setup(sc);
+ }
sc->hn_tx_chimney_max = sc->net_dev->send_section_size;
if (sc->hn_tx_ring[0].hn_tx_chimney_size >
@@ -3036,6 +3044,29 @@ hn_subchan_attach(struct hn_softc *sc, s
}
static void
+hn_subchan_setup(struct hn_softc *sc)
+{
+ struct hv_device *device_ctx = vmbus_get_devctx(sc->hn_dev);
+ struct hv_vmbus_channel **subchan;
+ int subchan_cnt = sc->net_dev->num_channel - 1;
+ int i;
+
+ /* Wait for sub-channels setup to complete. */
+ subchan = vmbus_get_subchan(device_ctx->channel, subchan_cnt);
+
+ /* Attach the sub-channels. */
+ for (i = 0; i < subchan_cnt; ++i) {
+ /* NOTE: Calling order is critical. */
+ hn_subchan_attach(sc, subchan[i]);
+ hv_nv_subchan_attach(subchan[i]);
+ }
+
+ /* Release the sub-channels */
+ vmbus_rel_subchan(subchan, subchan_cnt);
+ if_printf(sc->hn_ifp, "%d sub-channels setup done\n", subchan_cnt);
+}
+
+static void
hn_tx_taskq_create(void *arg __unused)
{
if (!hn_share_tx_taskq)
Modified: stable/10/sys/dev/hyperv/vmbus/hv_channel.c
==============================================================================
--- stable/10/sys/dev/hyperv/vmbus/hv_channel.c Tue Oct 11 06:35:29 2016 (r307018)
+++ stable/10/sys/dev/hyperv/vmbus/hv_channel.c Tue Oct 11 06:46:24 2016 (r307019)
@@ -111,7 +111,8 @@ vmbus_channel_sysctl_create(hv_vmbus_cha
ch_id = primary_ch->offer_msg.child_rel_id;
sub_ch_id = channel->offer_msg.offer.sub_channel_index;
}
- ctx = device_get_sysctl_ctx(dev);
+ ctx = &channel->ch_sysctl_ctx;
+ sysctl_ctx_init(ctx);
/* This creates dev.DEVNAME.DEVUNIT.channel tree */
devch_sysctl = SYSCTL_ADD_NODE(ctx,
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
@@ -423,6 +424,11 @@ hv_vmbus_channel_establish_gpadl(struct
device_printf(sc->vmbus_dev, "gpadl->chan%u failed: "
"status %u\n", channel->offer_msg.child_rel_id, status);
return EIO;
+ } else {
+ if (bootverbose) {
+ device_printf(sc->vmbus_dev, "gpadl->chan%u "
+ "succeeded\n", channel->offer_msg.child_rel_id);
+ }
}
return 0;
}
@@ -477,6 +483,7 @@ hv_vmbus_channel_close_internal(hv_vmbus
int error;
channel->state = HV_CHANNEL_OPEN_STATE;
+ sysctl_ctx_free(&channel->ch_sysctl_ctx);
/*
* set rxq to NULL to avoid more requests be scheduled
Modified: stable/10/sys/dev/hyperv/vmbus/hv_channel_mgmt.c
==============================================================================
--- stable/10/sys/dev/hyperv/vmbus/hv_channel_mgmt.c Tue Oct 11 06:35:29 2016 (r307018)
+++ stable/10/sys/dev/hyperv/vmbus/hv_channel_mgmt.c Tue Oct 11 06:46:24 2016 (r307019)
@@ -145,6 +145,19 @@ vmbus_channel_process_offer(hv_vmbus_cha
}
mtx_unlock(&sc->vmbus_chlist_lock);
+ if (bootverbose) {
+ char logstr[64];
+
+ logstr[0] = '\0';
+ if (channel != NULL) {
+ snprintf(logstr, sizeof(logstr), ", primary chan%u",
+ channel->offer_msg.child_rel_id);
+ }
+ device_printf(sc->vmbus_dev, "chan%u subchanid%u offer%s\n",
+ new_channel->offer_msg.child_rel_id,
+ new_channel->offer_msg.offer.sub_channel_index, logstr);
+ }
+
if (channel != NULL) {
/*
* Check if this is a sub channel.
@@ -160,13 +173,6 @@ vmbus_channel_process_offer(hv_vmbus_cha
new_channel, sc_list_entry);
mtx_unlock(&channel->sc_lock);
- if (bootverbose) {
- printf("VMBUS get multi-channel offer, "
- "rel=%u, sub=%u\n",
- new_channel->offer_msg.child_rel_id,
- new_channel->offer_msg.offer.sub_channel_index);
- }
-
/*
* Insert the new channel to the end of the global
* channel list.
@@ -181,11 +187,6 @@ vmbus_channel_process_offer(hv_vmbus_cha
ch_link);
mtx_unlock(&sc->vmbus_chlist_lock);
- if(bootverbose)
- printf("VMBUS: new multi-channel offer <%p>, "
- "its primary channel is <%p>.\n",
- new_channel, new_channel->primary_channel);
-
new_channel->state = HV_CHANNEL_OPEN_STATE;
/*
@@ -348,6 +349,10 @@ vmbus_channel_on_offer_rescind(struct vm
hv_vmbus_channel* channel;
rescind = (const hv_vmbus_channel_rescind_offer *)msg->msg_data;
+ if (bootverbose) {
+ device_printf(sc->vmbus_dev, "chan%u rescind\n",
+ rescind->child_rel_id);
+ }
channel = hv_vmbus_g_connection.channels[rescind->child_rel_id];
if (channel == NULL)
@@ -365,6 +370,54 @@ vmbus_chan_detach_task(void *xchan, int
if (HV_VMBUS_CHAN_ISPRIMARY(chan)) {
/* Only primary channel owns the hv_device */
hv_vmbus_child_device_unregister(chan->device);
+ /* NOTE: DO NOT free primary channel for now */
+ } else {
+ struct vmbus_softc *sc = chan->vmbus_sc;
+ struct hv_vmbus_channel *pri_chan = chan->primary_channel;
+ struct vmbus_chanmsg_chfree *req;
+ struct vmbus_msghc *mh;
+ int error;
+
+ mh = vmbus_msghc_get(sc, sizeof(*req));
+ if (mh == NULL) {
+ device_printf(sc->vmbus_dev,
+ "can not get msg hypercall for chfree(chan%u)\n",
+ chan->offer_msg.child_rel_id);
+ goto remove;
+ }
+
+ req = vmbus_msghc_dataptr(mh);
+ req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE;
+ req->chm_chanid = chan->offer_msg.child_rel_id;
+
+ error = vmbus_msghc_exec_noresult(mh);
+ vmbus_msghc_put(sc, mh);
+
+ if (error) {
+ device_printf(sc->vmbus_dev,
+ "chfree(chan%u) failed: %d",
+ chan->offer_msg.child_rel_id, error);
+ /* NOTE: Move on! */
+ } else {
+ if (bootverbose) {
+ device_printf(sc->vmbus_dev, "chan%u freed\n",
+ chan->offer_msg.child_rel_id);
+ }
+ }
+remove:
+ mtx_lock(&sc->vmbus_chlist_lock);
+ TAILQ_REMOVE(&sc->vmbus_chlist, chan, ch_link);
+ mtx_unlock(&sc->vmbus_chlist_lock);
+
+ mtx_lock(&pri_chan->sc_lock);
+ TAILQ_REMOVE(&pri_chan->sc_list_anchor, chan, sc_list_entry);
+ KASSERT(pri_chan->subchan_cnt > 0,
+ ("invalid subchan_cnt %d", pri_chan->subchan_cnt));
+ pri_chan->subchan_cnt--;
+ mtx_unlock(&pri_chan->sc_lock);
+ wakeup(pri_chan);
+
+ hv_vmbus_free_vmbus_channel(chan);
}
}
@@ -502,6 +555,15 @@ vmbus_rel_subchan(struct hv_vmbus_channe
}
void
+vmbus_drain_subchan(struct hv_vmbus_channel *pri_chan)
+{
+ mtx_lock(&pri_chan->sc_lock);
+ while (pri_chan->subchan_cnt > 0)
+ mtx_sleep(pri_chan, &pri_chan->sc_lock, 0, "dsubch", 0);
+ mtx_unlock(&pri_chan->sc_lock);
+}
+
+void
vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg)
{
vmbus_chanmsg_proc_t msg_proc;
Modified: stable/10/sys/dev/hyperv/vmbus/vmbus_reg.h
==============================================================================
--- stable/10/sys/dev/hyperv/vmbus/vmbus_reg.h Tue Oct 11 06:35:29 2016 (r307018)
+++ stable/10/sys/dev/hyperv/vmbus/vmbus_reg.h Tue Oct 11 06:46:24 2016 (r307019)
@@ -102,6 +102,7 @@ struct vmbus_gpa_range {
#define VMBUS_CHANMSG_TYPE_GPADL_CONNRESP 10 /* RESP */
#define VMBUS_CHANMSG_TYPE_GPADL_DISCONN 11 /* REQ */
#define VMBUS_CHANMSG_TYPE_GPADL_DISCONNRESP 12 /* RESP */
+#define VMBUS_CHANMSG_TYPE_CHFREE 13 /* REQ */
#define VMBUS_CHANMSG_TYPE_CONNECT 14 /* REQ */
#define VMBUS_CHANMSG_TYPE_CONNECT_RESP 15 /* RESP */
#define VMBUS_CHANMSG_TYPE_DISCONNECT 16 /* REQ */
@@ -206,4 +207,10 @@ struct vmbus_chanmsg_gpadl_disconn {
uint32_t chm_gpadl;
} __packed;
+/* VMBUS_CHANMSG_TYPE_CHFREE */
+struct vmbus_chanmsg_chfree {
+ struct vmbus_chanmsg_hdr chm_hdr;
+ uint32_t chm_chanid;
+} __packed;
+
#endif /* !_VMBUS_REG_H_ */
More information about the svn-src-stable-10
mailing list