git: 769b884e2e2e - main - vmm: Fix AP startup with old userspace binaries.

The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=769b884e2e2eb84d25eca2a5462ae0a6c4dcd2a7

commit 769b884e2e2eb84d25eca2a5462ae0a6c4dcd2a7
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2022-10-26 21:22:56 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2022-10-26 21:22:56 +0000

    vmm: Fix AP startup with old userspace binaries.
    
    Older binaries that do not request IPI exits to userspace do not
    start user threads for other vCPUs until a STARTUP IPI triggers a
    VM_EXITCODE_SPINUP_AP exit to userland.  This means that those vcpus
    are not yet active (in terms of vm_active_cpus) when the INIT and
    STARTUP IPIs are delivered to the vCPUs.
    
    The changes in commit 0bda8d3e9f7a changed the INIT and STARTUP IPIs
    to reuse the existing vlapic_calcdest() function.  This function
    silently ignores IPIs sent to inactive vCPUs.  As a result, when using
    an old bhyve binary, the INIT and STARTUP IPIs sent to wakeup APs were
    ignored.
    
    To fix, restructure the compat code for the INIT and STARTUP IPIs to
    ignore the results of vlapic_calcdest() and manually parse the APIC ID
    and resulting vcpuid.  As part of this, make the compat code always
    conditonal on the ipi_exit capability being disabled.
    
    Reviewed by:    c.koehne_beckhoff.com, markj
    Differential Revision:  https://reviews.freebsd.org/D37093
---
 sys/amd64/vmm/io/vlapic.c | 63 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 45 insertions(+), 18 deletions(-)

diff --git a/sys/amd64/vmm/io/vlapic.c b/sys/amd64/vmm/io/vlapic.c
index 8b547d9a9453..0e1fb1bbf006 100644
--- a/sys/amd64/vmm/io/vlapic.c
+++ b/sys/amd64/vmm/io/vlapic.c
@@ -1119,20 +1119,61 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 
 		break;
 	case APIC_DELMODE_INIT:
-		CPU_FOREACH_ISSET(i, &dmask) {
+		if (!vlapic->ipi_exit) {
+			if (!phys)
+				break;
+
+			i = vm_apicid2vcpuid(vlapic->vm, dest);
+			if (i >= vm_get_maxcpus(vlapic->vm) ||
+			    i == vlapic->vcpuid)
+				break;
+
 			/*
-			 * Userland which doesn't support the IPI exit requires
-			 * that the boot state is set to SIPI here.
+			 * Userland which doesn't support the IPI exit
+			 * requires that the boot state is set to SIPI
+			 * here.
 			 */
 			vlapic2 = vm_lapic(vlapic->vm, i);
 			vlapic2->boot_state = BS_SIPI;
-			CPU_SET(i, &ipimask);
+			break;
 		}
 
+		CPU_COPY(&dmask, &ipimask);
 		break;
 	case APIC_DELMODE_STARTUP:
+		if (!vlapic->ipi_exit) {
+			if (!phys)
+				break;
+
+			/*
+			 * Old bhyve versions don't support the IPI
+			 * exit. Translate it into the old style.
+			 */
+			i = vm_apicid2vcpuid(vlapic->vm, dest);
+			if (i >= vm_get_maxcpus(vlapic->vm) ||
+			    i == vlapic->vcpuid)
+				break;
+
+			/*
+			 * Ignore SIPIs in any state other than wait-for-SIPI
+			 */
+			vlapic2 = vm_lapic(vlapic->vm, i);
+			if (vlapic2->boot_state != BS_SIPI)
+				break;
+			vlapic2->boot_state = BS_RUNNING;
+
+			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
+			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
+			vmexit->u.spinup_ap.vcpu = i;
+			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
+
+			*retu = true;
+			break;
+		}
+
 		CPU_FOREACH_ISSET(i, &dmask) {
 			vlapic2 = vm_lapic(vlapic->vm, i);
+
 			/*
 			 * Ignore SIPIs in any state other than wait-for-SIPI
 			 */
@@ -1155,20 +1196,6 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 		vmexit->u.ipi.dmask = dmask;
 
 		*retu = true;
-
-		/*
-		 * Old bhyve versions don't support the IPI exit. Translate it
-		 * into the old style.
-		 */
-		if (!vlapic->ipi_exit) {
-			if (mode == APIC_DELMODE_STARTUP) {
-				vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
-				vmexit->u.spinup_ap.vcpu = CPU_FFS(&ipimask) - 1;
-				vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
-			} else {
-				*retu = false;
-			}
-		}
 	}
 
 	return (0);