git: 5f7fc790c64f - stable/13 - bhyve: create all vcpus on startup

The branch stable/13 has been updated by corvink:

URL: https://cgit.FreeBSD.org/src/commit/?id=5f7fc790c64fb120cd6918db84e985d40b874661

commit 5f7fc790c64fb120cd6918db84e985d40b874661
Author:     Corvin Köhne <CorvinK@beckhoff.com>
AuthorDate: 2022-09-07 07:05:36 +0000
Commit:     Corvin Köhne <corvink@FreeBSD.org>
CommitDate: 2022-12-09 13:18:04 +0000

    bhyve: create all vcpus on startup
    
    vcpus could be restarted by the guest by sending an INIT SIPI SIPI
    sequence to a vcpu. That's not supported by bhyve yet but it will be
    supported in a future commit. So, create the vcpu threads only once on
    startup to make restarting a vcpu easier.
    
    MFC after:              2 weeks
    Differential Revision:  https://reviews.freebsd.org/D35621
    Sponsored by:           Beckhoff Automation GmbH & Co. KG
    
    (cherry picked from commit 9cc9abf409cc9d761f42b16b0dc8f5c0cc4f9a44)
---
 usr.sbin/bhyve/bhyverun.c  | 41 ++++++++++++++++-------------------------
 usr.sbin/bhyve/bhyverun.h  |  1 -
 usr.sbin/bhyve/spinup_ap.c |  4 ++--
 usr.sbin/bhyve/spinup_ap.h |  2 +-
 4 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 27166d9046e5..c1e0904d13ed 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -547,13 +547,11 @@ fbsdrun_start_thread(void *param)
 	return (NULL);
 }
 
-void
-fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip)
+static void
+fbsdrun_addcpu(struct vmctx *ctx, int newcpu, uint64_t rip, bool suspend)
 {
 	int error;
 
-	assert(fromcpu == BSP);
-
 	/*
 	 * The 'newcpu' must be activated in the context of 'fromcpu'. If
 	 * vm_activate_cpu() is delayed until newcpu's pthread starts running
@@ -566,6 +564,9 @@ fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip)
 
 	CPU_SET_ATOMIC(newcpu, &cpumask);
 
+	if (suspend)
+		vm_suspend_cpu(ctx, newcpu);
+
 	/*
 	 * Set up the vmexit struct to allow execution to start
 	 * at the given RIP
@@ -688,8 +689,7 @@ static int
 vmexit_spinup_ap(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 {
 
-	(void)spinup_ap(ctx, *pvcpu,
-		    vme->u.spinup_ap.vcpu, vme->u.spinup_ap.rip);
+	(void)spinup_ap(ctx, vme->u.spinup_ap.vcpu, vme->u.spinup_ap.rip);
 
 	return (VMEXIT_CONTINUE);
 }
@@ -1127,8 +1127,8 @@ do_open(const char *vmname)
 	return (ctx);
 }
 
-void
-spinup_vcpu(struct vmctx *ctx, int vcpu)
+static void
+spinup_vcpu(struct vmctx *ctx, int vcpu, bool suspend)
 {
 	int error;
 	uint64_t rip;
@@ -1140,7 +1140,7 @@ spinup_vcpu(struct vmctx *ctx, int vcpu)
 	error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
 	assert(error == 0);
 
-	fbsdrun_addcpu(ctx, BSP, vcpu, rip);
+	fbsdrun_addcpu(ctx, vcpu, rip, suspend);
 }
 
 static bool
@@ -1569,25 +1569,16 @@ main(int argc, char *argv[])
 	mt_vmm_info = calloc(guest_ncpus, sizeof(*mt_vmm_info));
 
 	/*
-	 * Add CPU 0
+	 * Add all vCPUs.
 	 */
-	fbsdrun_addcpu(ctx, BSP, BSP, rip);
-
+	for (int vcpu = 0; vcpu < guest_ncpus; vcpu++) {
+		bool suspend = (vcpu != BSP);
 #ifdef BHYVE_SNAPSHOT
-	/*
-	 * If we restore a VM, start all vCPUs now (including APs), otherwise,
-	 * let the guest OS to spin them up later via vmexits.
-	 */
-	if (restore_file != NULL) {
-		for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
-			if (vcpu == BSP)
-				continue;
-
-			fprintf(stdout, "spinning up vcpu no %d...\r\n", vcpu);
-			spinup_vcpu(ctx, vcpu);
-		}
-	}
+		if (restore_file != NULL)
+			suspend = false;
 #endif
+		spinup_vcpu(ctx, vcpu, suspend);
+	}
 
 	/*
 	 * Head off to the main event dispatch loop
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index d10604011d31..5bd7542d8533 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -44,7 +44,6 @@ uintptr_t paddr_host2guest(struct vmctx *ctx, void *addr);
 #endif
 
 void fbsdrun_set_capabilities(struct vmctx *ctx, int cpu);
-void fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip);
 int  fbsdrun_virtio_msix(void);
 
 #endif
diff --git a/usr.sbin/bhyve/spinup_ap.c b/usr.sbin/bhyve/spinup_ap.c
index 7c4186f5ed15..2b7e602f8003 100644
--- a/usr.sbin/bhyve/spinup_ap.c
+++ b/usr.sbin/bhyve/spinup_ap.c
@@ -77,7 +77,7 @@ spinup_ap_realmode(struct vmctx *ctx, int newcpu, uint64_t *rip)
 }
 
 int
-spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
+spinup_ap(struct vmctx *ctx, int newcpu, uint64_t rip)
 {
 	int error;
 
@@ -100,7 +100,7 @@ spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 
 	spinup_ap_realmode(ctx, newcpu, &rip);
 
-	fbsdrun_addcpu(ctx, vcpu, newcpu, rip);
+	vm_resume_cpu(ctx, newcpu);
 
 	return (newcpu);
 }
diff --git a/usr.sbin/bhyve/spinup_ap.h b/usr.sbin/bhyve/spinup_ap.h
index 226542f6c331..e67bac42944b 100644
--- a/usr.sbin/bhyve/spinup_ap.h
+++ b/usr.sbin/bhyve/spinup_ap.h
@@ -31,6 +31,6 @@
 #ifndef	_SPINUP_AP_H_
 #define	_SPINUP_AP_H_
 
-int spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip);
+int spinup_ap(struct vmctx *ctx, int newcpu, uint64_t rip);
 
 #endif