git: eb9fac0edbc7 - main - bhyve: Refactor vmexit_suspend() a bit

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=eb9fac0edbc7ed5435f693ba3d3826363d8a7110

commit eb9fac0edbc7ed5435f693ba3d3826363d8a7110
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2023-06-19 19:46:32 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2023-06-19 19:46:32 +0000

    bhyve: Refactor vmexit_suspend() a bit
    
    Move some of its logic into fbsdrun_deletecpu().  This makes it easier
    to split vmexit handlers into a separate file, which in turn makes
    landing arm64 support easier.  Also increase the scope of the mutex and
    use it to synchronize updates to the vcpu mask.  No functional change
    intended.
    
    Reviewed by:    corvink, jhb
    MFC after:      2 weeks
    Sponsored by:   Innovate UK
    Differential Revision:  https://reviews.freebsd.org/D40573
---
 usr.sbin/bhyve/bhyverun.c | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 888fbe1cd8fc..9eef1ea7225d 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -553,17 +553,31 @@ fbsdrun_addcpu(struct vcpu_info *vi)
 	assert(error == 0);
 }
 
-static int
+static void
 fbsdrun_deletecpu(int vcpu)
 {
+	static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
+	static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
 
+	pthread_mutex_lock(&resetcpu_mtx);
 	if (!CPU_ISSET(vcpu, &cpumask)) {
 		fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
 		exit(4);
 	}
 
-	CPU_CLR_ATOMIC(vcpu, &cpumask);
-	return (CPU_EMPTY(&cpumask));
+	CPU_CLR(vcpu, &cpumask);
+
+	if (vcpu != BSP) {
+		pthread_cond_signal(&resetcpu_cond);
+		pthread_mutex_unlock(&resetcpu_mtx);
+		pthread_exit(NULL);
+		/* NOTREACHED */
+	}
+
+	while (!CPU_EMPTY(&cpumask)) {
+		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
+	}
+	pthread_mutex_unlock(&resetcpu_mtx);
 }
 
 static int
@@ -818,9 +832,6 @@ fail:
 	return (VMEXIT_ABORT);
 }
 
-static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
-
 static int
 vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 {
@@ -834,19 +845,6 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 
 	fbsdrun_deletecpu(vcpuid);
 
-	if (vcpuid != BSP) {
-		pthread_mutex_lock(&resetcpu_mtx);
-		pthread_cond_signal(&resetcpu_cond);
-		pthread_mutex_unlock(&resetcpu_mtx);
-		pthread_exit(NULL);
-	}
-
-	pthread_mutex_lock(&resetcpu_mtx);
-	while (!CPU_EMPTY(&cpumask)) {
-		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
-	}
-	pthread_mutex_unlock(&resetcpu_mtx);
-
 	switch (how) {
 	case VM_SUSPEND_RESET:
 		exit(0);