git: 1ee7a8fa58c3 - main - arm64/vmm: Handle VM_EXITCODE_SUSPENDED

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=1ee7a8fa58c3fe7161683f2a7d2e37dc5c672b46

commit 1ee7a8fa58c3fe7161683f2a7d2e37dc5c672b46
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2024-04-29 14:19:27 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2024-04-29 14:19:27 +0000

    arm64/vmm: Handle VM_EXITCODE_SUSPENDED
    
    This is required for bhyve reboot to work.  In particular, unless we
    suspend vcpu threads here, vm_reinit() will fail with EBUSY.
    
    The implementation is copied from amd64; in the not-too-distant future
    the amd64 and arm64 copies of vmm.c and vmm_dev.c will be merged, so
    for now it's useful to minimize diffs between amd64 and arm64.
    
    Reviewed by:    corvink, andrew
    MFC after:      2 weeks
    Sponsored by:   Innovate UK
    Differential Revision:  https://reviews.freebsd.org/D44934
---
 sys/arm64/vmm/vmm.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
index 2685e5869b4f..a2cc63448f19 100644
--- a/sys/arm64/vmm/vmm.c
+++ b/sys/arm64/vmm/vmm.c
@@ -1716,6 +1716,54 @@ vm_handle_paging(struct vcpu *vcpu, bool *retu)
 	return (0);
 }
 
+static int
+vm_handle_suspend(struct vcpu *vcpu, bool *retu)
+{
+	struct vm *vm = vcpu->vm;
+	int error, i;
+	struct thread *td;
+
+	error = 0;
+	td = curthread;
+
+	CPU_SET_ATOMIC(vcpu->vcpuid, &vm->suspended_cpus);
+
+	/*
+	 * Wait until all 'active_cpus' have suspended themselves.
+	 *
+	 * Since a VM may be suspended at any time including when one or
+	 * more vcpus are doing a rendezvous we need to call the rendezvous
+	 * handler while we are waiting to prevent a deadlock.
+	 */
+	vcpu_lock(vcpu);
+	while (error == 0) {
+		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0)
+			break;
+
+		vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
+		msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
+		vcpu_require_state_locked(vcpu, VCPU_FROZEN);
+		if (td_ast_pending(td, TDA_SUSPEND)) {
+			vcpu_unlock(vcpu);
+			error = thread_check_susp(td, false);
+			vcpu_lock(vcpu);
+		}
+	}
+	vcpu_unlock(vcpu);
+
+	/*
+	 * Wakeup the other sleeping vcpus and return to userspace.
+	 */
+	for (i = 0; i < vm->maxcpus; i++) {
+		if (CPU_ISSET(i, &vm->suspended_cpus)) {
+			vcpu_notify_event(vm_vcpu(vm, i));
+		}
+	}
+
+	*retu = true;
+	return (error);
+}
+
 int
 vm_run(struct vcpu *vcpu)
 {
@@ -1788,6 +1836,11 @@ restart:
 			error = vm_handle_paging(vcpu, &retu);
 			break;
 
+		case VM_EXITCODE_SUSPENDED:
+			vcpu->nextpc = vme->pc;
+			error = vm_handle_suspend(vcpu, &retu);
+			break;
+
 		default:
 			/* Handle in userland */
 			vcpu->nextpc = vme->pc;