svn commit: r366834 - head/sys/amd64/vmm/intel
Mark Johnston
markj at FreeBSD.org
Mon Oct 19 15:24:37 UTC 2020
Author: markj
Date: Mon Oct 19 15:24:35 2020
New Revision: 366834
URL: https://svnweb.freebsd.org/changeset/base/366834
Log:
vmx: Implement pmap (de)activation in C
Rewrite the code that maintains pm_active and invalidates EPTP-tagged
TLB entries in C. Previously this work was done in vmx_enter_guest(),
in assembly, but there is no good reason for that and it makes the TLB
invalidation algorithm for nested page tables harder to review.
No functional change intended. Now, an error from the invept
instruction results in a kernel panic rather than a vmexit. Such errors
should occur only as a result of VMM bugs.
Reviewed by: grehan, kib
MFC after: 2 weeks
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D26830
Modified:
head/sys/amd64/vmm/intel/vmx.c
head/sys/amd64/vmm/intel/vmx.h
head/sys/amd64/vmm/intel/vmx_genassym.c
head/sys/amd64/vmm/intel/vmx_support.S
Modified: head/sys/amd64/vmm/intel/vmx.c
==============================================================================
--- head/sys/amd64/vmm/intel/vmx.c Mon Oct 19 13:10:21 2020 (r366833)
+++ head/sys/amd64/vmm/intel/vmx.c Mon Oct 19 15:24:35 2020 (r366834)
@@ -2835,7 +2835,6 @@ vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, str
switch (rc) {
case VMX_VMRESUME_ERROR:
case VMX_VMLAUNCH_ERROR:
- case VMX_INVEPT_ERROR:
vmexit->u.vmx.inst_type = rc;
break;
default:
@@ -2940,6 +2939,29 @@ vmx_dr_leave_guest(struct vmxctx *vmxctx)
write_rflags(read_rflags() | vmxctx->host_tf);
}
+static __inline void
+vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)
+{
+ long eptgen;
+ int cpu;
+
+ cpu = curcpu;
+
+ CPU_SET_ATOMIC(cpu, &pmap->pm_active);
+ eptgen = atomic_load_long(&pmap->pm_eptgen);
+ if (eptgen != vmx->eptgen[cpu]) {
+ vmx->eptgen[cpu] = eptgen;
+ invept(INVEPT_TYPE_SINGLE_CONTEXT,
+ (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 });
+ }
+}
+
+static __inline void
+vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)
+{
+ CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
+}
+
static int
vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
struct vm_eventinfo *evinfo)
@@ -3088,11 +3110,19 @@ vmx_run(void *arg, int vcpu, register_t rip, pmap_t pm
*/
vmx_msr_guest_enter_tsc_aux(vmx, vcpu);
- vmx_run_trace(vmx, vcpu);
vmx_dr_enter_guest(vmxctx);
+
+ /*
+ * Mark the EPT as active on this host CPU and invalidate
+ * EPTP-tagged TLB entries if required.
+ */
+ vmx_pmap_activate(vmx, pmap);
+
+ vmx_run_trace(vmx, vcpu);
rc = vmx_enter_guest(vmxctx, vmx, launched);
- vmx_dr_leave_guest(vmxctx);
+ vmx_pmap_deactivate(vmx, pmap);
+ vmx_dr_leave_guest(vmxctx);
vmx_msr_guest_exit_tsc_aux(vmx, vcpu);
bare_lgdt(&gdtr);
Modified: head/sys/amd64/vmm/intel/vmx.h
==============================================================================
--- head/sys/amd64/vmm/intel/vmx.h Mon Oct 19 13:10:21 2020 (r366833)
+++ head/sys/amd64/vmm/intel/vmx.h Mon Oct 19 15:24:35 2020 (r366834)
@@ -142,7 +142,6 @@ CTASSERT((offsetof(struct vmx, pir_desc[0]) & 63) == 0
#define VMX_GUEST_VMEXIT 0
#define VMX_VMRESUME_ERROR 1
#define VMX_VMLAUNCH_ERROR 2
-#define VMX_INVEPT_ERROR 3
int vmx_enter_guest(struct vmxctx *ctx, struct vmx *vmx, int launched);
void vmx_call_isr(uintptr_t entry);
Modified: head/sys/amd64/vmm/intel/vmx_genassym.c
==============================================================================
--- head/sys/amd64/vmm/intel/vmx_genassym.c Mon Oct 19 13:10:21 2020 (r366833)
+++ head/sys/amd64/vmm/intel/vmx_genassym.c Mon Oct 19 15:24:35 2020 (r366834)
@@ -70,17 +70,12 @@ ASSYM(VMXCTX_HOST_RSP, offsetof(struct vmxctx, host_rs
ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx));
ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status));
-ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap));
-ASSYM(VMX_EPTGEN, offsetof(struct vmx, eptgen));
-ASSYM(VMX_EPTP, offsetof(struct vmx, eptp));
-
ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID);
ASSYM(VM_FAIL_VALID, VM_FAIL_VALID);
ASSYM(VMX_GUEST_VMEXIT, VMX_GUEST_VMEXIT);
ASSYM(VMX_VMRESUME_ERROR, VMX_VMRESUME_ERROR);
ASSYM(VMX_VMLAUNCH_ERROR, VMX_VMLAUNCH_ERROR);
-ASSYM(VMX_INVEPT_ERROR, VMX_INVEPT_ERROR);
ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
Modified: head/sys/amd64/vmm/intel/vmx_support.S
==============================================================================
--- head/sys/amd64/vmm/intel/vmx_support.S Mon Oct 19 13:10:21 2020 (r366833)
+++ head/sys/amd64/vmm/intel/vmx_support.S Mon Oct 19 15:24:35 2020 (r366834)
@@ -148,33 +148,6 @@ ENTRY(vmx_enter_guest)
*/
VMX_HOST_SAVE
- /*
- * Activate guest pmap on this cpu.
- */
- movq VMXCTX_PMAP(%rdi), %r11
- movl PCPU(CPUID), %eax
- LK btsl %eax, PM_ACTIVE(%r11)
-
- /*
- * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
- * then we must invalidate all mappings associated with this EPTP.
- */
- movq PM_EPTGEN(%r11), %r10
- cmpq %r10, VMX_EPTGEN(%rsi, %rax, 8)
- je guest_restore
-
- /* Refresh 'vmx->eptgen[curcpu]' */
- movq %r10, VMX_EPTGEN(%rsi, %rax, 8)
-
- /* Setup the invept descriptor on the host stack */
- mov %rsp, %r11
- movq VMX_EPTP(%rsi), %rax
- movq %rax, -16(%r11)
- movq $0x0, -8(%r11)
- mov $0x1, %eax /* Single context invalidate */
- invept -16(%r11), %rax
- jbe invept_error /* Check invept instruction error */
-
guest_restore:
movl %edx, %r8d
cmpb $0, guest_l1d_flush_sw(%rip)
@@ -208,10 +181,6 @@ do_launch:
movl $VMX_VMLAUNCH_ERROR, %eax
jmp decode_inst_error
-invept_error:
- movl $VMX_INVEPT_ERROR, %eax
- jmp decode_inst_error
-
decode_inst_error:
movl $VM_FAIL_VALID, %r11d
jz inst_error
@@ -224,13 +193,6 @@ inst_error:
* it as a scratch register beyond this point.
*/
- /*
- * Deactivate guest pmap from this cpu.
- */
- movq VMXCTX_PMAP(%rdi), %r11
- movl PCPU(CPUID), %r10d
- LK btrl %r10d, PM_ACTIVE(%r11)
-
VMX_HOST_RESTORE
VLEAVE
ret
@@ -248,13 +210,6 @@ vmx_exit_guest_flush_rsb:
*/
VMX_GUEST_SAVE
- /*
- * Deactivate guest pmap from this cpu.
- */
- movq VMXCTX_PMAP(%rdi), %r11
- movl PCPU(CPUID), %r10d
- LK btrl %r10d, PM_ACTIVE(%r11)
-
VMX_HOST_RESTORE
VMX_GUEST_CLOBBER
@@ -290,13 +245,6 @@ vmx_exit_guest:
* Save guest state that is not automatically saved in the vmcs.
*/
VMX_GUEST_SAVE
-
- /*
- * Deactivate guest pmap from this cpu.
- */
- movq VMXCTX_PMAP(%rdi), %r11
- movl PCPU(CPUID), %r10d
- LK btrl %r10d, PM_ACTIVE(%r11)
VMX_HOST_RESTORE
More information about the svn-src-all
mailing list