git: d3916eace506 - main - riscv/vmm: Initial import.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 31 Oct 2024 20:25:33 UTC
The branch main has been updated by br:
URL: https://cgit.FreeBSD.org/src/commit/?id=d3916eace506b8ab23537223f5c92924636a1c41
commit d3916eace506b8ab23537223f5c92924636a1c41
Author: Ruslan Bukin <br@FreeBSD.org>
AuthorDate: 2024-10-31 15:33:07 +0000
Commit: Ruslan Bukin <br@FreeBSD.org>
CommitDate: 2024-10-31 20:24:12 +0000
riscv/vmm: Initial import.
Add kernel code for 'H' — Hypervisor Extension[1] to support
virtualization on RISC-V ISA.
This comes with a separate userspace patch allowing us to boot
unmodified freebsd/riscv guest. Other operating systems are untested.
This also comes with a U-Boot port that is configured to run in bhyve
guest environment — in RISC-V virtual supervisor mode.
The vmm SBI code emulates RISC-V machine-mode for the guest, handling
SBI calls partly in vmm kernel and partly in bhyve userspace.
Developed in Spike simulator during short period of time, the support
is considered experimental. The first real hardware with hypervisor
spec included should have just reached the market, so this was tested
in Spike and QEMU only. Note that this depends on Sstc extension
presence in the hardware (both Spike and QEMU have it).
Note that booting multiple guests at the same time is not tested and
may require additional work. Some TODOs are indicated within the
code, and some listed in the project's home page[2].
Many thanks to Jessica Clarke, Mitchell Horne and Mark Johnston
for help with parts, test and review.
1. https://riscv.org/technical/specifications/
2. https://wiki.freebsd.org/riscv/bhyve
Sponsored by: UK Research and Innovation
Differential Revision: https://reviews.freebsd.org/D45553
---
sys/conf/files.riscv | 10 +
sys/conf/kern.mk | 2 +-
sys/riscv/include/cpu.h | 4 +-
sys/riscv/include/elf.h | 1 +
sys/riscv/include/md_var.h | 1 +
sys/riscv/include/riscvreg.h | 38 +
sys/riscv/include/vmm.h | 328 ++++++
sys/riscv/include/vmm_dev.h | 258 +++++
sys/riscv/include/vmm_instruction_emul.h | 85 ++
sys/riscv/include/vmm_snapshot.h | 1 +
sys/riscv/riscv/genassym.c | 34 +
sys/riscv/riscv/identcpu.c | 4 +
sys/riscv/vmm/riscv.h | 132 +++
sys/riscv/vmm/vmm.c | 1606 ++++++++++++++++++++++++++++++
sys/riscv/vmm/vmm_aplic.c | 528 ++++++++++
sys/riscv/vmm/vmm_aplic.h | 54 +
sys/riscv/vmm/vmm_dev_machdep.c | 126 +++
sys/riscv/vmm/vmm_instruction_emul.c | 109 ++
sys/riscv/vmm/vmm_riscv.c | 922 +++++++++++++++++
sys/riscv/vmm/vmm_sbi.c | 179 ++++
sys/riscv/vmm/vmm_stat.h | 43 +
sys/riscv/vmm/vmm_switch.S | 220 ++++
22 files changed, 4682 insertions(+), 3 deletions(-)
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
index 89fb6f3cbe5a..f75fee72fde2 100644
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -12,6 +12,8 @@ dev/pci/pci_host_generic.c optional pci
dev/pci/pci_host_generic_fdt.c optional pci fdt
dev/uart/uart_cpu_fdt.c optional uart fdt
dev/uart/uart_dev_lowrisc.c optional uart_lowrisc
+dev/vmm/vmm_dev.c optional vmm
+dev/vmm/vmm_stat.c optional vmm
dev/xilinx/axi_quad_spi.c optional xilinx_spi
dev/xilinx/axidma.c optional axidma xdma
dev/xilinx/if_xae.c optional xae
@@ -44,6 +46,7 @@ riscv/riscv/dump_machdep.c standard
riscv/riscv/elf_machdep.c standard
riscv/riscv/exception.S standard
riscv/riscv/exec_machdep.c standard
+riscv/riscv/fpe.c optional vmm
riscv/riscv/gdb_machdep.c optional gdb
riscv/riscv/intc.c standard
riscv/riscv/identcpu.c standard
@@ -72,6 +75,13 @@ riscv/riscv/timer.c standard
riscv/riscv/uio_machdep.c standard
riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack
riscv/riscv/vm_machdep.c standard
+riscv/vmm/vmm.c optional vmm
+riscv/vmm/vmm_aplic.c optional vmm
+riscv/vmm/vmm_dev_machdep.c optional vmm
+riscv/vmm/vmm_instruction_emul.c optional vmm
+riscv/vmm/vmm_riscv.c optional vmm
+riscv/vmm/vmm_sbi.c optional vmm
+riscv/vmm/vmm_switch.S optional vmm
# Zstd
contrib/zstd/lib/freebsd/zstd_kfreebsd.c optional zstdio compile-with ${ZSTD_C}
diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk
index 4c3014f2abb6..2f451f9286a6 100644
--- a/sys/conf/kern.mk
+++ b/sys/conf/kern.mk
@@ -163,7 +163,7 @@ INLINE_LIMIT?= 8000
# code model as "medium" and "medany" respectively.
#
.if ${MACHINE_CPUARCH} == "riscv"
-CFLAGS+= -march=rv64imafdc
+CFLAGS+= -march=rv64imafdch
CFLAGS+= -mabi=lp64
CFLAGS.clang+= -mcmodel=medium
CFLAGS.gcc+= -mcmodel=medany
diff --git a/sys/riscv/include/cpu.h b/sys/riscv/include/cpu.h
index cbf660dcda0c..0c33adb2abcd 100644
--- a/sys/riscv/include/cpu.h
+++ b/sys/riscv/include/cpu.h
@@ -47,8 +47,6 @@
#define cpu_spinwait() /* nothing */
#define cpu_lock_delay() DELAY(1)
-#ifdef _KERNEL
-
/*
* Core manufacturer IDs, as reported by the mvendorid CSR.
*/
@@ -89,6 +87,8 @@
#define MMU_SV48 0x2 /* 4-level paging */
#define MMU_SV57 0x4 /* 5-level paging */
+#ifdef _KERNEL
+
extern char btext[];
extern char etext[];
diff --git a/sys/riscv/include/elf.h b/sys/riscv/include/elf.h
index a14d6859902b..78788abe1e57 100644
--- a/sys/riscv/include/elf.h
+++ b/sys/riscv/include/elf.h
@@ -80,6 +80,7 @@ __ElfType(Auxinfo);
#define HWCAP_ISA_F HWCAP_ISA_BIT('f')
#define HWCAP_ISA_D HWCAP_ISA_BIT('d')
#define HWCAP_ISA_C HWCAP_ISA_BIT('c')
+#define HWCAP_ISA_H HWCAP_ISA_BIT('h')
#define HWCAP_ISA_G \
(HWCAP_ISA_I | HWCAP_ISA_M | HWCAP_ISA_A | HWCAP_ISA_F | HWCAP_ISA_D)
#define HWCAP_ISA_B HWCAP_ISA_BIT('b')
diff --git a/sys/riscv/include/md_var.h b/sys/riscv/include/md_var.h
index d9404db914a2..85a51e30b4a7 100644
--- a/sys/riscv/include/md_var.h
+++ b/sys/riscv/include/md_var.h
@@ -42,6 +42,7 @@ extern register_t mimpid;
extern u_int mmu_caps;
/* Supervisor-mode extension support */
+extern bool has_hyp;
extern bool has_sstc;
extern bool has_sscofpmf;
extern bool has_svpbmt;
diff --git a/sys/riscv/include/riscvreg.h b/sys/riscv/include/riscvreg.h
index e1ad09acedc8..23feb419d04c 100644
--- a/sys/riscv/include/riscvreg.h
+++ b/sys/riscv/include/riscvreg.h
@@ -47,9 +47,15 @@
#define SCAUSE_STORE_ACCESS_FAULT 7
#define SCAUSE_ECALL_USER 8
#define SCAUSE_ECALL_SUPERVISOR 9
+#define SCAUSE_VIRTUAL_SUPERVISOR_ECALL 10
+#define SCAUSE_MACHINE_ECALL 11
#define SCAUSE_INST_PAGE_FAULT 12
#define SCAUSE_LOAD_PAGE_FAULT 13
#define SCAUSE_STORE_PAGE_FAULT 15
+#define SCAUSE_FETCH_GUEST_PAGE_FAULT 20
+#define SCAUSE_LOAD_GUEST_PAGE_FAULT 21
+#define SCAUSE_VIRTUAL_INSTRUCTION 22
+#define SCAUSE_STORE_GUEST_PAGE_FAULT 23
#define SSTATUS_UIE (1 << 0)
#define SSTATUS_SIE (1 << 1)
@@ -116,6 +122,17 @@
#define MSTATUS_PRV_H 2 /* hypervisor */
#define MSTATUS_PRV_M 3 /* machine */
+#define HSTATUS_VSBE (1 << 5)
+#define HSTATUS_GVA (1 << 6)
+#define HSTATUS_SPV (1 << 7)
+#define HSTATUS_SPVP (1 << 8)
+#define HSTATUS_HU (1 << 9)
+#define HSTATUS_VGEIN_S 12
+#define HSTATUS_VGEIN_M (0xf << HSTATUS_VGEIN_S)
+#define HSTATUS_VTVM (1 << 20)
+#define HSTATUS_VTW (1 << 21)
+#define HSTATUS_VTSR (1 << 22)
+
#define MIE_USIE (1 << 0)
#define MIE_SSIE (1 << 1)
#define MIE_HSIE (1 << 2)
@@ -143,10 +160,31 @@
#define MIP_SEIP (1 << 9)
+#define HVIP_VSSIP (1 << 2)
+#define HVIP_VSTIP (1 << 6)
+#define HVIP_VSEIP (1 << 10)
+
+#define HIE_VSSIE (1 << 2)
+#define HIE_VSTIE (1 << 6)
+#define HIE_VSEIE (1 << 10)
+#define HIE_SGEIE (1 << 12)
+
/* Note: sip register has no SIP_STIP bit in Spike simulator */
#define SIP_SSIP (1 << 1)
#define SIP_STIP (1 << 5)
+#define HENVCFG_STCE (1UL << 63)
+#define HENVCFG_PBMTE (1UL << 62)
+#define HENVCFG_CBZE (1UL << 7)
+#define HENVCFG_CBCFE (1UL << 6)
+#define HENVCFG_CBIE_S 4
+#define HENVCFG_CBIE_M (0x3 << HENVCFG_CBIE_S)
+#define HENVCFG_FIOM (1UL << 0)
+
+#define HCOUNTEREN_CY (1UL << 0) /* Cycle */
+#define HCOUNTEREN_TM (1UL << 1) /* Time */
+#define HCOUNTEREN_IR (1UL << 2) /* Instret */
+
#define SATP_PPN_S 0
#define SATP_PPN_M (0xfffffffffffUL << SATP_PPN_S)
#define SATP_ASID_S 44
diff --git a/sys/riscv/include/vmm.h b/sys/riscv/include/vmm.h
new file mode 100644
index 000000000000..e148cd95b522
--- /dev/null
+++ b/sys/riscv/include/vmm.h
@@ -0,0 +1,328 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * Copyright (c) 2024 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by the University of Cambridge Computer
+ * Laboratory (Department of Computer Science and Technology) under Innovate
+ * UK project 105694, "Digital Security by Design (DSbD) Technology Platform
+ * Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_H_
+#define _VMM_H_
+
+#include <sys/param.h>
+#include <sys/cpuset.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include "pte.h"
+#include "pmap.h"
+
+struct vcpu;
+
+enum vm_suspend_how {
+ VM_SUSPEND_NONE,
+ VM_SUSPEND_RESET,
+ VM_SUSPEND_POWEROFF,
+ VM_SUSPEND_HALT,
+ VM_SUSPEND_LAST
+};
+
+/*
+ * Identifiers for architecturally defined registers.
+ */
+enum vm_reg_name {
+ VM_REG_GUEST_ZERO = 0,
+ VM_REG_GUEST_RA,
+ VM_REG_GUEST_SP,
+ VM_REG_GUEST_GP,
+ VM_REG_GUEST_TP,
+ VM_REG_GUEST_T0,
+ VM_REG_GUEST_T1,
+ VM_REG_GUEST_T2,
+ VM_REG_GUEST_S0,
+ VM_REG_GUEST_S1,
+ VM_REG_GUEST_A0,
+ VM_REG_GUEST_A1,
+ VM_REG_GUEST_A2,
+ VM_REG_GUEST_A3,
+ VM_REG_GUEST_A4,
+ VM_REG_GUEST_A5,
+ VM_REG_GUEST_A6,
+ VM_REG_GUEST_A7,
+ VM_REG_GUEST_S2,
+ VM_REG_GUEST_S3,
+ VM_REG_GUEST_S4,
+ VM_REG_GUEST_S5,
+ VM_REG_GUEST_S6,
+ VM_REG_GUEST_S7,
+ VM_REG_GUEST_S8,
+ VM_REG_GUEST_S9,
+ VM_REG_GUEST_S10,
+ VM_REG_GUEST_S11,
+ VM_REG_GUEST_T3,
+ VM_REG_GUEST_T4,
+ VM_REG_GUEST_T5,
+ VM_REG_GUEST_T6,
+ VM_REG_GUEST_SEPC,
+ VM_REG_LAST
+};
+
+#define VM_INTINFO_VECTOR(info) ((info) & 0xff)
+#define VM_INTINFO_DEL_ERRCODE 0x800
+#define VM_INTINFO_RSVD 0x7ffff000
+#define VM_INTINFO_VALID 0x80000000
+#define VM_INTINFO_TYPE 0x700
+#define VM_INTINFO_HWINTR (0 << 8)
+#define VM_INTINFO_NMI (2 << 8)
+#define VM_INTINFO_HWEXCEPTION (3 << 8)
+#define VM_INTINFO_SWINTR (4 << 8)
+
+#define VM_MAX_SUFFIXLEN 15
+
+#ifdef _KERNEL
+
+#define VM_MAX_NAMELEN 32
+
+struct vm;
+struct vm_exception;
+struct vm_exit;
+struct vm_run;
+struct vm_object;
+struct vm_guest_paging;
+struct vm_aplic_descr;
+struct pmap;
+
+struct vm_eventinfo {
+ void *rptr; /* rendezvous cookie */
+ int *sptr; /* suspend cookie */
+ int *iptr; /* reqidle cookie */
+};
+
+int vm_create(const char *name, struct vm **retvm);
+struct vcpu *vm_alloc_vcpu(struct vm *vm, int vcpuid);
+void vm_disable_vcpu_creation(struct vm *vm);
+void vm_slock_vcpus(struct vm *vm);
+void vm_unlock_vcpus(struct vm *vm);
+void vm_destroy(struct vm *vm);
+int vm_reinit(struct vm *vm);
+const char *vm_name(struct vm *vm);
+
+/*
+ * APIs that modify the guest memory map require all vcpus to be frozen.
+ */
+void vm_slock_memsegs(struct vm *vm);
+void vm_xlock_memsegs(struct vm *vm);
+void vm_unlock_memsegs(struct vm *vm);
+int vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t off,
+ size_t len, int prot, int flags);
+int vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len);
+int vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem);
+void vm_free_memseg(struct vm *vm, int ident);
+
+/*
+ * APIs that inspect the guest memory map require only a *single* vcpu to
+ * be frozen. This acts like a read lock on the guest memory map since any
+ * modification requires *all* vcpus to be frozen.
+ */
+int vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
+ vm_ooffset_t *segoff, size_t *len, int *prot, int *flags);
+int vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
+ struct vm_object **objptr);
+vm_paddr_t vmm_sysmem_maxaddr(struct vm *vm);
+void *vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len,
+ int prot, void **cookie);
+void *vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len,
+ int prot, void **cookie);
+void vm_gpa_release(void *cookie);
+bool vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa);
+
+int vm_gla2gpa_nofault(struct vcpu *vcpu, struct vm_guest_paging *paging,
+ uint64_t gla, int prot, uint64_t *gpa, int *is_fault);
+
+uint16_t vm_get_maxcpus(struct vm *vm);
+void vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
+ uint16_t *threads, uint16_t *maxcpus);
+int vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
+ uint16_t threads, uint16_t maxcpus);
+int vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval);
+int vm_set_register(struct vcpu *vcpu, int reg, uint64_t val);
+int vm_run(struct vcpu *vcpu);
+int vm_suspend(struct vm *vm, enum vm_suspend_how how);
+void* vm_get_cookie(struct vm *vm);
+int vcpu_vcpuid(struct vcpu *vcpu);
+void *vcpu_get_cookie(struct vcpu *vcpu);
+struct vm *vcpu_vm(struct vcpu *vcpu);
+struct vcpu *vm_vcpu(struct vm *vm, int cpu);
+int vm_get_capability(struct vcpu *vcpu, int type, int *val);
+int vm_set_capability(struct vcpu *vcpu, int type, int val);
+int vm_activate_cpu(struct vcpu *vcpu);
+int vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu);
+int vm_resume_cpu(struct vm *vm, struct vcpu *vcpu);
+int vm_inject_exception(struct vcpu *vcpu, uint64_t scause);
+int vm_attach_aplic(struct vm *vm, struct vm_aplic_descr *descr);
+int vm_assert_irq(struct vm *vm, uint32_t irq);
+int vm_deassert_irq(struct vm *vm, uint32_t irq);
+int vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
+ int func);
+struct vm_exit *vm_exitinfo(struct vcpu *vcpu);
+void vm_exit_suspended(struct vcpu *vcpu, uint64_t pc);
+void vm_exit_debug(struct vcpu *vcpu, uint64_t pc);
+void vm_exit_rendezvous(struct vcpu *vcpu, uint64_t pc);
+void vm_exit_astpending(struct vcpu *vcpu, uint64_t pc);
+
+cpuset_t vm_active_cpus(struct vm *vm);
+cpuset_t vm_debug_cpus(struct vm *vm);
+cpuset_t vm_suspended_cpus(struct vm *vm);
+
+static __inline int
+vcpu_rendezvous_pending(struct vm_eventinfo *info)
+{
+
+ return (*((uintptr_t *)(info->rptr)) != 0);
+}
+
+static __inline int
+vcpu_suspended(struct vm_eventinfo *info)
+{
+
+ return (*info->sptr);
+}
+
+int vcpu_debugged(struct vcpu *vcpu);
+
+enum vcpu_state {
+ VCPU_IDLE,
+ VCPU_FROZEN,
+ VCPU_RUNNING,
+ VCPU_SLEEPING,
+};
+
+int vcpu_set_state(struct vcpu *vcpu, enum vcpu_state state, bool from_idle);
+enum vcpu_state vcpu_get_state(struct vcpu *vcpu, int *hostcpu);
+
+static int __inline
+vcpu_is_running(struct vcpu *vcpu, int *hostcpu)
+{
+ return (vcpu_get_state(vcpu, hostcpu) == VCPU_RUNNING);
+}
+
+#ifdef _SYS_PROC_H_
+static int __inline
+vcpu_should_yield(struct vcpu *vcpu)
+{
+ struct thread *td;
+
+ td = curthread;
+ return (td->td_ast != 0 || td->td_owepreempt != 0);
+}
+#endif
+
+void *vcpu_stats(struct vcpu *vcpu);
+void vcpu_notify_event(struct vcpu *vcpu);
+
+enum vm_reg_name vm_segment_name(int seg_encoding);
+
+#endif /* _KERNEL */
+
+#define VM_DIR_READ 0
+#define VM_DIR_WRITE 1
+
+#define VM_GP_M_MASK 0x1f
+#define VM_GP_MMU_ENABLED (1 << 5)
+
+struct vm_guest_paging {
+ int flags;
+ int padding;
+};
+
+struct vie {
+ uint8_t access_size:4, sign_extend:1, dir:1, unused:2;
+ enum vm_reg_name reg;
+};
+
+struct vre {
+ uint32_t inst_syndrome;
+ uint8_t dir:1, unused:7;
+ enum vm_reg_name reg;
+};
+
+/*
+ * Identifiers for optional vmm capabilities
+ */
+enum vm_cap_type {
+ VM_CAP_UNRESTRICTED_GUEST,
+ VM_CAP_MAX
+};
+
+enum vm_exitcode {
+ VM_EXITCODE_BOGUS,
+ VM_EXITCODE_ECALL,
+ VM_EXITCODE_HYP,
+ VM_EXITCODE_PAGING,
+ VM_EXITCODE_SUSPENDED,
+ VM_EXITCODE_DEBUG,
+ VM_EXITCODE_INST_EMUL,
+ VM_EXITCODE_WFI,
+ VM_EXITCODE_MAX
+};
+
+struct vm_exit {
+ uint64_t scause;
+ uint64_t sepc;
+ uint64_t stval;
+ uint64_t htval;
+ uint64_t htinst;
+ enum vm_exitcode exitcode;
+ int inst_length;
+ uint64_t pc;
+ union {
+ struct {
+ uint64_t gpa;
+ } paging;
+
+ struct {
+ uint64_t gpa;
+ struct vm_guest_paging paging;
+ struct vie vie;
+ } inst_emul;
+
+ struct {
+ uint64_t args[8];
+ } ecall;
+
+ struct {
+ enum vm_suspend_how how;
+ } suspended;
+
+ struct {
+ uint64_t scause;
+ } hyp;
+ } u;
+};
+
+#endif /* _VMM_H_ */
diff --git a/sys/riscv/include/vmm_dev.h b/sys/riscv/include/vmm_dev.h
new file mode 100644
index 000000000000..a21528a8dc68
--- /dev/null
+++ b/sys/riscv/include/vmm_dev.h
@@ -0,0 +1,258 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * Copyright (c) 2024 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by the University of Cambridge Computer
+ * Laboratory (Department of Computer Science and Technology) under Innovate
+ * UK project 105694, "Digital Security by Design (DSbD) Technology Platform
+ * Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_DEV_H_
+#define _VMM_DEV_H_
+
+struct vm_memmap {
+ vm_paddr_t gpa;
+ int segid; /* memory segment */
+ vm_ooffset_t segoff; /* offset into memory segment */
+ size_t len; /* mmap length */
+ int prot; /* RWX */
+ int flags;
+};
+#define VM_MEMMAP_F_WIRED 0x01
+
+struct vm_munmap {
+ vm_paddr_t gpa;
+ size_t len;
+};
+
+#define VM_MEMSEG_NAME(m) ((m)->name[0] != '\0' ? (m)->name : NULL)
+struct vm_memseg {
+ int segid;
+ size_t len;
+ char name[VM_MAX_SUFFIXLEN + 1];
+};
+
+struct vm_register {
+ int cpuid;
+ int regnum; /* enum vm_reg_name */
+ uint64_t regval;
+};
+
+struct vm_register_set {
+ int cpuid;
+ unsigned int count;
+ const int *regnums; /* enum vm_reg_name */
+ uint64_t *regvals;
+};
+
+struct vm_run {
+ int cpuid;
+ cpuset_t *cpuset; /* CPU set storage */
+ size_t cpusetsize;
+ struct vm_exit *vm_exit;
+};
+
+struct vm_exception {
+ int cpuid;
+ uint64_t scause;
+};
+
+struct vm_msi {
+ uint64_t msg;
+ uint64_t addr;
+ int bus;
+ int slot;
+ int func;
+};
+
+struct vm_capability {
+ int cpuid;
+ enum vm_cap_type captype;
+ int capval;
+ int allcpus;
+};
+
+#define MAX_VM_STATS 64
+struct vm_stats {
+ int cpuid; /* in */
+ int index; /* in */
+ int num_entries; /* out */
+ struct timeval tv;
+ uint64_t statbuf[MAX_VM_STATS];
+};
+struct vm_stat_desc {
+ int index; /* in */
+ char desc[128]; /* out */
+};
+
+struct vm_suspend {
+ enum vm_suspend_how how;
+};
+
+struct vm_gla2gpa {
+ int vcpuid; /* inputs */
+ int prot; /* PROT_READ or PROT_WRITE */
+ uint64_t gla;
+ struct vm_guest_paging paging;
+ int fault; /* outputs */
+ uint64_t gpa;
+};
+
+struct vm_activate_cpu {
+ int vcpuid;
+};
+
+struct vm_cpuset {
+ int which;
+ int cpusetsize;
+ cpuset_t *cpus;
+};
+#define VM_ACTIVE_CPUS 0
+#define VM_SUSPENDED_CPUS 1
+#define VM_DEBUG_CPUS 2
+
+struct vm_aplic_descr {
+ uint64_t mem_start;
+ uint64_t mem_size;
+};
+
+struct vm_irq {
+ uint32_t irq;
+};
+
+struct vm_cpu_topology {
+ uint16_t sockets;
+ uint16_t cores;
+ uint16_t threads;
+ uint16_t maxcpus;
+};
+
+enum {
+ /* general routines */
+ IOCNUM_ABIVERS = 0,
+ IOCNUM_RUN = 1,
+ IOCNUM_SET_CAPABILITY = 2,
+ IOCNUM_GET_CAPABILITY = 3,
+ IOCNUM_SUSPEND = 4,
+ IOCNUM_REINIT = 5,
+
+ /* memory apis */
+ IOCNUM_GET_GPA_PMAP = 12,
+ IOCNUM_GLA2GPA_NOFAULT = 13,
+ IOCNUM_ALLOC_MEMSEG = 14,
+ IOCNUM_GET_MEMSEG = 15,
+ IOCNUM_MMAP_MEMSEG = 16,
+ IOCNUM_MMAP_GETNEXT = 17,
+ IOCNUM_MUNMAP_MEMSEG = 18,
+
+ /* register/state accessors */
+ IOCNUM_SET_REGISTER = 20,
+ IOCNUM_GET_REGISTER = 21,
+ IOCNUM_SET_REGISTER_SET = 24,
+ IOCNUM_GET_REGISTER_SET = 25,
+
+ /* statistics */
+ IOCNUM_VM_STATS = 50,
+ IOCNUM_VM_STAT_DESC = 51,
+
+ /* CPU Topology */
+ IOCNUM_SET_TOPOLOGY = 63,
+ IOCNUM_GET_TOPOLOGY = 64,
+
+ /* interrupt injection */
+ IOCNUM_ASSERT_IRQ = 80,
+ IOCNUM_DEASSERT_IRQ = 81,
+ IOCNUM_RAISE_MSI = 82,
+ IOCNUM_INJECT_EXCEPTION = 83,
+
+ /* vm_cpuset */
+ IOCNUM_ACTIVATE_CPU = 90,
+ IOCNUM_GET_CPUSET = 91,
+ IOCNUM_SUSPEND_CPU = 92,
+ IOCNUM_RESUME_CPU = 93,
+
+ /* vm_attach_aplic */
+ IOCNUM_ATTACH_APLIC = 110,
+};
+
+#define VM_RUN \
+ _IOWR('v', IOCNUM_RUN, struct vm_run)
+#define VM_SUSPEND \
+ _IOW('v', IOCNUM_SUSPEND, struct vm_suspend)
+#define VM_REINIT \
+ _IO('v', IOCNUM_REINIT)
+#define VM_ALLOC_MEMSEG \
+ _IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg)
+#define VM_GET_MEMSEG \
+ _IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg)
+#define VM_MMAP_MEMSEG \
+ _IOW('v', IOCNUM_MMAP_MEMSEG, struct vm_memmap)
+#define VM_MMAP_GETNEXT \
+ _IOWR('v', IOCNUM_MMAP_GETNEXT, struct vm_memmap)
+#define VM_MUNMAP_MEMSEG \
+ _IOW('v', IOCNUM_MUNMAP_MEMSEG, struct vm_munmap)
+#define VM_SET_REGISTER \
+ _IOW('v', IOCNUM_SET_REGISTER, struct vm_register)
+#define VM_GET_REGISTER \
+ _IOWR('v', IOCNUM_GET_REGISTER, struct vm_register)
+#define VM_SET_REGISTER_SET \
+ _IOW('v', IOCNUM_SET_REGISTER_SET, struct vm_register_set)
+#define VM_GET_REGISTER_SET \
+ _IOWR('v', IOCNUM_GET_REGISTER_SET, struct vm_register_set)
+#define VM_SET_CAPABILITY \
+ _IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability)
+#define VM_GET_CAPABILITY \
+ _IOWR('v', IOCNUM_GET_CAPABILITY, struct vm_capability)
+#define VM_STATS \
+ _IOWR('v', IOCNUM_VM_STATS, struct vm_stats)
+#define VM_STAT_DESC \
+ _IOWR('v', IOCNUM_VM_STAT_DESC, struct vm_stat_desc)
+#define VM_ASSERT_IRQ \
+ _IOW('v', IOCNUM_ASSERT_IRQ, struct vm_irq)
+#define VM_DEASSERT_IRQ \
+ _IOW('v', IOCNUM_DEASSERT_IRQ, struct vm_irq)
+#define VM_RAISE_MSI \
+ _IOW('v', IOCNUM_RAISE_MSI, struct vm_msi)
+#define VM_INJECT_EXCEPTION \
+ _IOW('v', IOCNUM_INJECT_EXCEPTION, struct vm_exception)
+#define VM_SET_TOPOLOGY \
+ _IOW('v', IOCNUM_SET_TOPOLOGY, struct vm_cpu_topology)
+#define VM_GET_TOPOLOGY \
+ _IOR('v', IOCNUM_GET_TOPOLOGY, struct vm_cpu_topology)
+#define VM_GLA2GPA_NOFAULT \
+ _IOWR('v', IOCNUM_GLA2GPA_NOFAULT, struct vm_gla2gpa)
+#define VM_ACTIVATE_CPU \
+ _IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
+#define VM_GET_CPUS \
+ _IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
+#define VM_SUSPEND_CPU \
+ _IOW('v', IOCNUM_SUSPEND_CPU, struct vm_activate_cpu)
+#define VM_RESUME_CPU \
+ _IOW('v', IOCNUM_RESUME_CPU, struct vm_activate_cpu)
+#define VM_ATTACH_APLIC \
+ _IOW('v', IOCNUM_ATTACH_APLIC, struct vm_aplic_descr)
+#endif
diff --git a/sys/riscv/include/vmm_instruction_emul.h b/sys/riscv/include/vmm_instruction_emul.h
new file mode 100644
index 000000000000..bee63d2f86ba
--- /dev/null
+++ b/sys/riscv/include/vmm_instruction_emul.h
@@ -0,0 +1,85 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_INSTRUCTION_EMUL_H_
+#define _VMM_INSTRUCTION_EMUL_H_
+
+/*
+ * Callback functions to read and write memory regions.
+ */
+typedef int (*mem_region_read_t)(struct vcpu *vcpu, uint64_t gpa,
+ uint64_t *rval, int rsize, void *arg);
+typedef int (*mem_region_write_t)(struct vcpu *vcpu, uint64_t gpa,
+ uint64_t wval, int wsize, void *arg);
+
+/*
+ * Callback functions to read and write registers.
+ */
+typedef int (*reg_read_t)(struct vcpu *vcpu, uint64_t *rval, void *arg);
+typedef int (*reg_write_t)(struct vcpu *vcpu, uint64_t wval, void *arg);
+
+/*
+ * Emulate the decoded 'vie' instruction when it contains a memory operation.
+ *
+ * The callbacks 'mrr' and 'mrw' emulate reads and writes to the memory region
+ * containing 'gpa'. 'mrarg' is an opaque argument that is passed into the
+ * callback functions.
+ *
+ * 'void *vm' should be 'struct vm *' when called from kernel context and
+ * 'struct vmctx *' when called from user context.
+ *
+ */
+int vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
+ struct vm_guest_paging *paging, mem_region_read_t mrr,
+ mem_region_write_t mrw, void *mrarg);
+
+/*
+ * Emulate the decoded 'vre' instruction when it contains a register access.
+ *
+ * The callbacks 'regread' and 'regwrite' emulate reads and writes to the
+ * register from 'vie'. 'regarg' is an opaque argument that is passed into the
+ * callback functions.
+ *
+ * 'void *vm' should be 'struct vm *' when called from kernel context and
+ * 'struct vmctx *' when called from user context.
+ *
+ */
+int vmm_emulate_register(struct vcpu *vcpu, struct vre *vre, reg_read_t regread,
+ reg_write_t regwrite, void *regarg);
+
+#ifdef _KERNEL
+void vm_register_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask,
+ reg_read_t reg_read, reg_write_t reg_write, void *arg);
+void vm_deregister_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask);
+
+void vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,
+ mem_region_read_t mmio_read, mem_region_write_t mmio_write);
+void vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size);
+#endif
+
+#endif /* _VMM_INSTRUCTION_EMUL_H_ */
diff --git a/sys/riscv/include/vmm_snapshot.h b/sys/riscv/include/vmm_snapshot.h
new file mode 100644
index 000000000000..da23dbe43a4f
--- /dev/null
+++ b/sys/riscv/include/vmm_snapshot.h
@@ -0,0 +1 @@
+/* $FreeBSD$ */
diff --git a/sys/riscv/riscv/genassym.c b/sys/riscv/riscv/genassym.c
index 637510db242e..74b70858edab 100644
--- a/sys/riscv/riscv/genassym.c
+++ b/sys/riscv/riscv/genassym.c
@@ -55,6 +55,8 @@
#include <machine/machdep.h>
#include <machine/vmparam.h>
+#include <riscv/vmm/riscv.h>
+
ASSYM(KERNBASE, KERNBASE);
ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
ASSYM(VM_MAX_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS);
@@ -98,6 +100,38 @@ ASSYM(TF_STVAL, offsetof(struct trapframe, tf_stval));
ASSYM(TF_SCAUSE, offsetof(struct trapframe, tf_scause));
ASSYM(TF_SSTATUS, offsetof(struct trapframe, tf_sstatus));
+ASSYM(HYP_H_RA, offsetof(struct hypctx, host_regs.hyp_ra));
+ASSYM(HYP_H_SP, offsetof(struct hypctx, host_regs.hyp_sp));
+ASSYM(HYP_H_GP, offsetof(struct hypctx, host_regs.hyp_gp));
+ASSYM(HYP_H_TP, offsetof(struct hypctx, host_regs.hyp_tp));
+ASSYM(HYP_H_T, offsetof(struct hypctx, host_regs.hyp_t));
+ASSYM(HYP_H_S, offsetof(struct hypctx, host_regs.hyp_s));
+ASSYM(HYP_H_A, offsetof(struct hypctx, host_regs.hyp_a));
+ASSYM(HYP_H_SEPC, offsetof(struct hypctx, host_regs.hyp_sepc));
+ASSYM(HYP_H_SSTATUS, offsetof(struct hypctx, host_regs.hyp_sstatus));
+ASSYM(HYP_H_HSTATUS, offsetof(struct hypctx, host_regs.hyp_hstatus));
+ASSYM(HYP_H_SSCRATCH, offsetof(struct hypctx, host_sscratch));
+ASSYM(HYP_H_STVEC, offsetof(struct hypctx, host_stvec));
+ASSYM(HYP_H_SCOUNTEREN, offsetof(struct hypctx, host_scounteren));
+
+ASSYM(HYP_G_RA, offsetof(struct hypctx, guest_regs.hyp_ra));
+ASSYM(HYP_G_SP, offsetof(struct hypctx, guest_regs.hyp_sp));
+ASSYM(HYP_G_GP, offsetof(struct hypctx, guest_regs.hyp_gp));
+ASSYM(HYP_G_TP, offsetof(struct hypctx, guest_regs.hyp_tp));
+ASSYM(HYP_G_T, offsetof(struct hypctx, guest_regs.hyp_t));
+ASSYM(HYP_G_S, offsetof(struct hypctx, guest_regs.hyp_s));
+ASSYM(HYP_G_A, offsetof(struct hypctx, guest_regs.hyp_a));
+ASSYM(HYP_G_SEPC, offsetof(struct hypctx, guest_regs.hyp_sepc));
+ASSYM(HYP_G_SSTATUS, offsetof(struct hypctx, guest_regs.hyp_sstatus));
+ASSYM(HYP_G_HSTATUS, offsetof(struct hypctx, guest_regs.hyp_hstatus));
+ASSYM(HYP_G_SCOUNTEREN, offsetof(struct hypctx, guest_scounteren));
+
+ASSYM(HYP_TRAP_SEPC, offsetof(struct hyptrap, sepc));
+ASSYM(HYP_TRAP_SCAUSE, offsetof(struct hyptrap, scause));
+ASSYM(HYP_TRAP_STVAL, offsetof(struct hyptrap, stval));
+ASSYM(HYP_TRAP_HTVAL, offsetof(struct hyptrap, htval));
+ASSYM(HYP_TRAP_HTINST, offsetof(struct hyptrap, htinst));
+
ASSYM(RISCV_BOOTPARAMS_SIZE, sizeof(struct riscv_bootparams));
ASSYM(RISCV_BOOTPARAMS_KERN_PHYS, offsetof(struct riscv_bootparams, kern_phys));
ASSYM(RISCV_BOOTPARAMS_KERN_STACK, offsetof(struct riscv_bootparams,
diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c
index 54eb302982f1..7823830c3136 100644
--- a/sys/riscv/riscv/identcpu.c
+++ b/sys/riscv/riscv/identcpu.c
@@ -72,6 +72,7 @@ register_t mimpid; /* The implementation ID */
u_int mmu_caps;
/* Supervisor-mode extension support. */
+bool has_hyp;
*** 4006 LINES SKIPPED ***