git: 1bd361eea28a - main - bhyve: allocate guest memory for graphics stolen memory

The branch main has been updated by corvink:

URL: https://cgit.FreeBSD.org/src/commit/?id=1bd361eea28ae74b42d49bfc96b109a7eae9f3d3

commit 1bd361eea28ae74b42d49bfc96b109a7eae9f3d3
Author:     Corvin Köhne <corvink@FreeBSD.org>
AuthorDate: 2023-05-11 09:10:07 +0000
Commit:     Corvin Köhne <corvink@FreeBSD.org>
CommitDate: 2023-06-16 05:54:02 +0000

    bhyve: allocate guest memory for graphics stolen memory
    
    The graphics stolen memory is only GPU accessible. So, we don't have to
    copy any data to it as the guest will be unable to access it anyway. We
    just have to allocate and reserve some memory. That's done by adding an
    E820 entry for the graphics stolen memory. The guest firmware will pick
    up the E820 and reserve this range.
    
    Note that we try to reuse the host address as Intel states that newer
    Tiger Lake platforms need this [1].
    
    [1]
    https://github.com/projectacrn/acrn-hypervisor/blob/e28d6fbfdfd556ff1bc3ff330e41d4ddbaa0f897/devicemodel/hw/pci/passthrough.c#L626-L629
    
    Reviewed by:            markj
    MFC after:              1 week
    Sponsored by:           Beckhoff Automation GmbH & Co. KG
    Differential Revision:  https://reviews.freebsd.org/D40059
---
 usr.sbin/bhyve/pci_gvt-d.c | 64 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 2 deletions(-)

diff --git a/usr.sbin/bhyve/pci_gvt-d.c b/usr.sbin/bhyve/pci_gvt-d.c
index 0be740921c92..9c5b0906ec29 100644
--- a/usr.sbin/bhyve/pci_gvt-d.c
+++ b/usr.sbin/bhyve/pci_gvt-d.c
@@ -8,17 +8,24 @@
 #include <sys/types.h>
 #include <sys/sysctl.h>
 
-#include <err.h>
-
 #include <dev/pci/pcireg.h>
 
+#include <err.h>
 #include <errno.h>
 
+#include "e820.h"
 #include "pci_gvt-d-opregion.h"
 #include "pci_passthru.h"
 
+#define KB (1024UL)
+#define MB (1024 * KB)
+#define GB (1024 * MB)
+
 #define PCI_VENDOR_INTEL 0x8086
 
+#define PCIM_BDSM_GSM_ALIGNMENT \
+	0x00100000 /* Graphics Stolen Memory is 1 MB aligned */
+
 #define GVT_D_MAP_GSM 0
 
 static int
@@ -41,6 +48,27 @@ gvt_d_probe(struct pci_devinst *const pi)
 	return (0);
 }
 
+static vm_paddr_t
+gvt_d_alloc_mmio_memory(const vm_paddr_t host_address, const vm_paddr_t length,
+    const vm_paddr_t alignment, const enum e820_memory_type type)
+{
+	vm_paddr_t address;
+
+	/* Try to reuse host address. */
+	address = e820_alloc(host_address, length, E820_ALIGNMENT_NONE, type,
+	    E820_ALLOCATE_SPECIFIC);
+	if (address != 0) {
+		return (address);
+	}
+
+	/*
+	 * We're not able to reuse the host address. Fall back to the highest usable
+	 * address below 4 GB.
+	 */
+	return (
+	    e820_alloc(4 * GB, length, alignment, type, E820_ALLOCATE_HIGHEST));
+}
+
 /*
  * Note that the graphics stolen memory is somehow confusing. On the one hand
  * the Intel Open Source HD Graphics Programmers' Reference Manual states that
@@ -95,6 +123,38 @@ gvt_d_setup_gsm(struct pci_devinst *const pi)
 		return (-1);
 	}
 	gsm->hva = NULL; /* unused */
+	gsm->gva = NULL; /* unused */
+	gsm->gpa = gvt_d_alloc_mmio_memory(gsm->hpa, gsm->len,
+	    PCIM_BDSM_GSM_ALIGNMENT, E820_TYPE_RESERVED);
+	if (gsm->gpa == 0) {
+		warnx(
+		    "%s: Unable to add Graphics Stolen Memory to E820 table (hpa 0x%lx len 0x%lx)",
+		    __func__, gsm->hpa, gsm->len);
+		e820_dump_table();
+		return (-1);
+	}
+	if (gsm->gpa != gsm->hpa) {
+		/*
+		 * ACRN source code implies that graphics driver for newer Intel
+		 * platforms like Tiger Lake will read the Graphics Stolen Memory
+		 * address from an MMIO register. We have three options to solve this
+		 * issue:
+		 *    1. Patch the value in the MMIO register
+		 *       This could have unintended side effects. Without any
+		 *       documentation how this register is used by the GPU, don't do
+		 *       it.
+		 *    2. Trap the MMIO register
+		 *       It's not possible to trap a single MMIO register. We need to
+		 *       trap a whole page. Trapping a bunch of MMIO register could
+		 *       degrade the performance noticeably. We have to test it.
+		 *    3. Use an 1:1 host to guest mapping
+		 *       Maybe not always possible. As far as we know, no supported
+		 *       platform requires a 1:1 mapping. For that reason, just log a
+		 *       warning.
+		 */
+		warnx(
+		    "Warning: Unable to reuse host address of Graphics Stolen Memory. GPU passthrough might not work properly.");
+	}
 
 	return (0);
 }