git: 65b133e5d292 - main - x86: allow to have more than one kind of IOMMU
Date: Wed, 04 Sep 2024 21:50:40 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=65b133e5d292686fe20f11dd39b53812226a8684
commit 65b133e5d292686fe20f11dd39b53812226a8684
Author: Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2024-05-29 14:38:14 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2024-09-04 21:50:18 +0000
x86: allow to have more than one kind of IOMMU
Sponsored by: Advanced Micro Devices (AMD)
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
---
sys/x86/iommu/intel_ctx.c | 22 +++++------
sys/x86/iommu/intel_dmar.h | 18 ++++++++-
sys/x86/iommu/intel_drv.c | 30 ++++++++++++--
sys/x86/iommu/intel_intrmap.c | 10 ++---
sys/x86/iommu/iommu_utils.c | 91 ++++++++++++++++++++++++++++++++++++++++++-
sys/x86/iommu/x86_iommu.h | 25 +++++++++++-
6 files changed, 173 insertions(+), 23 deletions(-)
diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
index baee0109e5a9..a3ff35dc527e 100644
--- a/sys/x86/iommu/intel_ctx.c
+++ b/sys/x86/iommu/intel_ctx.c
@@ -75,6 +75,9 @@ static void dmar_unref_domain_locked(struct dmar_unit *dmar,
struct dmar_domain *domain);
static void dmar_domain_destroy(struct dmar_domain *domain);
+static void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
+static void dmar_free_ctx(struct dmar_ctx *ctx);
+
static void
dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
{
@@ -745,7 +748,7 @@ dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain)
dmar_domain_destroy(domain);
}
-void
+static void
dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
{
struct sf_buf *sf;
@@ -819,7 +822,7 @@ dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
TD_PINNED_ASSERT;
}
-void
+static void
dmar_free_ctx(struct dmar_ctx *ctx)
{
struct dmar_unit *dmar;
@@ -867,7 +870,7 @@ dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)
* the entry's dmamap_link field.
*/
void
-iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
+dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free,
bool cansleep)
{
struct dmar_domain *domain;
@@ -909,7 +912,7 @@ dmar_domain_unload_emit_wait(struct dmar_domain *domain,
}
void
-iommu_domain_unload(struct iommu_domain *iodom,
+dmar_domain_unload(struct iommu_domain *iodom,
struct iommu_map_entries_tailq *entries, bool cansleep)
{
struct dmar_domain *domain;
@@ -947,37 +950,34 @@ iommu_domain_unload(struct iommu_domain *iodom,
}
struct iommu_ctx *
-iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
+dmar_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
bool id_mapped, bool rmrr_init)
{
struct dmar_unit *dmar;
struct dmar_ctx *ret;
dmar = IOMMU2DMAR(iommu);
-
ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init);
-
return (CTX2IOCTX(ret));
}
void
-iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context)
+dmar_free_ctx_locked_method(struct iommu_unit *iommu,
+ struct iommu_ctx *context)
{
struct dmar_unit *dmar;
struct dmar_ctx *ctx;
dmar = IOMMU2DMAR(iommu);
ctx = IOCTX2CTX(context);
-
dmar_free_ctx_locked(dmar, ctx);
}
void
-iommu_free_ctx(struct iommu_ctx *context)
+dmar_free_ctx_method(struct iommu_ctx *context)
{
struct dmar_ctx *ctx;
ctx = IOCTX2CTX(context);
-
dmar_free_ctx(ctx);
}
diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h
index 4ae005238074..0ede955e12b9 100644
--- a/sys/x86/iommu/intel_dmar.h
+++ b/sys/x86/iommu/intel_dmar.h
@@ -293,10 +293,17 @@ struct dmar_ctx *dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid,
int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
bool id_mapped, bool rmrr_init);
int dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx);
-void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
-void dmar_free_ctx(struct dmar_ctx *ctx);
+void dmar_free_ctx_locked_method(struct iommu_unit *dmar,
+ struct iommu_ctx *ctx);
+void dmar_free_ctx_method(struct iommu_ctx *ctx);
struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid);
+struct iommu_ctx *dmar_get_ctx(struct iommu_unit *iommu, device_t dev,
+ uint16_t rid, bool id_mapped, bool rmrr_init);
void dmar_domain_free_entry(struct iommu_map_entry *entry, bool free);
+void dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free,
+ bool cansleep);
+void dmar_domain_unload(struct iommu_domain *iodom,
+ struct iommu_map_entries_tailq *entries, bool cansleep);
void dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain,
int dev_busno, const void *dev_path, int dev_path_len,
@@ -308,6 +315,13 @@ void dmar_quirks_pre_use(struct iommu_unit *dmar);
int dmar_init_irt(struct dmar_unit *unit);
void dmar_fini_irt(struct dmar_unit *unit);
+int dmar_alloc_msi_intr(device_t src, u_int *cookies, u_int count);
+int dmar_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
+ uint64_t *addr, uint32_t *data);
+int dmar_unmap_msi_intr(device_t src, u_int cookie);
+int dmar_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
+ bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo);
+int dmar_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie);
extern int haw;
extern int dmar_batch_coalesce;
diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c
index 636534173715..79350358cced 100644
--- a/sys/x86/iommu/intel_drv.c
+++ b/sys/x86/iommu/intel_drv.c
@@ -64,6 +64,8 @@
#include <dev/pci/pcivar.h>
#include <machine/bus.h>
#include <machine/pci_cfgreg.h>
+#include <machine/md_var.h>
+#include <machine/cputypes.h>
#include <x86/include/busdma_impl.h>
#include <dev/iommu/busdma_iommu.h>
#include <x86/iommu/intel_reg.h>
@@ -1357,12 +1359,34 @@ DB_SHOW_ALL_COMMAND(dmars, db_show_all_dmars)
}
#endif
-struct iommu_unit *
-iommu_find(device_t dev, bool verbose)
+static struct iommu_unit *
+dmar_find_method(device_t dev, bool verbose)
{
struct dmar_unit *dmar;
dmar = dmar_find(dev, verbose);
-
return (&dmar->iommu);
}
+
+static struct x86_iommu dmar_x86_iommu = {
+ .domain_unload_entry = dmar_domain_unload_entry,
+ .domain_unload = dmar_domain_unload,
+ .get_ctx = dmar_get_ctx,
+ .free_ctx_locked = dmar_free_ctx_locked_method,
+ .free_ctx = dmar_free_ctx_method,
+ .find = dmar_find_method,
+ .alloc_msi_intr = dmar_alloc_msi_intr,
+ .map_msi_intr = dmar_map_msi_intr,
+ .unmap_msi_intr = dmar_unmap_msi_intr,
+ .map_ioapic_intr = dmar_map_ioapic_intr,
+ .unmap_ioapic_intr = dmar_unmap_ioapic_intr,
+};
+
+static void
+x86_iommu_set_intel(void *arg __unused)
+{
+ if (cpu_vendor_id == CPU_VENDOR_INTEL)
+ set_x86_iommu(&dmar_x86_iommu);
+}
+
+SYSINIT(x86_iommu, SI_SUB_TUNABLES, SI_ORDER_ANY, x86_iommu_set_intel, NULL);
diff --git a/sys/x86/iommu/intel_intrmap.c b/sys/x86/iommu/intel_intrmap.c
index 22fd51bc8be9..a6979a9d2501 100644
--- a/sys/x86/iommu/intel_intrmap.c
+++ b/sys/x86/iommu/intel_intrmap.c
@@ -65,7 +65,7 @@ static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);
int
-iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
+dmar_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
{
struct dmar_unit *unit;
vmem_addr_t vmem_res;
@@ -93,7 +93,7 @@ iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
}
int
-iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
+dmar_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
uint64_t *addr, uint32_t *data)
{
struct dmar_unit *unit;
@@ -139,7 +139,7 @@ iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
}
int
-iommu_unmap_msi_intr(device_t src, u_int cookie)
+dmar_unmap_msi_intr(device_t src, u_int cookie)
{
struct dmar_unit *unit;
@@ -150,7 +150,7 @@ iommu_unmap_msi_intr(device_t src, u_int cookie)
}
int
-iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
+dmar_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo)
{
struct dmar_unit *unit;
@@ -213,7 +213,7 @@ iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
}
int
-iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
+dmar_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
{
struct dmar_unit *unit;
u_int idx;
diff --git a/sys/x86/iommu/iommu_utils.c b/sys/x86/iommu/iommu_utils.c
index ffea1cc1a190..ea2c0358e072 100644
--- a/sys/x86/iommu/iommu_utils.c
+++ b/sys/x86/iommu/iommu_utils.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright (c) 2013, 2014 The FreeBSD Foundation
+ * Copyright (c) 2013, 2014, 2024 The FreeBSD Foundation
*
* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.
@@ -50,6 +50,7 @@
#include <dev/iommu/busdma_iommu.h>
#include <dev/iommu/iommu.h>
#include <x86/iommu/x86_iommu.h>
+#include <x86/iommu/iommu_intrmap.h>
vm_page_t
iommu_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
@@ -162,3 +163,91 @@ SYSCTL_NODE(_hw_iommu, OID_AUTO, dmar, CTLFLAG_RD | CTLFLAG_MPSAFE,
SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD,
&iommu_tbl_pagecnt, 0,
"Count of pages used for DMAR pagetables");
+
+static struct x86_iommu *x86_iommu;
+
+void
+set_x86_iommu(struct x86_iommu *x)
+{
+ MPASS(x86_iommu == NULL);
+ x86_iommu = x;
+}
+
+struct x86_iommu *
+get_x86_iommu(void)
+{
+ return (x86_iommu);
+}
+
+void
+iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
+ bool cansleep)
+{
+ x86_iommu->domain_unload_entry(entry, free, cansleep);
+}
+
+void
+iommu_domain_unload(struct iommu_domain *iodom,
+ struct iommu_map_entries_tailq *entries, bool cansleep)
+{
+ x86_iommu->domain_unload(iodom, entries, cansleep);
+}
+
+struct iommu_ctx *
+iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
+ bool id_mapped, bool rmrr_init)
+{
+ return (x86_iommu->get_ctx(iommu, dev, rid, id_mapped, rmrr_init));
+}
+
+void
+iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context)
+{
+ x86_iommu->free_ctx_locked(iommu, context);
+}
+
+void
+iommu_free_ctx(struct iommu_ctx *context)
+{
+ x86_iommu->free_ctx(context);
+}
+
+struct iommu_unit *
+iommu_find(device_t dev, bool verbose)
+{
+ return (x86_iommu->find(dev, verbose));
+}
+
+int
+iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
+{
+ return (x86_iommu->alloc_msi_intr(src, cookies, count));
+}
+
+int
+iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
+ uint64_t *addr, uint32_t *data)
+{
+ return (x86_iommu->map_msi_intr(src, cpu, vector, cookie,
+ addr, data));
+}
+
+int
+iommu_unmap_msi_intr(device_t src, u_int cookie)
+{
+ return (x86_iommu->unmap_msi_intr(src, cookie));
+}
+
+int
+iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
+ bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo)
+{
+ return (x86_iommu->map_ioapic_intr(ioapic_id, cpu, vector, edge,
+ activehi, irq, cookie, hi, lo));
+}
+
+int
+iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
+{
+ return (x86_iommu->unmap_ioapic_intr(ioapic_id, cookie));
+}
diff --git a/sys/x86/iommu/x86_iommu.h b/sys/x86/iommu/x86_iommu.h
index 3789586f1eaf..8c908964acd0 100644
--- a/sys/x86/iommu/x86_iommu.h
+++ b/sys/x86/iommu/x86_iommu.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright (c) 2013-2015 The FreeBSD Foundation
+ * Copyright (c) 2013-2015, 2024 The FreeBSD Foundation
*
* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.
@@ -59,4 +59,27 @@ extern int iommu_tbl_pagecnt;
SYSCTL_DECL(_hw_iommu);
SYSCTL_DECL(_hw_iommu_dmar);
+struct x86_iommu {
+ void (*domain_unload_entry)(struct iommu_map_entry *entry, bool free,
+ bool cansleep);
+ void (*domain_unload)(struct iommu_domain *iodom,
+ struct iommu_map_entries_tailq *entries, bool cansleep);
+ struct iommu_ctx *(*get_ctx)(struct iommu_unit *iommu,
+ device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init);
+ void (*free_ctx_locked)(struct iommu_unit *iommu,
+ struct iommu_ctx *context);
+ void (*free_ctx)(struct iommu_ctx *context);
+ struct iommu_unit *(*find)(device_t dev, bool verbose);
+ int (*alloc_msi_intr)(device_t src, u_int *cookies, u_int count);
+ int (*map_msi_intr)(device_t src, u_int cpu, u_int vector,
+ u_int cookie, uint64_t *addr, uint32_t *data);
+ int (*unmap_msi_intr)(device_t src, u_int cookie);
+ int (*map_ioapic_intr)(u_int ioapic_id, u_int cpu, u_int vector,
+ bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
+ uint32_t *lo);
+ int (*unmap_ioapic_intr)(u_int ioapic_id, u_int *cookie);
+};
+void set_x86_iommu(struct x86_iommu *);
+struct x86_iommu *get_x86_iommu(void);
+
#endif