git: 143371df7657 - stable/14 - x86: Introduce APIC ID limit by default on AMD hardware

The branch stable/14 has been updated by emaste:

URL: https://cgit.FreeBSD.org/src/commit/?id=143371df7657540228f12e01ba3a07eab41c3ab9

commit 143371df7657540228f12e01ba3a07eab41c3ab9
Author:     Ed Maste <emaste@FreeBSD.org>
AuthorDate: 2023-08-18 03:39:08 +0000
Commit:     Ed Maste <emaste@FreeBSD.org>
CommitDate: 2023-09-04 23:08:34 +0000

    x86: Introduce APIC ID limit by default on AMD hardware
    
    Lack of an AMD IOMMU driver means we cannot successfully route
    interrupts to APIC IDs 255 and over.  Do not add the corresponding CPUs
    to the per-domain lists of CPUs to which interrupts can be assigned.
    
    This change should be reverted (or, at least the APIC ID limit) once we
    have an AMD IOMMU / interrupt remapping driver.
    
    See also commits fa5f94140a83 ("msi: handle error from BUS_REMAP_INTR in
    msi_assign_cpu") and 4258eb5a0d97 ("x86: handle domains with no CPUs
    usable for intr delivery.").
    
    Reviewed by:    markj, jhb
    Tested by:      cperciva (earlier version)
    Sponsored by:   The FreeBSD Foundation
    Differential Revision: https://reviews.freebsd.org/D41618
    
    (cherry picked from commit 0b029e9e85943d565c72aa58353538aeac68aa36)
    
    Approved by:    re (kib)
---
 sys/x86/x86/mp_x86.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c
index 76a1d14f371a..ddcb54b63d88 100644
--- a/sys/x86/x86/mp_x86.c
+++ b/sys/x86/x86/mp_x86.c
@@ -153,6 +153,11 @@ SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_intr_allowed, CTLFLAG_RDTUN,
 	&hyperthreading_intr_allowed, 0,
 	"Allow interrupts on HTT logical CPUs");
 
+static int	intr_apic_id_limit = -1;
+SYSCTL_INT(_machdep, OID_AUTO, intr_apic_id_limit, CTLFLAG_RDTUN,
+	&intr_apic_id_limit, 0,
+	"Maximum permitted APIC ID for interrupt delivery (-1 is unlimited)");
+
 static struct topo_node topo_root;
 
 static int pkg_id_shift;
@@ -258,6 +263,22 @@ topo_probe_amd(void)
 	if ((amd_feature2 & AMDID2_CMP) == 0)
 		return;
 
+	/*
+	 * XXX Lack of an AMD IOMMU driver prevents use of APIC IDs above
+	 * xAPIC_MAX_APIC_ID.  This is a workaround so we boot and function on
+	 * AMD systems with high thread counts, albeit with reduced interrupt
+	 * performance.
+	 *
+	 * We should really set the limit to xAPIC_MAX_APIC_ID by default, and
+	 * have the IOMMU driver increase it.  That way if a driver is present
+	 * but disabled, or is otherwise not able to route the interrupts, the
+	 * system can fall back to a functional state.  That will require a more
+	 * substantial change though, including having the IOMMU initialize
+	 * earlier.
+	 */
+	if (intr_apic_id_limit == -1)
+		intr_apic_id_limit = xAPIC_MAX_APIC_ID;
+
 	/* For families 10h and newer. */
 	pkg_id_shift = (cpu_procinfo2 & AMDID_COREID_SIZE) >>
 	    AMDID_COREID_SIZE_SHIFT;
@@ -1170,6 +1191,8 @@ set_interrupt_apic_ids(void)
 			continue;
 		if (cpu_info[apic_id].cpu_disabled)
 			continue;
+		if (intr_apic_id_limit >= 0 && apic_id > intr_apic_id_limit)
+			continue;
 
 		/* Don't let hyperthreads service interrupts. */
 		if (cpu_info[apic_id].cpu_hyperthread &&