svn commit: r363493 - in head/sys/x86: include x86

[Alexander Motin]

Author: mav
Date: Fri Jul 24 20:44:50 2020
New Revision: 363493
URL: https://svnweb.freebsd.org/changeset/base/363493

Log:
  Use APIC_IPI_DEST_OTHERS for bitmapped IPIs too.
  
  It should save bunch of LAPIC register accesses.
  
  MFC after:	2 weeks

Modified:
  head/sys/x86/include/x86_smp.h
  head/sys/x86/x86/mp_x86.c

Modified: head/sys/x86/include/x86_smp.h
==============================================================================
--- head/sys/x86/include/x86_smp.h	Fri Jul 24 20:10:27 2020	(r363492)
+++ head/sys/x86/include/x86_smp.h	Fri Jul 24 20:44:50 2020	(r363493)
@@ -107,6 +107,5 @@ void	smp_masked_invltlb(cpuset_t mask, struct pmap *pm
 	    smp_invl_cb_t curcpu_cb);
 void	mem_range_AP_init(void);
 void	topo_probe(void);
-void	ipi_send_cpu(int cpu, u_int ipi);
 
 #endif

Modified: head/sys/x86/x86/mp_x86.c
==============================================================================
--- head/sys/x86/x86/mp_x86.c	Fri Jul 24 20:10:27 2020	(r363492)
+++ head/sys/x86/x86/mp_x86.c	Fri Jul 24 20:44:50 2020	(r363493)
@@ -1233,32 +1233,39 @@ ipi_startup(int apic_id, int vector)
 	DELAY(200);		/* wait ~200uS */
 }
 
+static bool
+ipi_bitmap_set(int cpu, u_int ipi)
+{
+	u_int bitmap, old, new;
+	u_int *cpu_bitmap;
+
+	bitmap = 1 << ipi;
+	cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
+	old = *cpu_bitmap;
+	for (;;) {
+		if ((old & bitmap) != 0)
+			break;
+		new = old | bitmap;
+		if (atomic_fcmpset_int(cpu_bitmap, &old, new))
+			break;
+	}
+	return (old != 0);
+}
+
 /*
  * Send an IPI to specified CPU handling the bitmap logic.
  */
-void
+static void
 ipi_send_cpu(int cpu, u_int ipi)
 {
-	u_int bitmap, old, new;
-	u_int *cpu_bitmap;
 
 	KASSERT((u_int)cpu < MAXCPU && cpu_apic_ids[cpu] != -1,
 	    ("IPI to non-existent CPU %d", cpu));
 
 	if (IPI_IS_BITMAPED(ipi)) {
-		bitmap = 1 << ipi;
-		ipi = IPI_BITMAP_VECTOR;
-		cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
-		old = *cpu_bitmap;
-		for (;;) {
-			if ((old & bitmap) == bitmap)
-				break;
-			new = old | bitmap;
-			if (atomic_fcmpset_int(cpu_bitmap, &old, new))
-				break;
-		}
-		if (old)
+		if (ipi_bitmap_set(cpu, ipi))
 			return;
+		ipi = IPI_BITMAP_VECTOR;
 	}
 	lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
 }
@@ -1366,23 +1373,28 @@ void
 ipi_all_but_self(u_int ipi)
 {
 	cpuset_t other_cpus;
+	int cpu, c;
 
-	other_cpus = all_cpus;
-	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
-	if (IPI_IS_BITMAPED(ipi)) {
-		ipi_selected(other_cpus, ipi);
-		return;
-	}
-
 	/*
 	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
 	 * of help in order to understand what is the source.
 	 * Set the mask of receiving CPUs for this purpose.
 	 */
-	if (ipi == IPI_STOP_HARD)
+	if (ipi == IPI_STOP_HARD) {
+		other_cpus = all_cpus;
+		CPU_CLR(PCPU_GET(cpuid), &other_cpus);
 		CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus);
+	}
 
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+	if (IPI_IS_BITMAPED(ipi)) {
+		cpu = PCPU_GET(cpuid);
+		CPU_FOREACH(c) {
+			if (c != cpu)
+				ipi_bitmap_set(c, ipi);
+		}
+		ipi = IPI_BITMAP_VECTOR;
+	}
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
 }