svn commit: r221499 - in projects/largeSMP: gnu/usr.bin/gdb/kgdb
lib/libmemstat sys/amd64/acpica sys/amd64/amd64
sys/amd64/include sys/arm/arm sys/arm/include
sys/cddl/dev/cyclic/i386 sys/cddl/dev/...
Attilio Rao
attilio at FreeBSD.org
Thu May 5 14:39:14 UTC 2011
Author: attilio
Date: Thu May 5 14:39:14 2011
New Revision: 221499
URL: http://svn.freebsd.org/changeset/base/221499
Log:
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
Added:
projects/largeSMP/sys/sys/_cpuset.h (contents, props changed)
Modified:
projects/largeSMP/gnu/usr.bin/gdb/kgdb/kthr.c
projects/largeSMP/lib/libmemstat/memstat_uma.c
projects/largeSMP/sys/amd64/acpica/acpi_wakeup.c
projects/largeSMP/sys/amd64/amd64/intr_machdep.c
projects/largeSMP/sys/amd64/amd64/mp_machdep.c
projects/largeSMP/sys/amd64/amd64/pmap.c
projects/largeSMP/sys/amd64/amd64/vm_machdep.c
projects/largeSMP/sys/amd64/include/_types.h
projects/largeSMP/sys/amd64/include/param.h
projects/largeSMP/sys/amd64/include/pmap.h
projects/largeSMP/sys/amd64/include/smp.h
projects/largeSMP/sys/arm/arm/pmap.c
projects/largeSMP/sys/arm/include/_types.h
projects/largeSMP/sys/arm/include/pmap.h
projects/largeSMP/sys/cddl/dev/cyclic/i386/cyclic_machdep.c
projects/largeSMP/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
projects/largeSMP/sys/cddl/dev/dtrace/i386/dtrace_subr.c
projects/largeSMP/sys/dev/hwpmc/hwpmc_mod.c
projects/largeSMP/sys/dev/xen/control/control.c
projects/largeSMP/sys/geom/eli/g_eli.c
projects/largeSMP/sys/i386/i386/intr_machdep.c
projects/largeSMP/sys/i386/i386/mp_machdep.c
projects/largeSMP/sys/i386/i386/pmap.c
projects/largeSMP/sys/i386/i386/vm_machdep.c
projects/largeSMP/sys/i386/include/_types.h
projects/largeSMP/sys/i386/include/pmap.h
projects/largeSMP/sys/i386/include/sf_buf.h
projects/largeSMP/sys/i386/include/smp.h
projects/largeSMP/sys/i386/xen/mp_machdep.c
projects/largeSMP/sys/i386/xen/pmap.c
projects/largeSMP/sys/ia64/ia64/mp_machdep.c
projects/largeSMP/sys/ia64/include/_types.h
projects/largeSMP/sys/ia64/include/smp.h
projects/largeSMP/sys/kern/kern_cpuset.c
projects/largeSMP/sys/kern/kern_pmc.c
projects/largeSMP/sys/kern/kern_rmlock.c
projects/largeSMP/sys/kern/sched_4bsd.c
projects/largeSMP/sys/kern/sched_ule.c
projects/largeSMP/sys/kern/subr_kdb.c
projects/largeSMP/sys/kern/subr_pcpu.c
projects/largeSMP/sys/kern/subr_smp.c
projects/largeSMP/sys/ofed/include/linux/list.h
projects/largeSMP/sys/sys/_rmlock.h
projects/largeSMP/sys/sys/cpuset.h
projects/largeSMP/sys/sys/pcpu.h
projects/largeSMP/sys/sys/pmckern.h
projects/largeSMP/sys/sys/smp.h
projects/largeSMP/sys/sys/types.h
projects/largeSMP/sys/x86/x86/local_apic.c
projects/largeSMP/usr.sbin/pmccontrol/pmccontrol.c
Modified: projects/largeSMP/gnu/usr.bin/gdb/kgdb/kthr.c
==============================================================================
--- projects/largeSMP/gnu/usr.bin/gdb/kgdb/kthr.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/gnu/usr.bin/gdb/kgdb/kthr.c Thu May 5 14:39:14 2011 (r221499)
@@ -28,6 +28,7 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
+#include <sys/cpuset.h>
#include <sys/proc.h>
#include <sys/types.h>
#include <sys/signal.h>
@@ -37,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
#include <defs.h>
#include <frame-unwind.h>
@@ -48,7 +50,7 @@ static CORE_ADDR dumppcb;
static int dumptid;
static CORE_ADDR stoppcbs;
-static __cpumask_t stopped_cpus;
+static cpuset_t stopped_cpus;
static struct kthr *first;
struct kthr *curkthr;
@@ -76,6 +78,7 @@ kgdb_thr_init(void)
{
struct proc p;
struct thread td;
+ long cpusetsize;
struct kthr *kt;
CORE_ADDR addr;
uintptr_t paddr;
@@ -102,10 +105,11 @@ kgdb_thr_init(void)
dumptid = -1;
addr = kgdb_lookup("stopped_cpus");
- if (addr != 0)
- kvm_read(kvm, addr, &stopped_cpus, sizeof(stopped_cpus));
- else
- stopped_cpus = 0;
+ CPU_ZERO(&stopped_cpus);
+ cpusetsize = sysconf(_SC_CPUSET_SIZE);
+ if (cpusetsize != -1 && (u_long)cpusetsize <= sizeof(cpuset_t) &&
+ addr != 0)
+ kvm_read(kvm, addr, &stopped_cpus, cpusetsize);
stoppcbs = kgdb_lookup("stoppcbs");
@@ -126,8 +130,8 @@ kgdb_thr_init(void)
kt->kaddr = addr;
if (td.td_tid == dumptid)
kt->pcb = dumppcb;
- else if (td.td_state == TDS_RUNNING && ((1 << td.td_oncpu) & stopped_cpus)
- && stoppcbs != 0)
+ else if (td.td_state == TDS_RUNNING && stoppcbs != 0 &&
+ CPU_ISSET(td.td_oncpu, &stopped_cpus))
kt->pcb = (uintptr_t) stoppcbs + sizeof(struct pcb) * td.td_oncpu;
else
kt->pcb = (uintptr_t)td.td_pcb;
Modified: projects/largeSMP/lib/libmemstat/memstat_uma.c
==============================================================================
--- projects/largeSMP/lib/libmemstat/memstat_uma.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/lib/libmemstat/memstat_uma.c Thu May 5 14:39:14 2011 (r221499)
@@ -45,6 +45,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
#include "memstat.h"
#include "memstat_internal.h"
@@ -315,6 +316,7 @@ memstat_kvm_uma(struct memory_type_list
int hint_dontsearch, i, mp_maxid, ret;
char name[MEMTYPE_MAXNAME];
cpuset_t all_cpus;
+ long cpusetsize;
kvm_t *kvm;
kvm = (kvm_t *)kvm_handle;
@@ -338,7 +340,13 @@ memstat_kvm_uma(struct memory_type_list
list->mtl_error = ret;
return (-1);
}
- ret = kread_symbol(kvm, X_ALL_CPUS, &all_cpus, sizeof(all_cpus), 0);
+ cpusetsize = sysconf(_SC_CPUSET_SIZE);
+ if (cpusetsize == -1 || (u_long)cpusetsize > sizeof(cpuset_t)) {
+ list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL;
+ return (-1);
+ }
+ CPU_ZERO(&all_cpus);
+ ret = kread_symbol(kvm, X_ALL_CPUS, &all_cpus, cpusetsize, 0);
if (ret != 0) {
list->mtl_error = ret;
return (-1);
Modified: projects/largeSMP/sys/amd64/acpica/acpi_wakeup.c
==============================================================================
--- projects/largeSMP/sys/amd64/acpica/acpi_wakeup.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/sys/amd64/acpica/acpi_wakeup.c Thu May 5 14:39:14 2011 (r221499)
@@ -78,7 +78,7 @@ static void acpi_stop_beep(void *);
#ifdef SMP
static int acpi_wakeup_ap(struct acpi_softc *, int);
-static void acpi_wakeup_cpus(struct acpi_softc *, cpumask_t);
+static void acpi_wakeup_cpus(struct acpi_softc *, const cpuset_t *);
#endif
#define WAKECODE_VADDR(sc) ((sc)->acpi_wakeaddr + (3 * PAGE_SIZE))
@@ -173,7 +173,7 @@ acpi_wakeup_ap(struct acpi_softc *sc, in
#define BIOS_WARM (0x0a)
static void
-acpi_wakeup_cpus(struct acpi_softc *sc, cpumask_t wakeup_cpus)
+acpi_wakeup_cpus(struct acpi_softc *sc, const cpuset_t *wakeup_cpus)
{
uint32_t mpbioswarmvec;
int cpu;
@@ -192,7 +192,7 @@ acpi_wakeup_cpus(struct acpi_softc *sc,
/* Wake up each AP. */
for (cpu = 1; cpu < mp_ncpus; cpu++) {
- if ((wakeup_cpus & (1 << cpu)) == 0)
+ if (!CPU_ISSET(cpu, wakeup_cpus))
continue;
if (acpi_wakeup_ap(sc, cpu) == 0) {
/* restore the warmstart vector */
@@ -214,7 +214,7 @@ int
acpi_sleep_machdep(struct acpi_softc *sc, int state)
{
#ifdef SMP
- cpumask_t wakeup_cpus;
+ cpuset_t wakeup_cpus;
#endif
register_t cr3, rf;
ACPI_STATUS status;
@@ -244,10 +244,9 @@ acpi_sleep_machdep(struct acpi_softc *sc
if (savectx(susppcbs[0])) {
#ifdef SMP
- if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) {
- device_printf(sc->acpi_dev,
- "Failed to suspend APs: CPU mask = 0x%jx\n",
- (uintmax_t)(wakeup_cpus & ~stopped_cpus));
+ if (!CPU_EMPTY(&wakeup_cpus) &&
+ suspend_cpus(wakeup_cpus) == 0) {
+ device_printf(sc->acpi_dev, "Failed to suspend APs\n");
goto out;
}
#endif
@@ -282,8 +281,8 @@ acpi_sleep_machdep(struct acpi_softc *sc
PCPU_SET(switchtime, 0);
PCPU_SET(switchticks, ticks);
#ifdef SMP
- if (wakeup_cpus != 0)
- acpi_wakeup_cpus(sc, wakeup_cpus);
+ if (!CPU_EMPTY(&wakeup_cpus))
+ acpi_wakeup_cpus(sc, &wakeup_cpus);
#endif
acpi_resync_clock(sc);
ret = 0;
@@ -291,7 +290,7 @@ acpi_sleep_machdep(struct acpi_softc *sc
out:
#ifdef SMP
- if (wakeup_cpus != 0)
+ if (!CPU_EMPTY(&wakeup_cpus))
restart_cpus(wakeup_cpus);
#endif
Modified: projects/largeSMP/sys/amd64/amd64/intr_machdep.c
==============================================================================
--- projects/largeSMP/sys/amd64/amd64/intr_machdep.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/sys/amd64/amd64/intr_machdep.c Thu May 5 14:39:14 2011 (r221499)
@@ -443,8 +443,7 @@ DB_SHOW_COMMAND(irqs, db_show_irqs)
* allocate CPUs round-robin.
*/
-/* The BSP is always a valid target. */
-static cpumask_t intr_cpus = (1 << 0);
+static cpuset_t intr_cpus;
static int current_cpu;
/*
@@ -466,7 +465,7 @@ intr_next_cpu(void)
current_cpu++;
if (current_cpu > mp_maxid)
current_cpu = 0;
- } while (!(intr_cpus & (1 << current_cpu)));
+ } while (!CPU_ISSET(current_cpu, &intr_cpus));
mtx_unlock_spin(&icu_lock);
return (apic_id);
}
@@ -497,7 +496,7 @@ intr_add_cpu(u_int cpu)
printf("INTR: Adding local APIC %d as a target\n",
cpu_apic_ids[cpu]);
- intr_cpus |= (1 << cpu);
+ CPU_SET(cpu, &intr_cpus);
}
/*
@@ -510,6 +509,9 @@ intr_shuffle_irqs(void *arg __unused)
struct intsrc *isrc;
int i;
+ /* The BSP is always a valid target. */
+ CPU_SETOF(0, &intr_cpus);
+
/* Don't bother on UP. */
if (mp_ncpus == 1)
return;
Modified: projects/largeSMP/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- projects/largeSMP/sys/amd64/amd64/mp_machdep.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/sys/amd64/amd64/mp_machdep.c Thu May 5 14:39:14 2011 (r221499)
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
+#include <sys/cpuset.h>
#ifdef GPROF
#include <sys/gmon.h>
#endif
@@ -125,7 +126,7 @@ extern inthand_t IDTVEC(fast_syscall), I
* Local data and functions.
*/
-static volatile cpumask_t ipi_nmi_pending;
+static volatile cpuset_t ipi_nmi_pending;
/* used to hold the AP's until we are ready to release them */
static struct mtx ap_boot_mtx;
@@ -161,7 +162,7 @@ static void release_aps(void *dummy);
static int hlt_logical_cpus;
static u_int hyperthreading_cpus; /* logical cpus sharing L1 cache */
-static cpumask_t hyperthreading_cpus_mask;
+static cpuset_t hyperthreading_cpus_mask;
static int hyperthreading_allowed = 1;
static struct sysctl_ctx_list logical_cpu_clist;
static u_int bootMP_size;
@@ -311,7 +312,7 @@ topo_probe(void)
if (cpu_topo_probed)
return;
- logical_cpus_mask = 0;
+ CPU_ZERO(&logical_cpus_mask);
if (mp_ncpus <= 1)
cpu_cores = cpu_logical = 1;
else if (cpu_vendor_id == CPU_VENDOR_AMD)
@@ -455,7 +456,7 @@ cpu_mp_probe(void)
* Always record BSP in CPU map so that the mbuf init code works
* correctly.
*/
- all_cpus = 1;
+ CPU_SETOF(0, &all_cpus);
if (mp_ncpus == 0) {
/*
* No CPUs were found, so this must be a UP system. Setup
@@ -582,6 +583,7 @@ cpu_mp_announce(void)
void
init_secondary(void)
{
+ cpuset_t tcpuset, tallcpus;
struct pcpu *pc;
struct nmi_pcpu *np;
u_int64_t msr, cr0;
@@ -713,19 +715,22 @@ init_secondary(void)
CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
+ tcpuset = PCPU_GET(cpumask);
/* Determine if we are a logical CPU. */
/* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */
if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
- logical_cpus_mask |= PCPU_GET(cpumask);
-
+ CPU_OR(&logical_cpus_mask, &tcpuset);
+
/* Determine if we are a hyperthread. */
if (hyperthreading_cpus > 1 &&
PCPU_GET(apic_id) % hyperthreading_cpus != 0)
- hyperthreading_cpus_mask |= PCPU_GET(cpumask);
+ CPU_OR(&hyperthreading_cpus_mask, &tcpuset);
/* Build our map of 'other' CPUs. */
- PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
+ tallcpus = all_cpus;
+ CPU_NAND(&tallcpus, &tcpuset);
+ PCPU_SET(other_cpus, tallcpus);
if (bootverbose)
lapic_dump("AP");
@@ -868,6 +873,7 @@ assign_cpu_ids(void)
static int
start_all_aps(void)
{
+ cpuset_t tallcpus, tcpuset;
vm_offset_t va = boot_address + KERNBASE;
u_int64_t *pt4, *pt3, *pt2;
u_int32_t mpbioswarmvec;
@@ -932,11 +938,14 @@ start_all_aps(void)
panic("AP #%d (PHY# %d) failed!", cpu, apic_id);
}
- all_cpus |= (1 << cpu); /* record AP in CPU map */
+ CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
}
/* build our map of 'other' CPUs */
- PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
+ tallcpus = all_cpus;
+ tcpuset = PCPU_GET(cpumask);
+ CPU_NAND(&tallcpus, &tcpuset);
+ PCPU_SET(other_cpus, tallcpus);
/* restore the warmstart vector */
*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
@@ -1065,6 +1074,30 @@ SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_
#endif /* COUNT_XINVLTLB_HITS */
/*
+ * Send an IPI to specified CPU handling the bitmap logic.
+ */
+static void
+ipi_send_cpu(int cpu, u_int ipi)
+{
+ u_int bitmap, old_pending, new_pending;
+
+ KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
+
+ if (IPI_IS_BITMAPED(ipi)) {
+ bitmap = 1 << ipi;
+ ipi = IPI_BITMAP_VECTOR;
+ do {
+ old_pending = cpu_ipi_pending[cpu];
+ new_pending = old_pending | bitmap;
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
+ if (old_pending)
+ return;
+ }
+ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+}
+
+/*
* Flush the TLB on all other CPU's
*/
static void
@@ -1088,28 +1121,19 @@ smp_tlb_shootdown(u_int vector, vm_offse
}
static void
-smp_targeted_tlb_shootdown(cpumask_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
+smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
- int ncpu, othercpus;
+ int cpu, ncpu, othercpus;
othercpus = mp_ncpus - 1;
- if (mask == (cpumask_t)-1) {
- ncpu = othercpus;
- if (ncpu < 1)
+ if (CPU_ISFULLSET(&mask)) {
+ if (othercpus < 1)
return;
} else {
- mask &= ~PCPU_GET(cpumask);
- if (mask == 0)
- return;
- ncpu = bitcount32(mask);
- if (ncpu > othercpus) {
- /* XXX this should be a panic offence */
- printf("SMP: tlb shootdown to %d other cpus (only have %d)\n",
- ncpu, othercpus);
- ncpu = othercpus;
- }
- /* XXX should be a panic, implied by mask == 0 above */
- if (ncpu < 1)
+ sched_pin();
+ CPU_NAND(&mask, PCPU_PTR(cpumask));
+ sched_unpin();
+ if (CPU_EMPTY(&mask))
return;
}
if (!(read_rflags() & PSL_I))
@@ -1118,39 +1142,25 @@ smp_targeted_tlb_shootdown(cpumask_t mas
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
- if (mask == (cpumask_t)-1)
+ if (CPU_ISFULLSET(&mask)) {
+ ncpu = othercpus;
ipi_all_but_self(vector);
- else
- ipi_selected(mask, vector);
+ } else {
+ ncpu = 0;
+ while ((cpu = cpusetobj_ffs(&mask)) != 0) {
+ cpu--;
+ CPU_CLR(cpu, &mask);
+ CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
+ cpu, vector);
+ ipi_send_cpu(cpu, vector);
+ ncpu++;
+ }
+ }
while (smp_tlb_wait < ncpu)
ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
-/*
- * Send an IPI to specified CPU handling the bitmap logic.
- */
-static void
-ipi_send_cpu(int cpu, u_int ipi)
-{
- u_int bitmap, old_pending, new_pending;
-
- KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
-
- if (IPI_IS_BITMAPED(ipi)) {
- bitmap = 1 << ipi;
- ipi = IPI_BITMAP_VECTOR;
- do {
- old_pending = cpu_ipi_pending[cpu];
- new_pending = old_pending | bitmap;
- } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
- old_pending, new_pending));
- if (old_pending)
- return;
- }
- lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
-}
-
void
smp_cache_flush(void)
{
@@ -1197,7 +1207,7 @@ smp_invlpg_range(vm_offset_t addr1, vm_o
}
void
-smp_masked_invltlb(cpumask_t mask)
+smp_masked_invltlb(cpuset_t mask)
{
if (smp_started) {
@@ -1209,7 +1219,7 @@ smp_masked_invltlb(cpumask_t mask)
}
void
-smp_masked_invlpg(cpumask_t mask, vm_offset_t addr)
+smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
{
if (smp_started) {
@@ -1221,7 +1231,7 @@ smp_masked_invlpg(cpumask_t mask, vm_off
}
void
-smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2)
+smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
@@ -1274,7 +1284,7 @@ ipi_bitmap_handler(struct trapframe fram
* send an IPI to a set of cpus.
*/
void
-ipi_selected(cpumask_t cpus, u_int ipi)
+ipi_selected(cpuset_t cpus, u_int ipi)
{
int cpu;
@@ -1284,12 +1294,12 @@ ipi_selected(cpumask_t cpus, u_int ipi)
* Set the mask of receiving CPUs for this purpose.
*/
if (ipi == IPI_STOP_HARD)
- atomic_set_int(&ipi_nmi_pending, cpus);
+ CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus);
- CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
- while ((cpu = ffs(cpus)) != 0) {
+ while ((cpu = cpusetobj_ffs(&cpus)) != 0) {
cpu--;
- cpus &= ~(1 << cpu);
+ CPU_CLR(cpu, &cpus);
+ CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
ipi_send_cpu(cpu, ipi);
}
}
@@ -1307,7 +1317,7 @@ ipi_cpu(int cpu, u_int ipi)
* Set the mask of receiving CPUs for this purpose.
*/
if (ipi == IPI_STOP_HARD)
- atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+ CPU_SET_ATOMIC(cpu, &ipi_nmi_pending);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
ipi_send_cpu(cpu, ipi);
@@ -1320,8 +1330,10 @@ void
ipi_all_but_self(u_int ipi)
{
+ sched_pin();
if (IPI_IS_BITMAPED(ipi)) {
ipi_selected(PCPU_GET(other_cpus), ipi);
+ sched_unpin();
return;
}
@@ -1331,7 +1343,8 @@ ipi_all_but_self(u_int ipi)
* Set the mask of receiving CPUs for this purpose.
*/
if (ipi == IPI_STOP_HARD)
- atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus));
+ CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus));
+ sched_unpin();
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
@@ -1340,7 +1353,7 @@ ipi_all_but_self(u_int ipi)
int
ipi_nmi_handler()
{
- cpumask_t cpumask;
+ cpuset_t cpumask;
/*
* As long as there is not a simple way to know about a NMI's
@@ -1348,11 +1361,13 @@ ipi_nmi_handler()
* the global pending bitword an IPI_STOP_HARD has been issued
* and should be handled.
*/
+ sched_pin();
cpumask = PCPU_GET(cpumask);
- if ((ipi_nmi_pending & cpumask) == 0)
+ sched_unpin();
+ if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
return (1);
- atomic_clear_int(&ipi_nmi_pending, cpumask);
+ CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
cpustop_handler();
return (0);
}
@@ -1364,23 +1379,25 @@ ipi_nmi_handler()
void
cpustop_handler(void)
{
- cpumask_t cpumask;
+ cpuset_t cpumask;
u_int cpu;
+ sched_pin();
cpu = PCPU_GET(cpuid);
cpumask = PCPU_GET(cpumask);
+ sched_unpin();
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
- atomic_set_int(&stopped_cpus, cpumask);
+ CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
/* Wait for restart */
- while (!(started_cpus & cpumask))
+ while (!CPU_OVERLAP(&started_cpus, &cpumask))
ia32_pause();
- atomic_clear_int(&started_cpus, cpumask);
- atomic_clear_int(&stopped_cpus, cpumask);
+ CPU_NAND_ATOMIC(&started_cpus, &cpumask);
+ CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
@@ -1395,7 +1412,7 @@ cpustop_handler(void)
void
cpususpend_handler(void)
{
- cpumask_t cpumask;
+ cpuset_t cpumask;
register_t cr3, rf;
u_int cpu;
@@ -1407,7 +1424,7 @@ cpususpend_handler(void)
if (savectx(susppcbs[cpu])) {
wbinvd();
- atomic_set_int(&stopped_cpus, cpumask);
+ CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
} else {
pmap_init_pat();
PCPU_SET(switchtime, 0);
@@ -1415,11 +1432,11 @@ cpususpend_handler(void)
}
/* Wait for resume */
- while (!(started_cpus & cpumask))
+ while (!CPU_OVERLAP(&started_cpus, &cpumask))
ia32_pause();
- atomic_clear_int(&started_cpus, cpumask);
- atomic_clear_int(&stopped_cpus, cpumask);
+ CPU_NAND_ATOMIC(&started_cpus, &cpumask);
+ CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
/* Restore CR3 and enable interrupts */
load_cr3(cr3);
@@ -1447,30 +1464,30 @@ SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_
static int
sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS)
{
- cpumask_t mask;
+ cpuset_t mask;
int error;
mask = hlt_cpus_mask;
- error = sysctl_handle_int(oidp, &mask, 0, req);
+ error = sysctl_handle_opaque(oidp, &mask, sizeof(mask), req);
if (error || !req->newptr)
return (error);
- if (logical_cpus_mask != 0 &&
- (mask & logical_cpus_mask) == logical_cpus_mask)
+ if (!CPU_EMPTY(&logical_cpus_mask) &&
+ CPU_SUBSET(&mask, &logical_cpus_mask))
hlt_logical_cpus = 1;
else
hlt_logical_cpus = 0;
if (! hyperthreading_allowed)
- mask |= hyperthreading_cpus_mask;
+ CPU_OR(&mask, &hyperthreading_cpus_mask);
- if ((mask & all_cpus) == all_cpus)
- mask &= ~(1<<0);
+ if (CPU_SUBSET(&mask, &all_cpus))
+ CPU_CLR(0, &mask);
hlt_cpus_mask = mask;
return (error);
}
-SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW,
- 0, 0, sysctl_hlt_cpus, "IU",
+SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus,
+ CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, sysctl_hlt_cpus, "S",
"Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2.");
static int
@@ -1484,15 +1501,15 @@ sysctl_hlt_logical_cpus(SYSCTL_HANDLER_A
return (error);
if (disable)
- hlt_cpus_mask |= logical_cpus_mask;
+ CPU_OR(&hlt_cpus_mask, &logical_cpus_mask);
else
- hlt_cpus_mask &= ~logical_cpus_mask;
+ CPU_NAND(&hlt_cpus_mask, &logical_cpus_mask);
if (! hyperthreading_allowed)
- hlt_cpus_mask |= hyperthreading_cpus_mask;
+ CPU_OR(&hlt_cpus_mask, &hyperthreading_cpus_mask);
- if ((hlt_cpus_mask & all_cpus) == all_cpus)
- hlt_cpus_mask &= ~(1<<0);
+ if (CPU_SUBSET(&hlt_cpus_mask, &all_cpus))
+ CPU_CLR(0, &hlt_cpus_mask);
hlt_logical_cpus = disable;
return (error);
@@ -1519,18 +1536,18 @@ sysctl_hyperthreading_allowed(SYSCTL_HAN
#endif
if (allowed)
- hlt_cpus_mask &= ~hyperthreading_cpus_mask;
+ CPU_NAND(&hlt_cpus_mask, &hyperthreading_cpus_mask);
else
- hlt_cpus_mask |= hyperthreading_cpus_mask;
+ CPU_OR(&hlt_cpus_mask, &hyperthreading_cpus_mask);
- if (logical_cpus_mask != 0 &&
- (hlt_cpus_mask & logical_cpus_mask) == logical_cpus_mask)
+ if (!CPU_EMPTY(&logical_cpus_mask) &&
+ CPU_SUBSET(&hlt_cpus_mask, &logical_cpus_mask))
hlt_logical_cpus = 1;
else
hlt_logical_cpus = 0;
- if ((hlt_cpus_mask & all_cpus) == all_cpus)
- hlt_cpus_mask &= ~(1<<0);
+ if (CPU_SUBSET(&hlt_cpus_mask, &all_cpus))
+ CPU_CLR(0, &hlt_cpus_mask);
hyperthreading_allowed = allowed;
return (error);
@@ -1540,7 +1557,7 @@ static void
cpu_hlt_setup(void *dummy __unused)
{
- if (logical_cpus_mask != 0) {
+ if (!CPU_EMPTY(&logical_cpus_mask)) {
TUNABLE_INT_FETCH("machdep.hlt_logical_cpus",
&hlt_logical_cpus);
sysctl_ctx_init(&logical_cpu_clist);
@@ -1554,20 +1571,21 @@ cpu_hlt_setup(void *dummy __unused)
&logical_cpus_mask, 0, "");
if (hlt_logical_cpus)
- hlt_cpus_mask |= logical_cpus_mask;
+ CPU_OR(&hlt_cpus_mask, &logical_cpus_mask);
/*
* If necessary for security purposes, force
* hyperthreading off, regardless of the value
* of hlt_logical_cpus.
*/
- if (hyperthreading_cpus_mask) {
+ if (!CPU_EMPTY(&hyperthreading_cpus_mask)) {
SYSCTL_ADD_PROC(&logical_cpu_clist,
SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
"hyperthreading_allowed", CTLTYPE_INT|CTLFLAG_RW,
0, 0, sysctl_hyperthreading_allowed, "IU", "");
if (! hyperthreading_allowed)
- hlt_cpus_mask |= hyperthreading_cpus_mask;
+ CPU_OR(&hlt_cpus_mask,
+ &hyperthreading_cpus_mask);
}
}
}
@@ -1576,7 +1594,7 @@ SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_AN
int
mp_grab_cpu_hlt(void)
{
- cpumask_t mask;
+ cpuset_t mask;
#ifdef MP_WATCHDOG
u_int cpuid;
#endif
@@ -1589,7 +1607,7 @@ mp_grab_cpu_hlt(void)
#endif
retval = 0;
- while (mask & hlt_cpus_mask) {
+ while (CPU_OVERLAP(&mask, &hlt_cpus_mask)) {
retval = 1;
__asm __volatile("sti; hlt" : : : "memory");
}
Modified: projects/largeSMP/sys/amd64/amd64/pmap.c
==============================================================================
--- projects/largeSMP/sys/amd64/amd64/pmap.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/sys/amd64/amd64/pmap.c Thu May 5 14:39:14 2011 (r221499)
@@ -123,6 +123,8 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#ifdef SMP
#include <sys/smp.h>
+#else
+#include <sys/cpuset.h>
#endif
#include <vm/vm.h>
@@ -581,7 +583,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
PMAP_LOCK_INIT(kernel_pmap);
kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
kernel_pmap->pm_root = NULL;
- kernel_pmap->pm_active = -1; /* don't allow deactivation */
+ CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvchunk);
/*
@@ -923,19 +925,20 @@ pmap_update_pde_invalidate(vm_offset_t v
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- cpumask_t cpumask, other_cpus;
+ cpuset_t cpumask, other_cpus;
sched_pin();
- if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+ if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invlpg(va);
smp_invlpg(va);
} else {
cpumask = PCPU_GET(cpumask);
other_cpus = PCPU_GET(other_cpus);
- if (pmap->pm_active & cpumask)
+ if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
invlpg(va);
- if (pmap->pm_active & other_cpus)
- smp_masked_invlpg(pmap->pm_active & other_cpus, va);
+ CPU_AND(&other_cpus, &pmap->pm_active);
+ if (!CPU_EMPTY(&other_cpus))
+ smp_masked_invlpg(other_cpus, va);
}
sched_unpin();
}
@@ -943,23 +946,23 @@ pmap_invalidate_page(pmap_t pmap, vm_off
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- cpumask_t cpumask, other_cpus;
+ cpuset_t cpumask, other_cpus;
vm_offset_t addr;
sched_pin();
- if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+ if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
smp_invlpg_range(sva, eva);
} else {
cpumask = PCPU_GET(cpumask);
other_cpus = PCPU_GET(other_cpus);
- if (pmap->pm_active & cpumask)
+ if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
- if (pmap->pm_active & other_cpus)
- smp_masked_invlpg_range(pmap->pm_active & other_cpus,
- sva, eva);
+ CPU_AND(&other_cpus, &pmap->pm_active);
+ if (!CPU_EMPTY(&other_cpus))
+ smp_masked_invlpg_range(other_cpus, sva, eva);
}
sched_unpin();
}
@@ -967,19 +970,20 @@ pmap_invalidate_range(pmap_t pmap, vm_of
void
pmap_invalidate_all(pmap_t pmap)
{
- cpumask_t cpumask, other_cpus;
+ cpuset_t cpumask, other_cpus;
sched_pin();
- if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+ if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invltlb();
smp_invltlb();
} else {
cpumask = PCPU_GET(cpumask);
other_cpus = PCPU_GET(other_cpus);
- if (pmap->pm_active & cpumask)
+ if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
invltlb();
- if (pmap->pm_active & other_cpus)
- smp_masked_invltlb(pmap->pm_active & other_cpus);
+ CPU_AND(&other_cpus, &pmap->pm_active);
+ if (!CPU_EMPTY(&other_cpus))
+ smp_masked_invltlb(other_cpus);
}
sched_unpin();
}
@@ -995,8 +999,8 @@ pmap_invalidate_cache(void)
}
struct pde_action {
- cpumask_t store; /* processor that updates the PDE */
- cpumask_t invalidate; /* processors that invalidate their TLB */
+ cpuset_t store; /* processor that updates the PDE */
+ cpuset_t invalidate; /* processors that invalidate their TLB */
vm_offset_t va;
pd_entry_t *pde;
pd_entry_t newpde;
@@ -1007,8 +1011,12 @@ pmap_update_pde_action(void *arg)
{
struct pde_action *act = arg;
- if (act->store == PCPU_GET(cpumask))
+ sched_pin();
+ if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
+ sched_unpin();
pde_store(act->pde, act->newpde);
+ } else
+ sched_unpin();
}
static void
@@ -1016,8 +1024,12 @@ pmap_update_pde_teardown(void *arg)
{
struct pde_action *act = arg;
- if ((act->invalidate & PCPU_GET(cpumask)) != 0)
+ sched_pin();
+ if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) {
+ sched_unpin();
pmap_update_pde_invalidate(act->va, act->newpde);
+ } else
+ sched_unpin();
}
/*
@@ -1032,26 +1044,28 @@ static void
pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
{
struct pde_action act;
- cpumask_t active, cpumask;
+ cpuset_t active, cpumask, other_cpus;
sched_pin();
cpumask = PCPU_GET(cpumask);
+ other_cpus = PCPU_GET(other_cpus);
if (pmap == kernel_pmap)
active = all_cpus;
else
active = pmap->pm_active;
- if ((active & PCPU_GET(other_cpus)) != 0) {
+ if (CPU_OVERLAP(&active, &other_cpus)) {
act.store = cpumask;
act.invalidate = active;
act.va = va;
act.pde = pde;
act.newpde = newpde;
- smp_rendezvous_cpus(cpumask | active,
+ CPU_OR(&cpumask, &active);
+ smp_rendezvous_cpus(cpumask,
smp_no_rendevous_barrier, pmap_update_pde_action,
pmap_update_pde_teardown, &act);
} else {
pde_store(pde, newpde);
- if ((active & cpumask) != 0)
+ if (CPU_OVERLAP(&active, &cpumask))
pmap_update_pde_invalidate(va, newpde);
}
sched_unpin();
@@ -1065,7 +1079,7 @@ PMAP_INLINE void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- if (pmap == kernel_pmap || pmap->pm_active)
+ if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
invlpg(va);
}
@@ -1074,7 +1088,7 @@ pmap_invalidate_range(pmap_t pmap, vm_of
{
vm_offset_t addr;
- if (pmap == kernel_pmap || pmap->pm_active)
+ if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
}
@@ -1083,7 +1097,7 @@ PMAP_INLINE void
pmap_invalidate_all(pmap_t pmap)
{
- if (pmap == kernel_pmap || pmap->pm_active)
+ if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
invltlb();
}
@@ -1099,7 +1113,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t
{
pde_store(pde, newpde);
- if (pmap == kernel_pmap || pmap->pm_active)
+ if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
pmap_update_pde_invalidate(va, newpde);
}
#endif /* !SMP */
@@ -1607,7 +1621,7 @@ pmap_pinit0(pmap_t pmap)
PMAP_LOCK_INIT(pmap);
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
pmap->pm_root = NULL;
- pmap->pm_active = 0;
+ CPU_ZERO(&pmap->pm_active);
PCPU_SET(curpmap, pmap);
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1649,7 +1663,7 @@ pmap_pinit(pmap_t pmap)
pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
pmap->pm_root = NULL;
- pmap->pm_active = 0;
+ CPU_ZERO(&pmap->pm_active);
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -5087,11 +5101,11 @@ pmap_activate(struct thread *td)
pmap = vmspace_pmap(td->td_proc->p_vmspace);
oldpmap = PCPU_GET(curpmap);
#ifdef SMP
- atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
- atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
+ CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask));
+ CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask));
#else
- oldpmap->pm_active &= ~PCPU_GET(cpumask);
- pmap->pm_active |= PCPU_GET(cpumask);
+ CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask));
+ CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask));
#endif
cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
td->td_pcb->pcb_cr3 = cr3;
Modified: projects/largeSMP/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- projects/largeSMP/sys/amd64/amd64/vm_machdep.c Thu May 5 14:19:37 2011 (r221498)
+++ projects/largeSMP/sys/amd64/amd64/vm_machdep.c Thu May 5 14:39:14 2011 (r221499)
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/pioctl.h>
#include <sys/proc.h>
+#include <sys/sched.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
@@ -70,6 +71,7 @@ __FBSDID("$FreeBSD$");
#include <machine/cpu.h>
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-projects
mailing list