svn commit: r306521 - in head/sys: amd64/amd64 amd64/include x86/include x86/x86
Conrad E. Meyer
cem at FreeBSD.org
Fri Sep 30 18:58:52 UTC 2016
Author: cem
Date: Fri Sep 30 18:58:50 2016
New Revision: 306521
URL: https://svnweb.freebsd.org/changeset/base/306521
Log:
Revert r306516 for now, it is incomplete on i386
Noted by: kib
Modified:
head/sys/amd64/amd64/mp_machdep.c
head/sys/amd64/include/pcpu.h
head/sys/x86/include/x86_smp.h
head/sys/x86/x86/mp_x86.c
Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c Fri Sep 30 18:47:34 2016 (r306520)
+++ head/sys/amd64/amd64/mp_machdep.c Fri Sep 30 18:58:50 2016 (r306521)
@@ -409,7 +409,6 @@ void
invltlb_invpcid_handler(void)
{
struct invpcid_descr d;
- uint64_t generation;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
@@ -418,20 +417,17 @@ invltlb_invpcid_handler(void)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- generation = smp_tlb_generation;
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
d.pad = 0;
d.addr = 0;
invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
INVPCID_CTX);
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
void
invltlb_pcid_handler(void)
{
- uint64_t generation;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -439,7 +435,6 @@ invltlb_pcid_handler(void)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- generation = smp_tlb_generation;
if (smp_tlb_pmap == kernel_pmap) {
invltlb_glob();
} else {
@@ -455,5 +450,5 @@ invltlb_pcid_handler(void)
smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid);
}
}
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h Fri Sep 30 18:47:34 2016 (r306520)
+++ head/sys/amd64/include/pcpu.h Fri Sep 30 18:58:50 2016 (r306521)
@@ -65,8 +65,7 @@
u_int pc_vcpu_id; /* Xen vCPU ID */ \
uint32_t pc_pcid_next; \
uint32_t pc_pcid_gen; \
- uint64_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
- char __pad[141] /* be divisor of PAGE_SIZE \
+ char __pad[149] /* be divisor of PAGE_SIZE \
after cache alignment */
#define PC_DBREG_CMD_NONE 0
Modified: head/sys/x86/include/x86_smp.h
==============================================================================
--- head/sys/x86/include/x86_smp.h Fri Sep 30 18:47:34 2016 (r306520)
+++ head/sys/x86/include/x86_smp.h Fri Sep 30 18:58:50 2016 (r306521)
@@ -35,7 +35,7 @@ extern volatile int aps_ready;
extern struct mtx ap_boot_mtx;
extern int cpu_logical;
extern int cpu_cores;
-extern volatile uint64_t smp_tlb_generation;
+extern volatile int smp_tlb_wait;
extern struct pmap *smp_tlb_pmap;
extern u_int xhits_gbl[];
extern u_int xhits_pg[];
Modified: head/sys/x86/x86/mp_x86.c
==============================================================================
--- head/sys/x86/x86/mp_x86.c Fri Sep 30 18:47:34 2016 (r306520)
+++ head/sys/x86/x86/mp_x86.c Fri Sep 30 18:58:50 2016 (r306521)
@@ -1304,15 +1304,12 @@ cpususpend_handler(void)
void
invlcache_handler(void)
{
- uint64_t generation;
-
#ifdef COUNT_IPIS
(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- generation = smp_tlb_generation;
wbinvd();
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
/*
@@ -1370,7 +1367,7 @@ SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_
/* Variables needed for SMP tlb shootdown. */
static vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
pmap_t smp_tlb_pmap;
-volatile uint64_t smp_tlb_generation;
+volatile int smp_tlb_wait;
#ifdef __amd64__
#define read_eflags() read_rflags()
@@ -1380,16 +1377,15 @@ static void
smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
vm_offset_t addr1, vm_offset_t addr2)
{
- cpuset_t other_cpus;
- volatile uint64_t *p_cpudone;
- uint64_t generation;
- int cpu;
+ int cpu, ncpu, othercpus;
+
+ othercpus = mp_ncpus - 1; /* does not shootdown self */
/*
* Check for other cpus. Return if none.
*/
if (CPU_ISFULLSET(&mask)) {
- if (mp_ncpus <= 1)
+ if (othercpus < 1)
return;
} else {
CPU_CLR(PCPU_GET(cpuid), &mask);
@@ -1403,28 +1399,23 @@ smp_targeted_tlb_shootdown(cpuset_t mask
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
smp_tlb_pmap = pmap;
- generation = ++smp_tlb_generation;
+ smp_tlb_wait = 0;
if (CPU_ISFULLSET(&mask)) {
+ ncpu = othercpus;
ipi_all_but_self(vector);
- other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
} else {
- other_cpus = mask;
+ ncpu = 0;
while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
cpu, vector);
ipi_send_cpu(cpu, vector);
+ ncpu++;
}
}
- while ((cpu = CPU_FFS(&other_cpus)) != 0) {
- cpu--;
- CPU_CLR(cpu, &other_cpus);
- p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done;
- while (*p_cpudone != generation)
- ia32_pause();
- }
+ while (smp_tlb_wait < ncpu)
+ ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
@@ -1482,8 +1473,6 @@ smp_cache_flush(void)
void
invltlb_handler(void)
{
- uint64_t generation;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -1491,19 +1480,16 @@ invltlb_handler(void)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- generation = smp_tlb_generation;
if (smp_tlb_pmap == kernel_pmap)
invltlb_glob();
else
invltlb();
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
- uint64_t generation;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -1511,16 +1497,14 @@ invlpg_handler(void)
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- generation = smp_tlb_generation;
invlpg(smp_tlb_addr1);
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
void
invlrng_handler(void)
{
- vm_offset_t addr, addr2;
- uint64_t generation;
+ vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
@@ -1530,12 +1514,10 @@ invlrng_handler(void)
#endif /* COUNT_IPIS */
addr = smp_tlb_addr1;
- addr2 = smp_tlb_addr2;
- generation = smp_tlb_generation;
do {
invlpg(addr);
addr += PAGE_SIZE;
- } while (addr < addr2);
+ } while (addr < smp_tlb_addr2);
- PCPU_SET(smp_tlb_done, generation);
+ atomic_add_int(&smp_tlb_wait, 1);
}
More information about the svn-src-all
mailing list