svn commit: r312555 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include

[Konstantin Belousov]

Author: kib
Date: Fri Jan 20 19:08:44 2017
New Revision: 312555
URL: https://svnweb.freebsd.org/changeset/base/312555

Log:
  Use SFENCE for ordering CLFLUSHOPT.
  
  SDM states that CLFLUSHOPT instructions can be ordered with other
  writes by SFENCE, heavier MFENCE is not required.
  
  Reviewed by:	alc
  Sponsored by:	The FreeBSD Foundation
  MFC after:	2 weeks

Modified:
  head/sys/amd64/amd64/pmap.c
  head/sys/amd64/include/cpufunc.h
  head/sys/i386/i386/pmap.c
  head/sys/i386/include/cpufunc.h

Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c	Fri Jan 20 18:37:14 2017	(r312554)
+++ head/sys/amd64/amd64/pmap.c	Fri Jan 20 19:08:44 2017	(r312555)
@@ -1862,16 +1862,16 @@ pmap_invalidate_cache_range(vm_offset_t 
 			return;
 
 		/*
-		 * Otherwise, do per-cache line flush.  Use the mfence
+		 * Otherwise, do per-cache line flush.  Use the sfence
 		 * instruction to insure that previous stores are
 		 * included in the write-back.  The processor
 		 * propagates flush to other processors in the cache
 		 * coherence domain.
 		 */
-		mfence();
+		sfence();
 		for (; sva < eva; sva += cpu_clflush_line_size)
 			clflushopt(sva);
-		mfence();
+		sfence();
 	} else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 	    eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 		if (pmap_kextract(sva) == lapic_paddr)
@@ -1915,7 +1915,9 @@ pmap_invalidate_cache_pages(vm_page_t *p
 	    ((cpu_feature & CPUID_CLFSH) == 0 && !useclflushopt))
 		pmap_invalidate_cache();
 	else {
-		if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL)
+		if (useclflushopt)
+			sfence();
+		else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 			mfence();
 		for (i = 0; i < count; i++) {
 			daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
@@ -1927,7 +1929,9 @@ pmap_invalidate_cache_pages(vm_page_t *p
 					clflush(daddr);
 			}
 		}
-		if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL)
+		if (useclflushopt)
+			sfence();
+		else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 			mfence();
 	}
 }

Modified: head/sys/amd64/include/cpufunc.h
==============================================================================
--- head/sys/amd64/include/cpufunc.h	Fri Jan 20 18:37:14 2017	(r312554)
+++ head/sys/amd64/include/cpufunc.h	Fri Jan 20 19:08:44 2017	(r312555)
@@ -327,6 +327,13 @@ mfence(void)
 }
 
 static __inline void
+sfence(void)
+{
+
+	__asm __volatile("sfence" : : : "memory");
+}
+
+static __inline void
 ia32_pause(void)
 {
 	__asm __volatile("pause");

Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c	Fri Jan 20 18:37:14 2017	(r312554)
+++ head/sys/i386/i386/pmap.c	Fri Jan 20 19:08:44 2017	(r312555)
@@ -1283,16 +1283,16 @@ pmap_invalidate_cache_range(vm_offset_t 
 			return;
 #endif
 		/*
-		 * Otherwise, do per-cache line flush.  Use the mfence
+		 * Otherwise, do per-cache line flush.  Use the sfence
 		 * instruction to insure that previous stores are
 		 * included in the write-back.  The processor
 		 * propagates flush to other processors in the cache
 		 * coherence domain.
 		 */
-		mfence();
+		sfence();
 		for (; sva < eva; sva += cpu_clflush_line_size)
 			clflushopt(sva);
-		mfence();
+		sfence();
 	} else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 	    eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 #ifdef DEV_APIC
@@ -5300,12 +5300,14 @@ pmap_flush_page(vm_page_t m)
 		eva = sva + PAGE_SIZE;
 
 		/*
-		 * Use mfence despite the ordering implied by
+		 * Use mfence or sfence despite the ordering implied by
 		 * mtx_{un,}lock() because clflush on non-Intel CPUs
 		 * and clflushopt are not guaranteed to be ordered by
 		 * any other instruction.
 		 */
-		if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL)
+		if (useclflushopt)
+			sfence();
+		else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 			mfence();
 		for (; sva < eva; sva += cpu_clflush_line_size) {
 			if (useclflushopt)
@@ -5313,7 +5315,9 @@ pmap_flush_page(vm_page_t m)
 			else
 				clflush(sva);
 		}
-		if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL)
+		if (useclflushopt)
+			sfence();
+		else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 			mfence();
 		*cmap_pte2 = 0;
 		sched_unpin();

Modified: head/sys/i386/include/cpufunc.h
==============================================================================
--- head/sys/i386/include/cpufunc.h	Fri Jan 20 18:37:14 2017	(r312554)
+++ head/sys/i386/include/cpufunc.h	Fri Jan 20 19:08:44 2017	(r312555)
@@ -158,6 +158,13 @@ mfence(void)
 	__asm __volatile("mfence" : : : "memory");
 }
 
+static __inline void
+sfence(void)
+{
+
+	__asm __volatile("sfence" : : : "memory");
+}
+
 #ifdef _KERNEL
 
 #define	HAVE_INLINE_FFS