svn commit: r346597 - in stable/12/sys: amd64/amd64 i386/i386

[Konstantin Belousov]

Author: kib
Date: Tue Apr 23 12:29:19 2019
New Revision: 346597
URL: https://svnweb.freebsd.org/changeset/base/346597

Log:
  MFC r345562 (by cem), r346294:
  x86: Use XSAVEOPT for fpusave(), when available.

Modified:
  stable/12/sys/amd64/amd64/fpu.c
  stable/12/sys/i386/i386/npx.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/amd64/amd64/fpu.c
==============================================================================
--- stable/12/sys/amd64/amd64/fpu.c	Tue Apr 23 12:23:44 2019	(r346596)
+++ stable/12/sys/amd64/amd64/fpu.c	Tue Apr 23 12:29:19 2019	(r346597)
@@ -100,6 +100,17 @@ xsave(char *addr, uint64_t mask)
 	    "memory");
 }
 
+static __inline void
+xsaveopt(char *addr, uint64_t mask)
+{
+	uint32_t low, hi;
+
+	low = mask;
+	hi = mask >> 32;
+	__asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) :
+	    "memory");
+}
+
 #else	/* !(__GNUCLIKE_ASM && !lint) */
 
 void	fldcw(u_short cw);
@@ -113,6 +124,7 @@ void	ldmxcsr(u_int csr);
 void	stmxcsr(u_int *csr);
 void	xrstor(char *addr, uint64_t mask);
 void	xsave(char *addr, uint64_t mask);
+void	xsaveopt(char *addr, uint64_t mask);
 
 #endif	/* __GNUCLIKE_ASM && !lint */
 
@@ -158,6 +170,13 @@ struct xsave_area_elm_descr {
 } *xsave_area_desc;
 
 static void
+fpusave_xsaveopt(void *addr)
+{
+
+	xsaveopt((char *)addr, xsave_mask);
+}
+
+static void
 fpusave_xsave(void *addr)
 {
 
@@ -201,7 +220,10 @@ DEFINE_IFUNC(, void, fpusave, (void *), static)
 {
 
 	init_xsave();
-	return (use_xsave ? fpusave_xsave : fpusave_fxsave);
+	if (use_xsave)
+		return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ?
+		    fpusave_xsaveopt : fpusave_xsave);
+	return (fpusave_fxsave);
 }
 
 DEFINE_IFUNC(, void, fpurestore, (void *), static)
@@ -348,6 +370,7 @@ fpuinit(void)
 static void
 fpuinitstate(void *arg __unused)
 {
+	uint64_t *xstate_bv;
 	register_t saveintr;
 	int cp[4], i, max_ext_n;
 
@@ -356,7 +379,7 @@ fpuinitstate(void *arg __unused)
 	saveintr = intr_disable();
 	stop_emulating();
 
-	fpusave(fpu_initialstate);
+	fpusave_fxsave(fpu_initialstate);
 	if (fpu_initialstate->sv_env.en_mxcsr_mask)
 		cpu_mxcsr_mask = fpu_initialstate->sv_env.en_mxcsr_mask;
 	else
@@ -378,6 +401,10 @@ fpuinitstate(void *arg __unused)
 	 * Save Area.
 	 */
 	if (use_xsave) {
+		xstate_bv = (uint64_t *)((char *)(fpu_initialstate + 1) +
+		    offsetof(struct xstate_hdr, xstate_bv));
+		*xstate_bv = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
+
 		max_ext_n = flsl(xsave_mask);
 		xsave_area_desc = malloc(max_ext_n * sizeof(struct
 		    xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO);

Modified: stable/12/sys/i386/i386/npx.c
==============================================================================
--- stable/12/sys/i386/i386/npx.c	Tue Apr 23 12:23:44 2019	(r346596)
+++ stable/12/sys/i386/i386/npx.c	Tue Apr 23 12:29:19 2019	(r346597)
@@ -313,7 +313,7 @@ cleanup:
 }
 
 static void
-npxsave_xsaveopt(union savefpu *addr)
+fpusave_xsaveopt(union savefpu *addr)
 {
 
 	xsaveopt((char *)addr, xsave_mask);
@@ -352,29 +352,18 @@ init_xsave(void)
 	TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
 }
 
-DEFINE_IFUNC(, void, npxsave_core, (union savefpu *), static)
+DEFINE_IFUNC(, void, fpusave, (union savefpu *), static)
 {
 
 	init_xsave();
 	if (use_xsave)
 		return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ?
-		    npxsave_xsaveopt : fpusave_xsave);
+		    fpusave_xsaveopt : fpusave_xsave);
 	if (cpu_fxsr)
 		return (fpusave_fxsave);
 	return (fpusave_fnsave);
 }
 
-DEFINE_IFUNC(, void, fpusave, (union savefpu *), static)
-{
-
-	init_xsave();
-	if (use_xsave)
-		return (fpusave_xsave);
-	if (cpu_fxsr)
-		return (fpusave_fxsave);
-	return (fpusave_fnsave);
-}
-
 /*
  * Enable XSAVE if supported and allowed by user.
  * Calculate the xsave_mask.
@@ -483,6 +472,7 @@ npxinit(bool bsp)
 static void
 npxinitstate(void *arg __unused)
 {
+	uint64_t *xstate_bv;
 	register_t saveintr;
 	int cp[4], i, max_ext_n;
 
@@ -494,7 +484,10 @@ npxinitstate(void *arg __unused)
 	saveintr = intr_disable();
 	stop_emulating();
 
-	fpusave(npx_initialstate);
+	if (cpu_fxsr)
+		fpusave_fxsave(npx_initialstate);
+	else
+		fpusave_fnsave(npx_initialstate);
 	if (cpu_fxsr) {
 		if (npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask)
 			cpu_mxcsr_mask = 
@@ -515,6 +508,7 @@ npxinitstate(void *arg __unused)
 		    sizeof(npx_initialstate->sv_xmm.sv_fp));
 		bzero(npx_initialstate->sv_xmm.sv_xmm,
 		    sizeof(npx_initialstate->sv_xmm.sv_xmm));
+
 	} else
 		bzero(npx_initialstate->sv_87.sv_ac,
 		    sizeof(npx_initialstate->sv_87.sv_ac));
@@ -524,6 +518,10 @@ npxinitstate(void *arg __unused)
 	 * Save Area.
 	 */
 	if (use_xsave) {
+		xstate_bv = (uint64_t *)((char *)(npx_initialstate + 1) +
+		    offsetof(struct xstate_hdr, xstate_bv));
+		*xstate_bv = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
+
 		if (xsave_mask >> 32 != 0)
 			max_ext_n = fls(xsave_mask >> 32) + 32;
 		else
@@ -922,7 +920,7 @@ npxsave(union savefpu *addr)
 {
 
 	stop_emulating();
-	npxsave_core(addr);
+	fpusave(addr);
 }
 
 void npxswitch(struct thread *td, struct pcb *pcb);