git: b2969efae83f - stable/13 - Use a builtin where possible in msun
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 14 Dec 2021 11:07:28 UTC
The branch stable/13 has been updated by andrew:
URL: https://cgit.FreeBSD.org/src/commit/?id=b2969efae83f9648c23297542cdce576ce7e5b49
commit b2969efae83f9648c23297542cdce576ce7e5b49
Author: Andrew Turner <andrew@FreeBSD.org>
AuthorDate: 2021-11-02 11:31:17 +0000
Commit: Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2021-12-14 10:58:01 +0000
Use a builtin where possible in msun
Some of the functions in msun can be implemented using a compiler
builtin function to generate a small number of instructions. Implement
this support in fma, fmax, fmin, and sqrt on arm64.
Care must be taken as the builtin can be implemented as a function
call on some architectures that lack direct support. In these cases
we need to use the original code path.
As we don't set errno on failure build with -fno-math-errno so the
toolchain doesn't convert a builtin into a function call when it
detects a failure, e.g. gcc will add a call to sqrt when the input
is negative leading to an infinite loop.
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D32801
(cherry picked from commit b2e843161dc3b79777e873183447c92ed9c3703a)
---
lib/msun/Makefile | 5 +++++
lib/msun/aarch64/Makefile.inc | 12 ++++++++++++
lib/msun/src/e_sqrt.c | 18 +++++++++++++-----
lib/msun/src/e_sqrtf.c | 8 ++++++++
lib/msun/src/s_fma.c | 8 ++++++++
lib/msun/src/s_fmaf.c | 8 ++++++++
lib/msun/src/s_fmax.c | 8 ++++++++
lib/msun/src/s_fmaxf.c | 8 ++++++++
lib/msun/src/s_fmin.c | 8 ++++++++
lib/msun/src/s_fminf.c | 8 ++++++++
10 files changed, 86 insertions(+), 5 deletions(-)
diff --git a/lib/msun/Makefile b/lib/msun/Makefile
index d7c0e2f88358..6274c27c12c5 100644
--- a/lib/msun/Makefile
+++ b/lib/msun/Makefile
@@ -45,6 +45,11 @@ CFLAGS+= -I${.CURDIR}/${ARCH_SUBDIR}
CFLAGS+= -ffp-exception-behavior=maytrap
.endif
+# Tell the compiler we don't set errno in any of the math functions. This
+# stops gcc from trying to generate a function call to set errno when using
+# a builtin in the implementation
+CFLAGS+= -fno-math-errno
+
.PATH: ${.CURDIR}/bsdsrc
.PATH: ${.CURDIR}/src
.PATH: ${.CURDIR}/man
diff --git a/lib/msun/aarch64/Makefile.inc b/lib/msun/aarch64/Makefile.inc
index 286a60859591..b0e01488b0bb 100644
--- a/lib/msun/aarch64/Makefile.inc
+++ b/lib/msun/aarch64/Makefile.inc
@@ -2,3 +2,15 @@
LDBL_PREC = 113
+# Use a builtin when it generates the needed instruction
+CFLAGS+=-DUSE_BUILTIN_FMAF
+CFLAGS+=-DUSE_BUILTIN_FMA
+
+CFLAGS+=-DUSE_BUILTIN_FMAXF
+CFLAGS+=-DUSE_BUILTIN_FMAX
+
+CFLAGS+=-DUSE_BUILTIN_FMINF
+CFLAGS+=-DUSE_BUILTIN_FMIN
+
+CFLAGS+=-DUSE_BUILTIN_SQRTF
+CFLAGS+=-DUSE_BUILTIN_SQRT
diff --git a/lib/msun/src/e_sqrt.c b/lib/msun/src/e_sqrt.c
index 12fb56e1340b..37351a425ce4 100644
--- a/lib/msun/src/e_sqrt.c
+++ b/lib/msun/src/e_sqrt.c
@@ -14,6 +14,18 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
+#include <float.h>
+
+#include "math.h"
+#include "math_private.h"
+
+#ifdef USE_BUILTIN_SQRT
+double
+__ieee754_sqrt(double x)
+{
+ return (__builtin_sqrt(x));
+}
+#else
/* __ieee754_sqrt(x)
* Return correctly rounded sqrt.
* ------------------------------------------
@@ -84,11 +96,6 @@ __FBSDID("$FreeBSD$");
*---------------
*/
-#include <float.h>
-
-#include "math.h"
-#include "math_private.h"
-
static const double one = 1.0, tiny=1.0e-300;
double
@@ -187,6 +194,7 @@ __ieee754_sqrt(double x)
INSERT_WORDS(z,ix0,ix1);
return z;
}
+#endif
#if (LDBL_MANT_DIG == 53)
__weak_reference(sqrt, sqrtl);
diff --git a/lib/msun/src/e_sqrtf.c b/lib/msun/src/e_sqrtf.c
index 7eba4d07f92c..06e5d62fa5f9 100644
--- a/lib/msun/src/e_sqrtf.c
+++ b/lib/msun/src/e_sqrtf.c
@@ -20,6 +20,13 @@ static char rcsid[] = "$FreeBSD$";
#include "math.h"
#include "math_private.h"
+#ifdef USE_BUILTIN_SQRTF
+float
+__ieee754_sqrtf(float x)
+{
+ return (__builtin_sqrtf(x));
+}
+#else
static const float one = 1.0, tiny=1.0e-30;
float
@@ -87,3 +94,4 @@ __ieee754_sqrtf(float x)
SET_FLOAT_WORD(z,ix);
return z;
}
+#endif
diff --git a/lib/msun/src/s_fma.c b/lib/msun/src/s_fma.c
index 41a6424fdf7f..95cffd0ba36d 100644
--- a/lib/msun/src/s_fma.c
+++ b/lib/msun/src/s_fma.c
@@ -35,6 +35,13 @@ __FBSDID("$FreeBSD$");
#include "math_private.h"
+#ifdef USE_BUILTIN_FMA
+double
+fma(double x, double y, double z)
+{
+ return (__builtin_fma(x, y, z));
+}
+#else
/*
* A struct dd represents a floating-point number with twice the precision
* of a double. We maintain the invariant that "hi" stores the 53 high-order
@@ -284,6 +291,7 @@ fma(double x, double y, double z)
else
return (add_and_denormalize(r.hi, adj, spread));
}
+#endif /* !USE_BUILTIN_FMA */
#if (LDBL_MANT_DIG == 53)
__weak_reference(fma, fmal);
diff --git a/lib/msun/src/s_fmaf.c b/lib/msun/src/s_fmaf.c
index 389cf1b11bee..4591cc219bbb 100644
--- a/lib/msun/src/s_fmaf.c
+++ b/lib/msun/src/s_fmaf.c
@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
#include "math.h"
#include "math_private.h"
+#ifdef USE_BUILTIN_FMAF
+float
+fmaf(float x, float y, float z)
+{
+ return (__builtin_fmaf(x, y, z));
+}
+#else
/*
* Fused multiply-add: Compute x * y + z with a single rounding error.
*
@@ -69,3 +76,4 @@ fmaf(float x, float y, float z)
SET_LOW_WORD(adjusted_result, lr + 1);
return (adjusted_result);
}
+#endif /* !USE_BUILTIN_FMAF */
diff --git a/lib/msun/src/s_fmax.c b/lib/msun/src/s_fmax.c
index 0c234bcc5f7f..b53b1e64da04 100644
--- a/lib/msun/src/s_fmax.c
+++ b/lib/msun/src/s_fmax.c
@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
#include "fpmath.h"
+#ifdef USE_BUILTIN_FMAX
+double
+fmax(double x, double y)
+{
+ return (__builtin_fmax(x, y));
+}
+#else
double
fmax(double x, double y)
{
@@ -54,6 +61,7 @@ fmax(double x, double y)
return (x > y ? x : y);
}
+#endif
#if (LDBL_MANT_DIG == 53)
__weak_reference(fmax, fmaxl);
diff --git a/lib/msun/src/s_fmaxf.c b/lib/msun/src/s_fmaxf.c
index 8e9d1baf3e8e..8d3d14f4b52a 100644
--- a/lib/msun/src/s_fmaxf.c
+++ b/lib/msun/src/s_fmaxf.c
@@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$");
#include "fpmath.h"
+#ifdef USE_BUILTIN_FMAXF
+float
+fmaxf(float x, float y)
+{
+ return (__builtin_fmaxf(x, y));
+}
+#else
float
fmaxf(float x, float y)
{
@@ -53,3 +60,4 @@ fmaxf(float x, float y)
return (x > y ? x : y);
}
+#endif
diff --git a/lib/msun/src/s_fmin.c b/lib/msun/src/s_fmin.c
index d7f24c121fb2..53f36c1c2d3d 100644
--- a/lib/msun/src/s_fmin.c
+++ b/lib/msun/src/s_fmin.c
@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
#include "fpmath.h"
+#ifdef USE_BUILTIN_FMIN
+double
+fmin(double x, double y)
+{
+ return (__builtin_fmin(x, y));
+}
+#else
double
fmin(double x, double y)
{
@@ -54,6 +61,7 @@ fmin(double x, double y)
return (x < y ? x : y);
}
+#endif
#if (LDBL_MANT_DIG == 53)
__weak_reference(fmin, fminl);
diff --git a/lib/msun/src/s_fminf.c b/lib/msun/src/s_fminf.c
index 258316725ddd..58b6a489b820 100644
--- a/lib/msun/src/s_fminf.c
+++ b/lib/msun/src/s_fminf.c
@@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$");
#include "fpmath.h"
+#ifdef USE_BUILTIN_FMINF
+float
+fminf(float x, float y)
+{
+ return (__builtin_fminf(x, y));
+}
+#else
float
fminf(float x, float y)
{
@@ -53,3 +60,4 @@ fminf(float x, float y)
return (x < y ? x : y);
}
+#endif