git: 6287b5978c5f - stable/11 - Fix incorrect hypotl(3) result with subnormal numbers

[Dimitry Andric]

The branch stable/11 has been updated by dim:

URL: https://cgit.FreeBSD.org/src/commit/?id=6287b5978c5f3959ad7f9494681cf94d1243120e

commit 6287b5978c5f3959ad7f9494681cf94d1243120e
Author:     Dimitry Andric <dim at FreeBSD.org>
AuthorDate: 2021-02-10 22:28:43 +0000
Commit:     Dimitry Andric <dim at FreeBSD.org>
CommitDate: 2021-02-18 18:11:19 +0000

    Fix incorrect hypotl(3) result with subnormal numbers
    
    This adjusts the factor used to scale the subnormal numbers, so it
    becomes the right value after adjusting its exponent. Thanks to Steve
    Kargl for finding the most elegant fix.
    
    Also enable the hypot tests, and add a test case for this bug.
    
    PR:             253313
    
    (cherry picked from commit d3338f3355a612cf385632291f46c5777bba8d18)
    
    Fix lib/msun/test builds on platforms without 80-bit long doubles
    
    After d3338f3355a612cf385632291f46c5777bba8d18, the lib/msun test case
    'hypotl_near_underflow' would fail to compile on platforms where long
    doubles weren't 80 bit, like on x86. Disable this particular test on
    such platforms for now.
    
    PR:             253313
    
    (cherry picked from commit 25120662284466ecef976df8f86e97bafdedf991)
---
 contrib/netbsd-tests/lib/libm/t_hypot.c | 24 ++++++++++++++++++++++++
 lib/msun/src/e_hypotl.c                 |  2 +-
 lib/msun/tests/Makefile                 |  1 +
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/contrib/netbsd-tests/lib/libm/t_hypot.c b/contrib/netbsd-tests/lib/libm/t_hypot.c
index deb7e86ad5ac..cbb056ee470e 100644
--- a/contrib/netbsd-tests/lib/libm/t_hypot.c
+++ b/contrib/netbsd-tests/lib/libm/t_hypot.c
@@ -70,12 +70,36 @@ ATF_TC_BODY(pr50698, tc)
 	ATF_CHECK(!isnan(val));
 }
 
+#if __LDBL_MANT_DIG__ == 64
+ATF_TC(hypotl_near_underflow);
+ATF_TC_HEAD(hypotl_near_underflow, tc)
+{
+	atf_tc_set_md_var(tc, "descr", "Test hypotl near underflow");
+}
+
+ATF_TC_BODY(hypotl_near_underflow, tc)
+{
+	volatile long double a = 0x1.b2933cafa0bb7p-16383L;
+	volatile long double b = 0x1.fffffffffffffp-16351L;
+	volatile long double e = 0x1.fffffffffffffp-16351L;
+	volatile long double ulp = __LDBL_EPSILON__;
+
+	volatile long double val = hypotl(a, b);
+
+	ATF_CHECK(!isinf(val));
+	ATF_CHECK(fabsl(val - e) <= 2 * ulp);
+}
+#endif /* __LDBL_MANT_DIG__ == 64 */
+
 ATF_TP_ADD_TCS(tp)
 {
 
 	ATF_TP_ADD_TC(tp, hypot_integer);
 	ATF_TP_ADD_TC(tp, hypotf_integer);
 	ATF_TP_ADD_TC(tp, pr50698);
+#if __LDBL_MANT_DIG__ == 64
+	ATF_TP_ADD_TC(tp, hypotl_near_underflow);
+#endif /* __LDBL_MANT_DIG__ == 64 */
 
 	return atf_no_error();
 }
diff --git a/lib/msun/src/e_hypotl.c b/lib/msun/src/e_hypotl.c
index 7b5ab89562fe..3633079a10f4 100644
--- a/lib/msun/src/e_hypotl.c
+++ b/lib/msun/src/e_hypotl.c
@@ -82,7 +82,7 @@ hypotl(long double x, long double y)
 	        man_t manh, manl;
 		GET_LDBL_MAN(manh,manl,b);
 		if((manh|manl)==0) return a;
-		t1=0;
+		t1=1;
 		SET_HIGH_WORD(t1,ESW(MAX_EXP-2));	/* t1=2^(MAX_EXP-2) */
 		b *= t1;
 		a *= t1;
diff --git a/lib/msun/tests/Makefile b/lib/msun/tests/Makefile
index 949b0a5b7aad..cd60f7c71a15 100644
--- a/lib/msun/tests/Makefile
+++ b/lib/msun/tests/Makefile
@@ -30,6 +30,7 @@ NETBSD_ATF_TESTS_C+=	erf_test
 NETBSD_ATF_TESTS_C+=	exp_test
 NETBSD_ATF_TESTS_C+=	fmod_test
 NETBSD_ATF_TESTS_C+=	fe_round_test
+NETBSD_ATF_TESTS_C+=	hypot_test
 NETBSD_ATF_TESTS_C+=	infinity_test
 NETBSD_ATF_TESTS_C+=	ilogb_test
 NETBSD_ATF_TESTS_C+=	ldexp_test