git: 88ac318759f2 - main - Support LSE atomics in the arm64 casue* functions
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 19 May 2022 10:31:27 UTC
The branch main has been updated by andrew:
URL: https://cgit.FreeBSD.org/src/commit/?id=88ac318759f28f34aa991ecc6f336326ec00c501
commit 88ac318759f28f34aa991ecc6f336326ec00c501
Author: Andrew Turner <andrew@FreeBSD.org>
AuthorDate: 2022-05-17 13:52:30 +0000
Commit: Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2022-05-19 10:30:21 +0000
Support LSE atomics in the arm64 casue* functions
As with atomic(9) use the ARMv8.1 Large System Extension atomic
instructions to implement the userspace compare and swap functions.
Reviewed by: kib
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D35234
---
sys/arm64/arm64/support.S | 52 +++++++++++++++++++++++++++++++-----
sys/arm64/arm64/support_ifunc.c | 58 +++++++++++++++++++++++++++++++++++++++++
sys/conf/files.arm64 | 1 +
3 files changed, 105 insertions(+), 6 deletions(-)
diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 4771814a9b45..885af5676159 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -56,9 +56,9 @@ fsu_fault_nopcb:
END(fsu_fault)
/*
- * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
+ * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
*/
-ENTRY(casueword32)
+ENTRY(casueword32_llsc)
check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
adr x6, fsu_fault /* Load the fault handler */
mov w5, #1
@@ -73,12 +73,32 @@ ENTRY(casueword32)
str w4, [x2] /* Store the read data */
mov w0, w5 /* Result same as store status */
ret /* Return */
-END(casueword32)
+END(casueword32_llsc)
/*
- * int casueword(volatile u_long *, u_long, u_long *, u_long)
+ * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
*/
-ENTRY(casueword)
+ENTRY(casueword32_lse)
+ check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+ adr x6, fsu_fault /* Load the fault handler */
+ SET_FAULT_HANDLER(x6, x4) /* And set it */
+ ENTER_USER_ACCESS(w6, x4)
+ mov w7, w1 /* Back up the compare value */
+ .arch_extension lse
+ cas w1, w3, [x0] /* Compare and Swap */
+ .arch_extension nolse
+ cmp w1, w7 /* Check if successful */
+ cset w0, ne /* Return 0 on success, 1 on failure */
+ EXIT_USER_ACCESS(w6)
+ SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
+ str w1, [x2] /* Store the read data */
+ ret /* Return */
+END(casueword32_lse)
+
+/*
+ * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long)
+ */
+ENTRY(casueword_llsc)
check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb
adr x6, fsu_fault /* Load the fault handler */
mov w5, #1
@@ -93,7 +113,27 @@ ENTRY(casueword)
str x4, [x2] /* Store the read data */
mov w0, w5 /* Result same as store status */
ret /* Return */
-END(casueword)
+END(casueword_llsc)
+
+/*
+ * int casueword_lse(volatile u_long *, u_long, u_long *, u_long)
+ */
+ENTRY(casueword_lse)
+ check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+ adr x6, fsu_fault /* Load the fault handler */
+ SET_FAULT_HANDLER(x6, x4) /* And set it */
+ ENTER_USER_ACCESS(w6, x4)
+ mov x7, x1 /* Back up the compare value */
+ .arch_extension lse
+ cas x1, x3, [x0] /* Compare and Swap */
+ .arch_extension nolse
+ cmp x1, x7 /* Check if successful */
+ cset w0, ne /* Return 0 on success, 1 on failure */
+ EXIT_USER_ACCESS(w6)
+ SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
+ str x1, [x2] /* Store the read data */
+ ret /* Return */
+END(casueword_lse)
.macro fsudata insn, ret_reg, user_arg
adr x7, fsu_fault /* Load the fault handler */
diff --git a/sys/arm64/arm64/support_ifunc.c b/sys/arm64/arm64/support_ifunc.c
new file mode 100644
index 000000000000..8abf3f3c75af
--- /dev/null
+++ b/sys/arm64/arm64/support_ifunc.c
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * This software was developed by Andrew Turner under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+
+#include <machine/atomic.h>
+#include <machine/ifunc.h>
+
+int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
+int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
+
+int casueword_llsc(volatile u_long *, u_long, u_long *, u_long);
+int casueword_lse(volatile u_long *, u_long, u_long *, u_long);
+
+DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *base, uint32_t oldval,
+ uint32_t *oldvalp, uint32_t newval))
+{
+ if (lse_supported)
+ return (casueword32_lse);
+
+ return (casueword32_llsc);
+}
+
+DEFINE_IFUNC(, int, casueword, (volatile u_long *base, u_long oldval,
+ u_long *oldvalp, u_long newval))
+{
+ if (lse_supported)
+ return (casueword_lse);
+
+ return (casueword_llsc);
+}
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 30358a5d67d1..e0f529506d3d 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -72,6 +72,7 @@ arm64/arm64/pmap.c standard
arm64/arm64/ptrace_machdep.c standard
arm64/arm64/sigtramp.S standard
arm64/arm64/stack_machdep.c optional ddb | stack
+arm64/arm64/support_ifunc.c standard
arm64/arm64/support.S standard
arm64/arm64/swtch.S standard
arm64/arm64/sys_machdep.c standard