git: 86c31aca33ff - main - arm64: add swapueword8/32

The branch main has been updated by kevans:

URL: https://cgit.FreeBSD.org/src/commit/?id=86c31aca33ff771b845acbbed3b3659fde7e710f

commit 86c31aca33ff771b845acbbed3b3659fde7e710f
Author:     Kyle Evans <kevans@FreeBSD.org>
AuthorDate: 2023-05-11 18:23:03 +0000
Commit:     Kyle Evans <kevans@FreeBSD.org>
CommitDate: 2023-05-11 18:23:14 +0000

    arm64: add swapueword8/32
    
    Much like casueword*, except just a plain old swap.  Maintains a similar
    interface to casu(9)- return value -1 (fault), 0 (success), or 1 (fail),
    and also both ll/sc and LSE variants are implemented.
    
    These will be used to implement 32-bit swp/swpb emulation on aarch64.
    
    Reveiwed by:    andrew
    Sponsored by:   Stormshield
    Sponsored by:   Klara, Inc.
    Differential Revision:  https://reviews.freebsd.org/D39837
---
 sys/arm64/arm64/support.S       | 92 +++++++++++++++++++++++++++++++++++++++++
 sys/arm64/arm64/support_ifunc.c | 24 +++++++++++
 sys/arm64/include/md_var.h      | 11 +++++
 3 files changed, 127 insertions(+)

diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 28ac1d553fbc..f7e9abcd910c 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -55,6 +55,98 @@ fsu_fault_nopcb:
 	ret
 END(fsu_fault)
 
+/*
+ * int swapueword8_llsc(volatile uint8_t *, uint8_t *)
+ */
+ENTRY(swapueword8_llsc)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+
+	ldrb	w7, [x1]
+
+	ldxrb	w2, [x0]
+	stxrb	w3, w7, [x0]
+	cbnz	w3, 1f
+
+	strb	w2, [x1]		/* Stash old value in *val */
+
+1:	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)
+	mov	w0, w3
+	ret
+END(swapueword8_llsc)
+
+/*
+ * int swapueword8_lse(volatile uint8_t *, uint8_t *)
+ */
+ENTRY(swapueword8_lse)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+
+	ldrb	w7, [x1]
+
+	.arch_extension lse
+	swpb	w7, w2, [x0]
+	.arch_extension nolse
+
+	strb	w2, [x1]		/* Stash old value in *val */
+
+	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)
+	mov	w0, #0
+	ret
+END(swapueword8_lse)
+
+/*
+ * int swapueword32_llsc(volatile uint32_t *, uint32_t *)
+ */
+ENTRY(swapueword32_llsc)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+
+	ldr	w7, [x1]
+
+	ldxr	w2, [x0]		/* Stash the old value in w2 */
+	stxr	w3, w7, [x0]		/* Store new value */
+	cbnz	w3, 1f
+
+	str	w2, [x1]		/* Stash old value in *val */
+
+1:	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)
+	mov	w0, w3
+	ret
+END(swapueword32_llsc)
+
+/*
+ * int swapueword32_lse(volatile uint32_t *, uint32_t *)
+ */
+ENTRY(swapueword32_lse)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+
+	ldr	w7, [x1]
+
+	.arch_extension lse
+	swp	w7, w2, [x0]
+	.arch_extension nolse
+
+	str	w2, [x1]		/* Stash old value in *val */
+
+	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)
+	mov	w0, #0
+	ret
+END(swapueword32_llsc)
+
 /*
  * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
  */
diff --git a/sys/arm64/arm64/support_ifunc.c b/sys/arm64/arm64/support_ifunc.c
index 8abf3f3c75af..8cbe1c97b0a1 100644
--- a/sys/arm64/arm64/support_ifunc.c
+++ b/sys/arm64/arm64/support_ifunc.c
@@ -32,6 +32,8 @@
 
 #include <machine/atomic.h>
 #include <machine/ifunc.h>
+#define _MD_WANT_SWAPWORD
+#include <machine/md_var.h>
 
 int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
 int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
@@ -39,6 +41,12 @@ int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
 int casueword_llsc(volatile u_long *, u_long, u_long *, u_long);
 int casueword_lse(volatile u_long *, u_long, u_long *, u_long);
 
+int swapueword8_llsc(volatile uint8_t *, uint8_t *);
+int swapueword8_lse(volatile uint8_t *, uint8_t *);
+
+int swapueword32_llsc(volatile uint32_t *, uint32_t *);
+int swapueword32_lse(volatile uint32_t *, uint32_t *);
+
 DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *base, uint32_t oldval,
     uint32_t *oldvalp, uint32_t newval))
 {
@@ -56,3 +64,19 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *base, u_long oldval,
 
 	return (casueword_llsc);
 }
+
+DEFINE_IFUNC(, int, swapueword8, (volatile uint8_t *base, uint8_t *val))
+{
+	if (lse_supported)
+		return (swapueword8_lse);
+
+	return (swapueword8_llsc);
+}
+
+DEFINE_IFUNC(, int, swapueword32, (volatile uint32_t *base, uint32_t *val))
+{
+	if (lse_supported)
+		return (swapueword32_lse);
+
+	return (swapueword32_llsc);
+}
diff --git a/sys/arm64/include/md_var.h b/sys/arm64/include/md_var.h
index 6c78317c830c..d6c1b4ca2764 100644
--- a/sys/arm64/include/md_var.h
+++ b/sys/arm64/include/md_var.h
@@ -57,4 +57,15 @@ void generic_bs_poke_2(void) __asm(__STRING(generic_bs_poke_2));
 void generic_bs_poke_4(void) __asm(__STRING(generic_bs_poke_4));
 void generic_bs_poke_8(void) __asm(__STRING(generic_bs_poke_8));
 
+#ifdef _MD_WANT_SWAPWORD
+/*
+ * XXX These are implemented primarily for swp/swpb emulation at the moment, and
+ * should be used sparingly with consideration -- they aren't implemented for
+ * any other platform.  If we use them anywhere else, at a minimum they need
+ * KASAN/KMSAN interceptors added.
+ */
+int	swapueword8(volatile uint8_t *base, uint8_t *val);
+int	swapueword32(volatile uint32_t *base, uint32_t *val);
+#endif
+
 #endif /* !_MACHINE_MD_VAR_H_ */