git: 30c8127e3c76 - stable/13 - linux(4): Implement futex_op for arm64.

The branch stable/13 has been updated by dchagin:

URL: https://cgit.FreeBSD.org/src/commit/?id=30c8127e3c76905d05473d1c6200b69cde0bd9e0

commit 30c8127e3c76905d05473d1c6200b69cde0bd9e0
Author:     Dmitry Chagin <dchagin@FreeBSD.org>
AuthorDate: 2022-05-15 17:49:42 +0000
Commit:     Dmitry Chagin <dchagin@FreeBSD.org>
CommitDate: 2022-06-17 19:35:19 +0000

    linux(4): Implement futex_op for arm64.
    
    It's mostly modeled like the Linux does.
    
    Differential revision:  https://reviews.freebsd.org/D35154
    MFC after:              2 weeks
    
    (cherry picked from commit 3bae1cd68a2ac3e2c37d54cf550e01257dfb1a04)
---
 sys/arm64/linux/linux_support.s | 130 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 123 insertions(+), 7 deletions(-)

diff --git a/sys/arm64/linux/linux_support.s b/sys/arm64/linux/linux_support.s
index 1048e9579627..933930b5096f 100644
--- a/sys/arm64/linux/linux_support.s
+++ b/sys/arm64/linux/linux_support.s
@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (C) 2018 Turing Robotic Industries Inc.
+ * Copyright (C) 2022 Dmitry Chagin <dchagin@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -27,36 +28,151 @@
  * $FreeBSD$
  */
 
-#include "linux_assym.h"
 #include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <machine/param.h>
+#include <machine/vmparam.h>
+
+#include <sys/errno.h>
 
 #include "assym.inc"
 
+.macro check_user_access user_arg, limit, bad_addr_func
+	ldr	x7, =(\limit)
+	cmp	x\user_arg, x7
+	b.cs	\bad_addr_func
+.endm
+
+futex_fault:
+	SET_FAULT_HANDLER(xzr, x1)
+	EXIT_USER_ACCESS_CHECK(w0, x1)
+futex_fault_nopcb:
+	mov	x0, #EFAULT
+	ret
+
+#define	LINUX_FUTEX_MAX_LOOPS	128
+
 /*
- * LINUXTODO: implement futex_*
+ * int oparg, uint32_t *uaddr, int *oldval
+ *
+ * Return 0 on success, errno on failure,
+ * EAGAIN is returned if LL/SC operation fails.
+ *
+ * XXX. VM_MAXUSER_ADDRESS is not applicable here, should be replaced
+ * by something like LINUX_SHAREDPAGE.
  */
 
+/* (int *)uaddr2 = oparg */
 ENTRY(futex_xchgl)
-	brk #0
+	check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb
+	adr	x9, futex_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x9, x4)	/* And set it */
+	ENTER_USER_ACCESS(w9, x4)
+	mov	w5, #LINUX_FUTEX_MAX_LOOPS
+	prfm	pstl1strm, [x1]
+	mov	w6, w0			/* Save oparg */
+1:	ldxr	w4, [x1]		/* Load oldval from uaddr */
+	stlxr	w0, w6, [x1]		/* Store oparg to uaddr */
+	cbz	w0, 3f			/* Exit on success */
+	sub	w5, w5, w0		/* Dec loop counter, w0 is 1 */
+	cbnz	w5, 1b			/* Loop */
+	mov	x0, #EAGAIN		/* Store of newval failed */
+3:	dmb	ish
+	EXIT_USER_ACCESS(w9)
+	SET_FAULT_HANDLER(xzr, x9)	/* Reset the fault handler */
+	str w4, [x2]			/* Store oldval */
 	ret
 END(futex_xchgl)
 
+/* (int *)uaddr2 += oparg */
 ENTRY(futex_addl)
-	brk #0
+	check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb
+	adr	x9, futex_fault
+	SET_FAULT_HANDLER(x9, x4)
+	ENTER_USER_ACCESS(w9, x4)
+	mov	w5, #LINUX_FUTEX_MAX_LOOPS
+	prfm	pstl1strm, [x1]
+	mov	w6, w0
+1:	ldxr	w4, [x1]
+	add	w3, w4, w6		/* oldval + oparg */
+	stlxr	w0, w3, [x1]
+	cbz	w0, 3f
+	sub	w5, w5, w0
+	cbnz	w5, 1b
+	mov	x0, #EAGAIN
+3:	dmb	ish
+	EXIT_USER_ACCESS(w9)
+	SET_FAULT_HANDLER(xzr, x9)
+	str w4, [x2]
 	ret
 END(futex_addl)
 
+/* (int *)uaddr2 |= oparg */
 ENTRY(futex_orl)
-	brk #0
+	check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb
+	adr	x9, futex_fault
+	SET_FAULT_HANDLER(x9, x4)
+	ENTER_USER_ACCESS(w9, x4)
+	mov	w5, #LINUX_FUTEX_MAX_LOOPS
+	prfm	pstl1strm, [x1]
+	mov	w6, w0
+1:	ldxr	w4, [x1]
+	orr	w3, w4, w6		/* oldavl |= oparg */
+	stlxr	w0, w3, [x1]
+	cbz	w0, 3f
+	sub	w5, w5, w0
+	cbnz	w5, 1b
+	mov	x0, #EAGAIN
+3:	dmb	ish
+	EXIT_USER_ACCESS(w9)
+	SET_FAULT_HANDLER(xzr, x9)
+	str w4, [x2]
 	ret
 END(futex_orl)
 
+/* (int *)uaddr2 &= oparg */
 ENTRY(futex_andl)
-	brk #0
+	check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb
+	adr	x9, futex_fault
+	SET_FAULT_HANDLER(x9, x4)
+	ENTER_USER_ACCESS(w9, x4)
+	mov	w5, #LINUX_FUTEX_MAX_LOOPS
+	prfm	pstl1strm, [x1]
+	mov	w6, w0
+1:	ldxr	w4, [x1]
+	and	w3, w4, w6		/* oldval &= oparg */
+	stlxr	w0, w3, [x1]
+	cbz	w0, 3f
+	sub	w5, w5, w0
+	cbnz	w5, 1b
+	mov	x0, #EAGAIN
+3:	dmb	ish
+	EXIT_USER_ACCESS(w9)
+	SET_FAULT_HANDLER(xzr, x9)
+	str w4, [x2]
 	ret
 END(futex_andl)
 
+/* (int *)uaddr2 ^= oparg */
 ENTRY(futex_xorl)
-	brk #0
+	check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb
+	adr	x9, futex_fault
+	SET_FAULT_HANDLER(x9, x4)
+	ENTER_USER_ACCESS(w9, x4)
+	mov	w5, #LINUX_FUTEX_MAX_LOOPS
+	prfm	pstl1strm, [x1]
+	mov	w6, w0
+1:	ldxr	w4, [x1]
+	eor	w3, w4, w6		/* oldval ^= oparg */
+	stlxr	w0, w3, [x1]
+	cbz	w0, 3f
+	sub	w5, w5, w0
+	cbnz	w5, 1b
+	mov	x0, #EAGAIN
+3:	dmb	ish
+	EXIT_USER_ACCESS(w9)
+	SET_FAULT_HANDLER(xzr, x9)
+	str w4, [x2]
 	ret
 END(futex_xorl)