svn commit: r338858 - head/sys/amd64/amd64
Mateusz Guzik
mjg at FreeBSD.org
Fri Sep 21 15:00:47 UTC 2018
Author: mjg
Date: Fri Sep 21 15:00:46 2018
New Revision: 338858
URL: https://svnweb.freebsd.org/changeset/base/338858
Log:
amd64: even up copyin/copyout with memcpy + other cleanup
- _fault handlers for both primitives are identical, provide just one
- change the copying scheme to match memcpy (in particular jump
avoidance for the most common case of multiply of 8)
- stop re-reading pcb address on exit, just store it locally (in r9)
Reviewed by: kib
Approved by: re (gjb)
Differential Revision: https://reviews.freebsd.org/D17265
Modified:
head/sys/amd64/amd64/support.S
Modified: head/sys/amd64/amd64/support.S
==============================================================================
--- head/sys/amd64/amd64/support.S Fri Sep 21 13:44:05 2018 (r338857)
+++ head/sys/amd64/amd64/support.S Fri Sep 21 15:00:46 2018 (r338858)
@@ -309,9 +309,9 @@ END(fillw)
*/
.macro COPYOUT smap erms
PUSH_FRAME_POINTER
- movq PCPU(CURPCB),%rax
+ movq PCPU(CURPCB),%r9
/* Trap entry clears PSL.AC */
- movq $copyout_fault,PCB_ONFAULT(%rax)
+ movq $copy_fault,PCB_ONFAULT(%r9)
testq %rdx,%rdx /* anything to do? */
jz 2f
@@ -327,7 +327,7 @@ END(fillw)
*/
movq %rsi,%rax
addq %rdx,%rax
- jc copyout_fault
+ jc copy_fault
/*
* XXX STOP USING VM_MAXUSER_ADDRESS.
* It is an end address, not a max, so every time it is used correctly it
@@ -336,7 +336,7 @@ END(fillw)
*/
movq $VM_MAXUSER_ADDRESS,%rcx
cmpq %rcx,%rax
- ja copyout_fault
+ ja copy_fault
xchgq %rdi,%rsi
/* bcopy(%rsi, %rdi, %rdx) */
@@ -344,21 +344,27 @@ END(fillw)
SMAP_DISABLE \smap
.if \erms == 0
+ cmpq $15,%rcx
+ jbe 1f
shrq $3,%rcx
rep
movsq
movb %dl,%cl
andb $7,%cl
- je 1f
+ jne 1f
+ SMAP_ENABLE \smap
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r9)
+ POP_FRAME_POINTER
+ ret
.endif
+1:
rep
movsb
-1:
SMAP_ENABLE \smap
2:
xorl %eax,%eax
- movq PCPU(CURPCB),%rdx
- movq %rax,PCB_ONFAULT(%rdx)
+ movq %rax,PCB_ONFAULT(%r9)
POP_FRAME_POINTER
ret
.endmacro
@@ -379,22 +385,14 @@ ENTRY(copyout_smap_erms)
COPYOUT smap=1 erms=1
END(copyout_smap_erms)
- ALIGN_TEXT
-copyout_fault:
- movq PCPU(CURPCB),%rdx
- movq $0,PCB_ONFAULT(%rdx)
- movq $EFAULT,%rax
- POP_FRAME_POINTER
- ret
-
/*
* copyin(from_user, to_kernel, len)
* %rdi, %rsi, %rdx
*/
.macro COPYIN smap erms
PUSH_FRAME_POINTER
- movq PCPU(CURPCB),%rax
- movq $copyin_fault,PCB_ONFAULT(%rax)
+ movq PCPU(CURPCB),%r9
+ movq $copy_fault,PCB_ONFAULT(%r9)
testq %rdx,%rdx /* anything to do? */
jz 2f
@@ -403,10 +401,10 @@ copyout_fault:
*/
movq %rdi,%rax
addq %rdx,%rax
- jc copyin_fault
+ jc copy_fault
movq $VM_MAXUSER_ADDRESS,%rcx
cmpq %rcx,%rax
- ja copyin_fault
+ ja copy_fault
xchgq %rdi,%rsi
movq %rdx,%rcx
@@ -414,22 +412,28 @@ copyout_fault:
SMAP_DISABLE \smap
.if \erms == 0
+ cmpq $15,%rcx
+ jbe 1f
shrq $3,%rcx /* copy longword-wise */
rep
movsq
movb %al,%cl
andb $7,%cl /* copy remaining bytes */
- je 1f
+ jne 1f
+ SMAP_ENABLE \smap
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r9)
+ POP_FRAME_POINTER
+ ret
.endif
+1:
rep
movsb
-1:
SMAP_ENABLE \smap
2:
xorl %eax,%eax
- movq PCPU(CURPCB),%rdx
- movq %rax,PCB_ONFAULT(%rdx)
+ movq %rax,PCB_ONFAULT(%r9)
POP_FRAME_POINTER
ret
.endmacro
@@ -451,10 +455,10 @@ ENTRY(copyin_smap_erms)
END(copyin_smap_erms)
ALIGN_TEXT
-copyin_fault:
+copy_fault:
movq PCPU(CURPCB),%rdx
movq $0,PCB_ONFAULT(%rdx)
- movq $EFAULT,%rax
+ movl $EFAULT,%eax
POP_FRAME_POINTER
ret
More information about the svn-src-all
mailing list