svn commit: r342238 - in stable/12: lib/libc/amd64/string sys/amd64/amd64

[Mateusz Guzik]

Author: mjg
Date: Wed Dec 19 21:03:17 2018
New Revision: 342238
URL: https://svnweb.freebsd.org/changeset/base/342238

Log:
  MFC r341364
  
  amd64: align target memmove buffer to 16 bytes before using rep movs

Modified:
  stable/12/lib/libc/amd64/string/memmove.S
  stable/12/sys/amd64/amd64/support.S
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/lib/libc/amd64/string/memmove.S
==============================================================================
--- stable/12/lib/libc/amd64/string/memmove.S	Wed Dec 19 20:29:52 2018	(r342237)
+++ stable/12/lib/libc/amd64/string/memmove.S	Wed Dec 19 21:03:17 2018	(r342238)
@@ -130,6 +130,8 @@ __FBSDID("$FreeBSD$");
 
 	ALIGN_TEXT
 1256:
+	testb	$15,%dil
+	jnz	100f
 .if \erms == 1
 	rep
 	movsb
@@ -140,6 +142,34 @@ __FBSDID("$FreeBSD$");
 	movq	%rdx,%rcx
 	andb	$7,%cl                         /* any bytes left? */
 	jne	1004b
+.endif
+	\end
+	ret
+100:
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	movq	%rdi,%r10
+	movq	%rdi,%rcx
+	andq	$15,%rcx
+	leaq	-16(%rdx,%rcx),%rdx
+	neg	%rcx
+	leaq	16(%rdi,%rcx),%rdi
+	leaq	16(%rsi,%rcx),%rsi
+	movq	%rdx,%rcx
+.if \erms == 1
+	rep
+	movsb
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+.else
+	shrq	$3,%rcx                         /* copy by 64-bit words */
+	rep
+	movsq
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+	movq	%rdx,%rcx
+	andl	$7,%ecx                         /* any bytes left? */
+	jne	100408b
 .endif
 	\end
 	ret

Modified: stable/12/sys/amd64/amd64/support.S
==============================================================================
--- stable/12/sys/amd64/amd64/support.S	Wed Dec 19 20:29:52 2018	(r342237)
+++ stable/12/sys/amd64/amd64/support.S	Wed Dec 19 21:03:17 2018	(r342238)
@@ -293,6 +293,8 @@ END(memcmp)
 
 	ALIGN_TEXT
 1256:
+	testb	$15,%dil
+	jnz	100f
 .if \erms == 1
 	rep
 	movsb
@@ -303,6 +305,34 @@ END(memcmp)
 	movq	%rdx,%rcx
 	andb	$7,%cl                         /* any bytes left? */
 	jne	1004b
+.endif
+	\end
+	ret
+100:
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	movq	%rdi,%r10
+	movq	%rdi,%rcx
+	andq	$15,%rcx
+	leaq	-16(%rdx,%rcx),%rdx
+	neg	%rcx
+	leaq	16(%rdi,%rcx),%rdi
+	leaq	16(%rsi,%rcx),%rsi
+	movq	%rdx,%rcx
+.if \erms == 1
+	rep
+	movsb
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+.else
+	shrq	$3,%rcx                         /* copy by 64-bit words */
+	rep
+	movsq
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+	movq	%rdx,%rcx
+	andl	$7,%ecx                         /* any bytes left? */
+	jne	100408b
 .endif
 	\end
 	ret