git: 04df02b2f42e - stable/13 - armv8crypto: Factor out some duplicated GCM code

The branch stable/13 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=04df02b2f42ecb78bbc38b94c4764ee5eda385c2

commit 04df02b2f42ecb78bbc38b94c4764ee5eda385c2
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2022-02-16 02:45:59 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2022-03-18 15:29:24 +0000

    armv8crypto: Factor out some duplicated GCM code
    
    This is in preparation for using buffer cursors.  No functional change
    intended.
    
    Reviewed by:    jhb
    Sponsored by:   Ampere Computing LLC
    Submitted by:   Klara Inc.
    
    (cherry picked from commit 0b3235ef743d1561c57989042b3c364a5a955f4f)
---
 sys/crypto/armv8/armv8_crypto_wrap.c | 111 ++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 60 deletions(-)

diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c
index 3c0223964ee4..b5aee0cc1cf6 100644
--- a/sys/crypto/armv8/armv8_crypto_wrap.c
+++ b/sys/crypto/armv8/armv8_crypto_wrap.c
@@ -249,46 +249,71 @@ struct armv8_gcm_state {
 	uint8_t aes_counter[AES_BLOCK_LEN];
 };
 
-void
-armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
-    const uint8_t *from, uint8_t *to,
-    size_t authdatalen, const uint8_t *authdata,
-    uint8_t tag[static GMAC_DIGEST_LEN],
-    const uint8_t iv[static AES_GCM_IV_LEN],
-    const __uint128_val_t *Htable)
+static void
+armv8_aes_gmac_setup(struct armv8_gcm_state *s, AES_key_t *aes_key,
+    const uint8_t *authdata, size_t authdatalen,
+    const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable)
 {
-	struct armv8_gcm_state s;
-	const uint64_t *from64;
-	uint64_t *to64;
 	uint8_t block[AES_BLOCK_LEN];
-	size_t i, trailer;
+	size_t trailer;
 
-	bzero(&s.aes_counter, AES_BLOCK_LEN);
-	memcpy(s.aes_counter, iv, AES_GCM_IV_LEN);
+	bzero(s->aes_counter, AES_BLOCK_LEN);
+	memcpy(s->aes_counter, iv, AES_GCM_IV_LEN);
 
 	/* Setup the counter */
-	s.aes_counter[AES_BLOCK_LEN - 1] = 1;
+	s->aes_counter[AES_BLOCK_LEN - 1] = 1;
 
 	/* EK0 for a final GMAC round */
-	aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key);
+	aes_v8_encrypt(s->aes_counter, s->EK0.c, aes_key);
 
 	/* GCM starts with 2 as counter, 1 is used for final xor of tag. */
-	s.aes_counter[AES_BLOCK_LEN - 1] = 2;
+	s->aes_counter[AES_BLOCK_LEN - 1] = 2;
 
-	memset(s.Xi.c, 0, sizeof(s.Xi.c));
+	memset(s->Xi.c, 0, sizeof(s->Xi.c));
 	trailer = authdatalen % AES_BLOCK_LEN;
 	if (authdatalen - trailer > 0) {
-		gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer);
+		gcm_ghash_v8(s->Xi.u, Htable, authdata, authdatalen - trailer);
 		authdata += authdatalen - trailer;
 	}
 	if (trailer > 0 || authdatalen == 0) {
 		memset(block, 0, sizeof(block));
 		memcpy(block, authdata, trailer);
-		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
+		gcm_ghash_v8(s->Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
+}
 
-	from64 = (const uint64_t*)from;
-	to64 = (uint64_t*)to;
+static void
+armv8_aes_gmac_finish(struct armv8_gcm_state *s, size_t len,
+    size_t authdatalen, const __uint128_val_t *Htable)
+{
+	/* Lengths block */
+	s->lenblock.u[0] = s->lenblock.u[1] = 0;
+	s->lenblock.d[1] = htobe32(authdatalen * 8);
+	s->lenblock.d[3] = htobe32(len * 8);
+	gcm_ghash_v8(s->Xi.u, Htable, s->lenblock.c, AES_BLOCK_LEN);
+
+	s->Xi.u[0] ^= s->EK0.u[0];
+	s->Xi.u[1] ^= s->EK0.u[1];
+}
+
+void
+armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
+    const uint8_t *from, uint8_t *to,
+    size_t authdatalen, const uint8_t *authdata,
+    uint8_t tag[static GMAC_DIGEST_LEN],
+    const uint8_t iv[static AES_GCM_IV_LEN],
+    const __uint128_val_t *Htable)
+{
+	struct armv8_gcm_state s;
+	const uint64_t *from64;
+	uint64_t *to64;
+	uint8_t block[AES_BLOCK_LEN];
+	size_t i, trailer;
+
+	armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable);
+
+	from64 = (const uint64_t *)from;
+	to64 = (uint64_t *)to;
 	trailer = len % AES_BLOCK_LEN;
 
 	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
@@ -316,14 +341,7 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
 		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
 
-	/* Lengths block */
-	s.lenblock.u[0] = s.lenblock.u[1] = 0;
-	s.lenblock.d[1] = htobe32(authdatalen * 8);
-	s.lenblock.d[3] = htobe32(len * 8);
-	gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN);
-
-	s.Xi.u[0] ^= s.EK0.u[0];
-	s.Xi.u[1] ^= s.EK0.u[1];
+	armv8_aes_gmac_finish(&s, len, authdatalen, Htable);
 	memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN);
 
 	explicit_bzero(&s, sizeof(s));
@@ -345,26 +363,8 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
 	int error;
 
 	error = 0;
-	bzero(&s.aes_counter, AES_BLOCK_LEN);
-	memcpy(s.aes_counter, iv, AES_GCM_IV_LEN);
-
-	/* Setup the counter */
-	s.aes_counter[AES_BLOCK_LEN - 1] = 1;
-
-	/* EK0 for a final GMAC round */
-	aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key);
 
-	memset(s.Xi.c, 0, sizeof(s.Xi.c));
-	trailer = authdatalen % AES_BLOCK_LEN;
-	if (authdatalen - trailer > 0) {
-		gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer);
-		authdata += authdatalen - trailer;
-	}
-	if (trailer > 0 || authdatalen == 0) {
-		memset(block, 0, sizeof(block));
-		memcpy(block, authdata, trailer);
-		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
-	}
+	armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable);
 
 	trailer = len % AES_BLOCK_LEN;
 	if (len - trailer > 0)
@@ -375,24 +375,15 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
 		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
 
-	/* Lengths block */
-	s.lenblock.u[0] = s.lenblock.u[1] = 0;
-	s.lenblock.d[1] = htobe32(authdatalen * 8);
-	s.lenblock.d[3] = htobe32(len * 8);
-	gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN);
+	armv8_aes_gmac_finish(&s, len, authdatalen, Htable);
 
-	s.Xi.u[0] ^= s.EK0.u[0];
-	s.Xi.u[1] ^= s.EK0.u[1];
 	if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) {
 		error = EBADMSG;
 		goto out;
 	}
 
-	/* GCM starts with 2 as counter, 1 is used for final xor of tag. */
-	s.aes_counter[AES_BLOCK_LEN - 1] = 2;
-
-	from64 = (const uint64_t*)from;
-	to64 = (uint64_t*)to;
+	from64 = (const uint64_t *)from;
+	to64 = (uint64_t *)to;
 
 	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
 		aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);