git: bc6265e8cf8d - stable/13 - ossl: Add Poly1305 digest support.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 21 Oct 2021 17:06:56 UTC
The branch stable/13 has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=bc6265e8cf8dd4c6897b25307f89d1b7a3372e28
commit bc6265e8cf8dd4c6897b25307f89d1b7a3372e28
Author: John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2021-03-03 23:17:29 +0000
Commit: John Baldwin <jhb@FreeBSD.org>
CommitDate: 2021-10-21 15:51:23 +0000
ossl: Add Poly1305 digest support.
Reviewed by: cem
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D28754
(cherry picked from commit a079e38b08f2f07c50ba915dae66d099559abdcc)
---
share/man/man4/ossl.4 | 4 +-
sys/conf/files | 1 +
sys/conf/files.amd64 | 1 +
sys/conf/files.arm64 | 2 +
sys/conf/files.i386 | 1 +
sys/crypto/openssl/ossl.c | 46 ++++++----
sys/crypto/openssl/ossl.h | 3 +-
sys/crypto/openssl/ossl_poly1305.c | 181 +++++++++++++++++++++++++++++++++++++
sys/crypto/openssl/ossl_poly1305.h | 35 +++++++
sys/modules/ossl/Makefile | 4 +
10 files changed, 260 insertions(+), 18 deletions(-)
diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4
index ce97a9fc0f71..9c0d7f897d53 100644
--- a/share/man/man4/ossl.4
+++ b/share/man/man4/ossl.4
@@ -26,7 +26,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd December 4, 2020
+.Dd March 3, 2021
.Dt OSSL 4
.Os
.Sh NAME
@@ -74,6 +74,8 @@ driver includes support for the following algorithms:
.Pp
.Bl -bullet -compact
.It
+Poly1305
+.It
SHA1
.It
SHA1-HMAC
diff --git a/sys/conf/files b/sys/conf/files
index f21c216288b1..e680382210cd 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -741,6 +741,7 @@ crypto/chacha20/chacha-sw.c optional crypto | ipsec | ipsec_support
crypto/des/des_ecb.c optional netsmb
crypto/des/des_setkey.c optional netsmb
crypto/openssl/ossl.c optional ossl
+crypto/openssl/ossl_poly1305.c optional ossl
crypto/openssl/ossl_sha1.c optional ossl
crypto/openssl/ossl_sha256.c optional ossl
crypto/openssl/ossl_sha512.c optional ossl
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 60184978fb51..f6271d0e0e92 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -139,6 +139,7 @@ cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
crypto/aesni/aeskeys_amd64.S optional aesni
crypto/des/des_enc.c optional netsmb
+crypto/openssl/amd64/poly1305-x86_64.S optional ossl
crypto/openssl/amd64/sha1-x86_64.S optional ossl
crypto/openssl/amd64/sha256-x86_64.S optional ossl
crypto/openssl/amd64/sha512-x86_64.S optional ossl
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 6e0f5658f33c..d6c9586f58f7 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -127,6 +127,8 @@ ghashv8-armx.o optional armv8crypto \
crypto/des/des_enc.c optional netsmb
crypto/openssl/ossl_aarch64.c optional ossl
+crypto/openssl/aarch64/poly1305-armv8.S optional ossl \
+ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}"
crypto/openssl/aarch64/sha1-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}"
crypto/openssl/aarch64/sha256-armv8.S optional ossl \
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 6a0ca49c0432..5238e2bab40a 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -77,6 +77,7 @@ compat/linux/linux.c optional compat_linux
compat/ndis/winx32_wrap.S optional ndisapi pci
crypto/aesni/aeskeys_i386.S optional aesni
crypto/des/arch/i386/des_enc.S optional netsmb
+crypto/openssl/i386/poly1305-x86.S optional ossl
crypto/openssl/i386/sha1-586.S optional ossl
crypto/openssl/i386/sha256-586.S optional ossl
crypto/openssl/i386/sha512-586.S optional ossl
diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c
index 35f19c9fbca7..229729c27c21 100644
--- a/sys/crypto/openssl/ossl.c
+++ b/sys/crypto/openssl/ossl.c
@@ -135,6 +135,8 @@ ossl_lookup_hash(const struct crypto_session_params *csp)
case CRYPTO_SHA2_512:
case CRYPTO_SHA2_512_HMAC:
return (&ossl_hash_sha512);
+ case CRYPTO_POLY1305:
+ return (&ossl_hash_poly1305);
default:
return (NULL);
}
@@ -159,14 +161,6 @@ ossl_probesession(device_t dev, const struct crypto_session_params *csp)
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
}
-static void
-ossl_setkey_hmac(struct ossl_session *s, const void *key, int klen)
-{
-
- hmac_init_ipad(s->hash.axf, key, klen, &s->hash.ictx);
- hmac_init_opad(s->hash.axf, key, klen, &s->hash.octx);
-}
-
static int
ossl_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp)
@@ -188,8 +182,16 @@ ossl_newsession(device_t dev, crypto_session_t cses,
} else {
if (csp->csp_auth_key != NULL) {
fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
- ossl_setkey_hmac(s, csp->csp_auth_key,
- csp->csp_auth_klen);
+ if (axf->Setkey != NULL) {
+ axf->Init(&s->hash.ictx);
+ axf->Setkey(&s->hash.ictx, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ } else {
+ hmac_init_ipad(axf, csp->csp_auth_key,
+ csp->csp_auth_klen, &s->hash.ictx);
+ hmac_init_opad(axf, csp->csp_auth_key,
+ csp->csp_auth_klen, &s->hash.octx);
+ }
fpu_kern_leave(curthread, NULL);
}
}
@@ -218,10 +220,18 @@ ossl_process(device_t dev, struct cryptop *crp, int hint)
fpu_entered = true;
}
- if (crp->crp_auth_key != NULL)
- ossl_setkey_hmac(s, crp->crp_auth_key, csp->csp_auth_klen);
-
- ctx = s->hash.ictx;
+ if (crp->crp_auth_key == NULL) {
+ ctx = s->hash.ictx;
+ } else {
+ if (axf->Setkey != NULL) {
+ axf->Init(&ctx);
+ axf->Setkey(&ctx, crp->crp_auth_key,
+ csp->csp_auth_klen);
+ } else {
+ hmac_init_ipad(axf, crp->crp_auth_key,
+ csp->csp_auth_klen, &ctx);
+ }
+ }
if (crp->crp_aad != NULL)
error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length);
@@ -238,8 +248,12 @@ ossl_process(device_t dev, struct cryptop *crp, int hint)
axf->Final(digest, &ctx);
- if (csp->csp_auth_klen != 0) {
- ctx = s->hash.octx;
+ if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) {
+ if (crp->crp_auth_key == NULL)
+ ctx = s->hash.octx;
+ else
+ hmac_init_opad(axf, crp->crp_auth_key,
+ csp->csp_auth_klen, &ctx);
axf->Update(&ctx, digest, axf->hashsize);
axf->Final(digest, &ctx);
}
diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h
index 533d497f1be3..55022b10f377 100644
--- a/sys/crypto/openssl/ossl.h
+++ b/sys/crypto/openssl/ossl.h
@@ -38,9 +38,10 @@ void ossl_cpuid(void);
/* Needs to be big enough to hold any hash context. */
struct ossl_hash_context {
- uint32_t dummy[54];
+ uint32_t dummy[61];
} __aligned(32);
+extern struct auth_hash ossl_hash_poly1305;
extern struct auth_hash ossl_hash_sha1;
extern struct auth_hash ossl_hash_sha224;
extern struct auth_hash ossl_hash_sha256;
diff --git a/sys/crypto/openssl/ossl_poly1305.c b/sys/crypto/openssl/ossl_poly1305.c
new file mode 100644
index 000000000000..8f8c5bc4b6e7
--- /dev/null
+++ b/sys/crypto/openssl/ossl_poly1305.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_poly1305.h>
+
+#define POLY1305_ASM
+
+/* From crypto/poly1305/poly1305.c */
+
+/* pick 32-bit unsigned integer in little endian order */
+static unsigned int U8TOU32(const unsigned char *p)
+{
+ return (((unsigned int)(p[0] & 0xff)) |
+ ((unsigned int)(p[1] & 0xff) << 8) |
+ ((unsigned int)(p[2] & 0xff) << 16) |
+ ((unsigned int)(p[3] & 0xff) << 24));
+}
+
+/*
+ * Implementations can be classified by amount of significant bits in
+ * words making up the multi-precision value, or in other words radix
+ * or base of numerical representation, e.g. base 2^64, base 2^32,
+ * base 2^26. Complementary characteristic is how wide is the result of
+ * multiplication of pair of digits, e.g. it would take 128 bits to
+ * accommodate multiplication result in base 2^64 case. These are used
+ * interchangeably. To describe implementation that is. But interface
+ * is designed to isolate this so that low-level primitives implemented
+ * in assembly can be self-contained/self-coherent.
+ */
+int poly1305_init(void *ctx, const unsigned char key[16], void *func);
+void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len,
+ unsigned int padbit);
+void poly1305_emit(void *ctx, unsigned char mac[16],
+ const unsigned int nonce[4]);
+
+static void Poly1305_Init(POLY1305 *ctx, const unsigned char key[32])
+{
+ ctx->nonce[0] = U8TOU32(&key[16]);
+ ctx->nonce[1] = U8TOU32(&key[20]);
+ ctx->nonce[2] = U8TOU32(&key[24]);
+ ctx->nonce[3] = U8TOU32(&key[28]);
+
+ /*
+ * Unlike reference poly1305_init assembly counterpart is expected
+ * to return a value: non-zero if it initializes ctx->func, and zero
+ * otherwise. Latter is to simplify assembly in cases when there no
+ * multiple code paths to switch between.
+ */
+ if (!poly1305_init(ctx->opaque, key, &ctx->func)) {
+ ctx->func.blocks = poly1305_blocks;
+ ctx->func.emit = poly1305_emit;
+ }
+
+ ctx->num = 0;
+
+}
+
+#ifdef POLY1305_ASM
+/*
+ * This "eclipses" poly1305_blocks and poly1305_emit, but it's
+ * conscious choice imposed by -Wshadow compiler warnings.
+ */
+# define poly1305_blocks (*poly1305_blocks_p)
+# define poly1305_emit (*poly1305_emit_p)
+#endif
+
+static void Poly1305_Update(POLY1305 *ctx, const unsigned char *inp, size_t len)
+{
+#ifdef POLY1305_ASM
+ /*
+ * As documented, poly1305_blocks is never called with input
+ * longer than single block and padbit argument set to 0. This
+ * property is fluently used in assembly modules to optimize
+ * padbit handling on loop boundary.
+ */
+ poly1305_blocks_f poly1305_blocks_p = ctx->func.blocks;
+#endif
+ size_t rem, num;
+
+ if ((num = ctx->num)) {
+ rem = POLY1305_BLOCK_SIZE - num;
+ if (len >= rem) {
+ memcpy(ctx->data + num, inp, rem);
+ poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1);
+ inp += rem;
+ len -= rem;
+ } else {
+ /* Still not enough data to process a block. */
+ memcpy(ctx->data + num, inp, len);
+ ctx->num = num + len;
+ return;
+ }
+ }
+
+ rem = len % POLY1305_BLOCK_SIZE;
+ len -= rem;
+
+ if (len >= POLY1305_BLOCK_SIZE) {
+ poly1305_blocks(ctx->opaque, inp, len, 1);
+ inp += len;
+ }
+
+ if (rem)
+ memcpy(ctx->data, inp, rem);
+
+ ctx->num = rem;
+}
+
+static void Poly1305_Final(POLY1305 *ctx, unsigned char mac[16])
+{
+#ifdef POLY1305_ASM
+ poly1305_blocks_f poly1305_blocks_p = ctx->func.blocks;
+ poly1305_emit_f poly1305_emit_p = ctx->func.emit;
+#endif
+ size_t num;
+
+ if ((num = ctx->num)) {
+ ctx->data[num++] = 1; /* pad bit */
+ while (num < POLY1305_BLOCK_SIZE)
+ ctx->data[num++] = 0;
+ poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0);
+ }
+
+ poly1305_emit(ctx->opaque, mac, ctx->nonce);
+
+ /* zero out the state */
+ OPENSSL_cleanse(ctx, sizeof(*ctx));
+}
+
+static void
+ossl_poly1305_init(void *vctx)
+{
+}
+
+static void
+ossl_poly1305_setkey(void *vctx, const uint8_t *key, u_int klen)
+{
+ MPASS(klen == 32);
+ Poly1305_Init(vctx, key);
+}
+
+static int
+ossl_poly1305_update(void *vctx, const void *buf, u_int len)
+{
+ Poly1305_Update(vctx, buf, len);
+ return (0);
+}
+
+static void
+ossl_poly1305_final(uint8_t *digest, void *vctx)
+{
+ Poly1305_Final(vctx, digest);
+}
+
+struct auth_hash ossl_hash_poly1305 = {
+ .type = CRYPTO_POLY1305,
+ .name = "OpenSSL-Poly1305",
+ .hashsize = POLY1305_HASH_LEN,
+ .ctxsize = sizeof(struct poly1305_context),
+ .blocksize = POLY1305_BLOCK_SIZE,
+ .Init = ossl_poly1305_init,
+ .Setkey = ossl_poly1305_setkey,
+ .Update = ossl_poly1305_update,
+ .Final = ossl_poly1305_final,
+};
+
+_Static_assert(sizeof(struct poly1305_context) <=
+ sizeof(struct ossl_hash_context), "ossl_hash_context too small");
diff --git a/sys/crypto/openssl/ossl_poly1305.h b/sys/crypto/openssl/ossl_poly1305.h
new file mode 100644
index 000000000000..d1b2db6d5cba
--- /dev/null
+++ b/sys/crypto/openssl/ossl_poly1305.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/* From include/crypto/poly1305.h */
+
+#define POLY1305_BLOCK_SIZE 16
+
+typedef struct poly1305_context POLY1305;
+
+/* From crypto/poly1305/poly1305_local.h */
+
+typedef void (*poly1305_blocks_f) (void *ctx, const unsigned char *inp,
+ size_t len, unsigned int padbit);
+typedef void (*poly1305_emit_f) (void *ctx, unsigned char mac[16],
+ const unsigned int nonce[4]);
+
+struct poly1305_context {
+ double opaque[24]; /* large enough to hold internal state, declared
+ * 'double' to ensure at least 64-bit invariant
+ * alignment across all platforms and
+ * configurations */
+ unsigned int nonce[4];
+ unsigned char data[POLY1305_BLOCK_SIZE];
+ size_t num;
+ struct {
+ poly1305_blocks_f blocks;
+ poly1305_emit_f emit;
+ } func;
+};
diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile
index 6fe8c5a6c812..2ddebefebd1a 100644
--- a/sys/modules/ossl/Makefile
+++ b/sys/modules/ossl/Makefile
@@ -8,24 +8,28 @@ SRCS= bus_if.h \
cryptodev_if.h \
device_if.h \
ossl.c \
+ ossl_poly1305.c \
ossl_sha1.c \
ossl_sha256.c \
ossl_sha512.c \
${SRCS.${MACHINE_CPUARCH}}
SRCS.aarch64= \
+ poly1305-armv8.S \
sha1-armv8.S \
sha256-armv8.S \
sha512-armv8.S \
ossl_aarch64.c
SRCS.amd64= \
+ poly1305-x86_64.S \
sha1-x86_64.S \
sha256-x86_64.S \
sha512-x86_64.S \
ossl_x86.c
SRCS.i386= \
+ poly1305-x86.S \
sha1-586.S \
sha256-586.S \
sha512-586.S \