svn commit: r363418 - in stable/12: share/man/man4 sys/arm64/conf sys/conf sys/dev/safexcel sys/modules sys/modules/safexcel
Mark Johnston
markj at FreeBSD.org
Wed Jul 22 14:22:36 UTC 2020
Author: markj
Date: Wed Jul 22 14:22:35 2020
New Revision: 363418
URL: https://svnweb.freebsd.org/changeset/base/363418
Log:
MFC r363180, r363182, r363251:
Add a driver for the SafeXcel EIP-97.
Added:
stable/12/share/man/man4/safexcel.4
- copied unchanged from r363180, head/share/man/man4/safexcel.4
stable/12/sys/dev/safexcel/
- copied from r363180, head/sys/dev/safexcel/
stable/12/sys/modules/safexcel/
- copied from r363180, head/sys/modules/safexcel/
Modified:
stable/12/share/man/man4/Makefile
stable/12/sys/arm64/conf/GENERIC
stable/12/sys/conf/files.arm64
stable/12/sys/dev/safexcel/safexcel.c
stable/12/sys/dev/safexcel/safexcel_var.h
stable/12/sys/modules/Makefile
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/share/man/man4/Makefile
==============================================================================
--- stable/12/share/man/man4/Makefile Wed Jul 22 13:49:54 2020 (r363417)
+++ stable/12/share/man/man4/Makefile Wed Jul 22 14:22:35 2020 (r363418)
@@ -461,6 +461,7 @@ MAN= aac.4 \
rue.4 \
sa.4 \
safe.4 \
+ safexcel.4 \
sbp.4 \
sbp_targ.4 \
scc.4 \
Copied: stable/12/share/man/man4/safexcel.4 (from r363180, head/share/man/man4/safexcel.4)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ stable/12/share/man/man4/safexcel.4 Wed Jul 22 14:22:35 2020 (r363418, copy of r363180, head/share/man/man4/safexcel.4)
@@ -0,0 +1,84 @@
+.\"-
+.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd June 23, 2020
+.Dt SAFEXCEL 4
+.Os
+.Sh NAME
+.Nm safexcel
+.Nd Inside Secure SafeXcel-IP-97 security packet engine
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device crypto"
+.Cd "device cryptodev"
+.Cd "device safexcel"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+safexcel_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver implements
+.Xr crypto 4
+support for the cryptographic acceleration functions of the EIP-97 device
+found on some Marvell systems-on-chip.
+The driver can accelerate the following AES modes:
+.Pp
+.Bl -bullet -compact
+.It
+AES-CBC
+.It
+AES-CTR
+.It
+AES-XTS
+.It
+AES-GCM
+.It
+AES-CCM
+.El
+.Pp
+.Nm
+also implements SHA1 and SHA2 transforms, and can combine AES-CBC and AES-CTR
+with SHA1-HMAC and SHA2-HMAC for encrypt-then-authenticate operations.
+.Sh SEE ALSO
+.Xr crypto 4 ,
+.Xr ipsec 4 ,
+.Xr random 4 ,
+.Xr geli 8 ,
+.Xr crypto 9
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Fx 13.0 .
Modified: stable/12/sys/arm64/conf/GENERIC
==============================================================================
--- stable/12/sys/arm64/conf/GENERIC Wed Jul 22 13:49:54 2020 (r363417)
+++ stable/12/sys/arm64/conf/GENERIC Wed Jul 22 14:22:35 2020 (r363418)
@@ -238,6 +238,9 @@ device mv_ap806_gicp # Marvell AP806 GICP
device aw_rtc # Allwinner Real-time Clock
device mv_rtc # Marvell Real-time Clock
+# Crypto accelerators
+device safexcel # Inside Secure EIP-97
+
# Watchdog controllers
device aw_wdog # Allwinner Watchdog
Modified: stable/12/sys/conf/files.arm64
==============================================================================
--- stable/12/sys/conf/files.arm64 Wed Jul 22 13:49:54 2020 (r363417)
+++ stable/12/sys/conf/files.arm64 Wed Jul 22 14:22:35 2020 (r363418)
@@ -240,6 +240,7 @@ dev/pci/pci_host_generic_fdt.c optional pci fdt
dev/psci/psci.c standard
dev/psci/psci_arm64.S standard
dev/psci/smccc.c standard
+dev/safexcel/safexcel.c optional safexcel fdt
dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci fdt
dev/uart/uart_cpu_arm64.c optional uart
dev/uart/uart_dev_mu.c optional uart uart_mu
Modified: stable/12/sys/dev/safexcel/safexcel.c
==============================================================================
--- head/sys/dev/safexcel/safexcel.c Tue Jul 14 14:09:29 2020 (r363180)
+++ stable/12/sys/dev/safexcel/safexcel.c Wed Jul 22 14:22:35 2020 (r363418)
@@ -901,7 +901,8 @@ static int
safexcel_dma_init(struct safexcel_softc *sc)
{
struct safexcel_ring *ring;
- int error, i, size;
+ bus_size_t size;
+ int error, i;
for (i = 0; i < sc->sc_config.rings; i++) {
ring = &sc->sc_ring[i];
@@ -937,8 +938,9 @@ safexcel_dma_init(struct safexcel_softc *sc)
(struct safexcel_cmd_descr *)ring->cdr.dma.vaddr;
/* Allocate additional CDR token memory. */
- error = safexcel_dma_alloc_mem(sc, &ring->dma_atok,
- sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE);
+ size = (bus_size_t)sc->sc_config.atok_offset *
+ SAFEXCEL_RING_SIZE;
+ error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size);
if (error != 0) {
device_printf(sc->sc_dev,
"failed to allocate atoken DMA memory, error %d\n",
@@ -1144,6 +1146,12 @@ safexcel_probe(device_t dev)
return (BUS_PROBE_DEFAULT);
}
+static void
+safexcel_crypto_register(struct safexcel_softc *sc, int alg)
+{
+ (void)crypto_register(sc->sc_cid, alg, SAFEXCEL_MAX_REQUEST_SIZE, 0);
+}
+
static int
safexcel_attach(device_t dev)
{
@@ -1211,6 +1219,26 @@ safexcel_attach(device_t dev)
if (sc->sc_cid < 0)
goto err2;
+ safexcel_crypto_register(sc, CRYPTO_AES_CBC);
+ safexcel_crypto_register(sc, CRYPTO_AES_ICM);
+ safexcel_crypto_register(sc, CRYPTO_AES_XTS);
+ safexcel_crypto_register(sc, CRYPTO_AES_CCM_16);
+ safexcel_crypto_register(sc, CRYPTO_AES_CCM_CBC_MAC);
+ safexcel_crypto_register(sc, CRYPTO_AES_NIST_GCM_16);
+ safexcel_crypto_register(sc, CRYPTO_AES_128_NIST_GMAC);
+ safexcel_crypto_register(sc, CRYPTO_AES_192_NIST_GMAC);
+ safexcel_crypto_register(sc, CRYPTO_AES_256_NIST_GMAC);
+ safexcel_crypto_register(sc, CRYPTO_SHA1);
+ safexcel_crypto_register(sc, CRYPTO_SHA1_HMAC);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_224);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_224_HMAC);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_256);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_256_HMAC);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_384);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_384_HMAC);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_512);
+ safexcel_crypto_register(sc, CRYPTO_SHA2_512_HMAC);
+
return (0);
err2:
@@ -1256,7 +1284,6 @@ safexcel_detach(device_t dev)
static int
safexcel_set_context(struct safexcel_request *req)
{
- const struct crypto_session_params *csp;
struct cryptop *crp;
struct safexcel_context_record *ctx;
struct safexcel_session *sess;
@@ -1264,58 +1291,51 @@ safexcel_set_context(struct safexcel_request *req)
int off;
crp = req->crp;
- csp = crypto_get_params(crp->crp_session);
sess = req->sess;
ctx = (struct safexcel_context_record *)req->ctx.vaddr;
data = (uint8_t *)ctx->data;
- if (csp->csp_cipher_alg != 0) {
- if (crp->crp_cipher_key != NULL)
- memcpy(data, crp->crp_cipher_key, sess->klen);
+ if (req->enc != NULL) {
+ if ((req->enc->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
+ memcpy(data, req->enc->crd_key, sess->klen);
else
- memcpy(data, csp->csp_cipher_key, sess->klen);
+ memcpy(data, sess->key, sess->klen);
off = sess->klen;
- } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
- if (crp->crp_auth_key != NULL)
- memcpy(data, crp->crp_auth_key, sess->klen);
- else
- memcpy(data, csp->csp_auth_key, sess->klen);
- off = sess->klen;
} else {
off = 0;
}
- switch (csp->csp_cipher_alg) {
- case CRYPTO_AES_NIST_GCM_16:
- memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN);
- off += GMAC_BLOCK_LEN;
- break;
- case CRYPTO_AES_CCM_16:
- memcpy(data + off, sess->xcbc_key,
- AES_BLOCK_LEN * 2 + sess->klen);
- off += AES_BLOCK_LEN * 2 + sess->klen;
- break;
- case CRYPTO_AES_XTS:
- memcpy(data + off, sess->tweak_key, sess->klen);
- off += sess->klen;
- break;
+ if (req->enc != NULL) {
+ switch (req->enc->crd_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN);
+ off += GMAC_BLOCK_LEN;
+ break;
+ case CRYPTO_AES_CCM_16:
+ memcpy(data + off, sess->xcbc_key,
+ AES_BLOCK_LEN * 2 + sess->klen);
+ off += AES_BLOCK_LEN * 2 + sess->klen;
+ break;
+ case CRYPTO_AES_XTS:
+ memcpy(data + off, sess->tweak_key, sess->klen);
+ off += sess->klen;
+ break;
+ }
}
- switch (csp->csp_auth_alg) {
- case CRYPTO_AES_NIST_GMAC:
- memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN);
- off += GMAC_BLOCK_LEN;
- break;
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- memcpy(data + off, sess->hmac_ipad, sess->statelen);
- off += sess->statelen;
- memcpy(data + off, sess->hmac_opad, sess->statelen);
- off += sess->statelen;
- break;
+ if (req->mac != NULL) {
+ switch (req->mac->crd_alg) {
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ memcpy(data + off, sess->hmac_ipad, sess->statelen);
+ off += sess->statelen;
+ memcpy(data + off, sess->hmac_opad, sess->statelen);
+ off += sess->statelen;
+ break;
+ }
}
return (off);
@@ -1331,14 +1351,12 @@ static void
safexcel_set_command(struct safexcel_request *req,
struct safexcel_cmd_descr *cdesc)
{
- const struct crypto_session_params *csp;
struct cryptop *crp;
struct safexcel_session *sess;
uint32_t ctrl0, ctrl1, ctxr_len;
int alg;
crp = req->crp;
- csp = crypto_get_params(crp->crp_session);
sess = req->sess;
ctrl0 = sess->alg | sess->digest | sess->hash;
@@ -1347,13 +1365,14 @@ safexcel_set_command(struct safexcel_request *req,
ctxr_len = safexcel_set_context(req) / sizeof(uint32_t);
ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len);
- alg = csp->csp_cipher_alg;
- if (alg == 0)
- alg = csp->csp_auth_alg;
+ if (req->enc != NULL)
+ alg = req->enc->crd_alg;
+ else
+ alg = req->mac->crd_alg;
switch (alg) {
case CRYPTO_AES_CCM_16:
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) {
ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT |
SAFEXCEL_CONTROL0_KEY_EN;
} else {
@@ -1366,23 +1385,23 @@ safexcel_set_command(struct safexcel_request *req,
case CRYPTO_AES_CBC:
case CRYPTO_AES_ICM:
case CRYPTO_AES_XTS:
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) {
ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT |
SAFEXCEL_CONTROL0_KEY_EN;
- if (csp->csp_auth_alg != 0)
+ if (req->mac != NULL)
ctrl0 |=
SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT;
} else {
ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN |
SAFEXCEL_CONTROL0_KEY_EN;
- if (csp->csp_auth_alg != 0)
+ if (req->mac != NULL) {
ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN;
+ ctrl1 |= SAFEXCEL_CONTROL1_HASH_STORE;
+ }
}
break;
case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_NIST_GMAC:
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ||
- csp->csp_auth_alg != 0) {
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) {
ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT |
SAFEXCEL_CONTROL0_KEY_EN |
SAFEXCEL_CONTROL0_TYPE_HASH_OUT;
@@ -1391,7 +1410,8 @@ safexcel_set_command(struct safexcel_request *req,
SAFEXCEL_CONTROL0_KEY_EN |
SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN;
}
- if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
+ if (req->enc != NULL &&
+ req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16) {
ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE |
SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 |
SAFEXCEL_CONTROL1_IV2;
@@ -1454,11 +1474,28 @@ safexcel_instr_insert_digest(struct safexcel_instr **i
*instrp = instr + 1;
}
+static void
+safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, struct safexcel_request *req, int len)
+{
+ struct safexcel_instr *instr;
+
+ instr = *instrp;
+
+ instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
+ instr->length = len;
+ instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH |
+ SAFEXCEL_INSTR_STATUS_LAST_PACKET;
+ instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST |
+ SAFEXCEL_INSTR_DEST_OUTPUT;
+
+ *instrp = instr + 1;
+}
+
/*
* Retrieve and verify a digest.
*/
static void
-safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len)
+safexcel_instr_verify_digest(struct safexcel_instr **instrp, int len)
{
struct safexcel_instr *instr;
@@ -1507,13 +1544,9 @@ static void
safexcel_instr_cipher(struct safexcel_request *req,
struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc)
{
- struct cryptop *crp;
-
- crp = req->crp;
-
/* Insert the payload. */
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
+ instr->length = req->enc->crd_len;
instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET |
SAFEXCEL_INSTR_STATUS_LAST_HASH;
instr->instructions = SAFEXCEL_INSTR_INS_LAST |
@@ -1526,42 +1559,27 @@ static void
safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr,
struct safexcel_cmd_descr *cdesc)
{
- const struct crypto_session_params *csp;
- struct cryptop *crp;
struct safexcel_instr *start;
- crp = req->crp;
- csp = crypto_get_params(crp->crp_session);
start = instr;
- /* Insert the AAD. */
+ /* Encrypt any data left in the request. */
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_aad_length;
- instr->status = crp->crp_payload_length == 0 ?
- SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
+ instr->length = req->enc->crd_len;
+ instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
instr->instructions = SAFEXCEL_INSTR_INS_LAST |
- SAFEXCEL_INSTR_DEST_HASH;
+ SAFEXCEL_INSTR_DEST_CRYPTO |
+ SAFEXCEL_INSTR_DEST_HASH |
+ SAFEXCEL_INSTR_DEST_OUTPUT;
instr++;
- /* Encrypt any data left in the request. */
- if (crp->crp_payload_length > 0) {
- instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
- instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
- instr->instructions = SAFEXCEL_INSTR_INS_LAST |
- SAFEXCEL_INSTR_DEST_CRYPTO |
- SAFEXCEL_INSTR_DEST_HASH |
- SAFEXCEL_INSTR_DEST_OUTPUT;
- instr++;
- }
-
/*
* Compute the digest, or extract it and place it in the output stream.
*/
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0)
safexcel_instr_insert_digest(&instr, req->sess->digestlen);
else
- safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
+ safexcel_instr_retrieve_digest(&instr, req, req->sess->digestlen);
cdesc->additional_cdata_size = instr - start;
}
@@ -1577,7 +1595,7 @@ safexcel_instr_sha_hash(struct safexcel_request *req,
/* Pass the input data to the hash engine. */
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
+ instr->length = req->mac->crd_len;
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
instr++;
@@ -1624,7 +1642,7 @@ safexcel_instr_ccm(struct safexcel_request *req, struc
* Insert B0 and the AAD length into the input stream.
*/
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
- instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0);
+ instr->length = blen + (req->mac->crd_len > 0 ? 2 : 0);
instr->status = 0;
instr->instructions = SAFEXCEL_INSTR_DEST_HASH |
SAFEXCEL_INSTR_INSERT_IMMEDIATE;
@@ -1635,33 +1653,33 @@ safexcel_instr_ccm(struct safexcel_request *req, struc
b0[0] =
L - 1 | /* payload length size */
((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ |
- (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */;
+ (req->mac->crd_len > 0 ? 1 : 0) << 6 /* AAD present bit */;
memcpy(&b0[1], req->iv, AES_CCM_IV_LEN);
- b0[14] = crp->crp_payload_length >> 8;
- b0[15] = crp->crp_payload_length & 0xff;
+ b0[14] = req->enc->crd_len >> 8;
+ b0[15] = req->enc->crd_len & 0xff;
instr += blen / sizeof(*instr);
/* Insert the AAD length and data into the input stream. */
- if (crp->crp_aad_length > 0) {
+ if (req->mac->crd_len > 0) {
alenp = (uint8_t *)instr;
- alenp[0] = crp->crp_aad_length >> 8;
- alenp[1] = crp->crp_aad_length & 0xff;
+ alenp[0] = req->mac->crd_len >> 8;
+ alenp[1] = req->mac->crd_len & 0xff;
alenp[2] = 0;
alenp[3] = 0;
instr++;
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_aad_length;
+ instr->length = req->mac->crd_len;
instr->status = 0;
instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
instr++;
/* Insert zero padding. */
- aalign = (crp->crp_aad_length + 2) & (blen - 1);
+ aalign = (req->mac->crd_len + 2) & (blen - 1);
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
instr->length = aalign == 0 ? 0 :
- blen - ((crp->crp_aad_length + 2) & (blen - 1));
- instr->status = crp->crp_payload_length == 0 ?
+ blen - ((req->mac->crd_len + 2) & (blen - 1));
+ instr->status = req->enc->crd_len == 0 ?
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
instr++;
@@ -1670,10 +1688,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc
safexcel_instr_temp_aes_block(&instr);
/* Insert the cipher payload into the input stream. */
- if (crp->crp_payload_length > 0) {
+ if (req->enc->crd_len > 0) {
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
- instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ?
+ instr->length = req->enc->crd_len;
+ instr->status = (req->enc->crd_len & (blen - 1)) == 0 ?
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
SAFEXCEL_INSTR_DEST_CRYPTO |
@@ -1682,10 +1700,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc
instr++;
/* Insert zero padding. */
- if (crp->crp_payload_length & (blen - 1)) {
+ if (req->enc->crd_len & (blen - 1)) {
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
instr->length = blen -
- (crp->crp_payload_length & (blen - 1));
+ (req->enc->crd_len & (blen - 1));
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
instr++;
@@ -1695,10 +1713,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc
/*
* Compute the digest, or extract it and place it in the output stream.
*/
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0)
safexcel_instr_insert_digest(&instr, req->sess->digestlen);
else
- safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
+ safexcel_instr_verify_digest(&instr, req->sess->digestlen);
cdesc->additional_cdata_size = instr - start;
}
@@ -1718,8 +1736,8 @@ safexcel_instr_gcm(struct safexcel_request *req, struc
/* Insert the AAD into the input stream. */
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_aad_length;
- instr->status = crp->crp_payload_length == 0 ?
+ instr->length = req->mac->crd_len;
+ instr->status = req->enc->crd_len == 0 ?
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
instr->instructions = SAFEXCEL_INSTR_INS_LAST |
SAFEXCEL_INSTR_DEST_HASH;
@@ -1728,9 +1746,9 @@ safexcel_instr_gcm(struct safexcel_request *req, struc
safexcel_instr_temp_aes_block(&instr);
/* Insert the cipher payload into the input stream. */
- if (crp->crp_payload_length > 0) {
+ if (req->enc->crd_len > 0) {
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
+ instr->length = req->enc->crd_len;
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH |
@@ -1741,51 +1759,22 @@ safexcel_instr_gcm(struct safexcel_request *req, struc
/*
* Compute the digest, or extract it and place it in the output stream.
*/
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0)
safexcel_instr_insert_digest(&instr, req->sess->digestlen);
else
- safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
+ safexcel_instr_verify_digest(&instr, req->sess->digestlen);
cdesc->additional_cdata_size = instr - start;
}
static void
-safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr,
- struct safexcel_cmd_descr *cdesc)
-{
- struct cryptop *crp;
- struct safexcel_instr *start;
-
- memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN);
- cdesc->control_data.token[3] = htobe32(1);
-
- crp = req->crp;
- start = instr;
-
- instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
- instr->length = crp->crp_payload_length;
- instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
- instr->instructions = SAFEXCEL_INSTR_INS_LAST |
- SAFEXCEL_INSTR_DEST_HASH;
- instr++;
-
- safexcel_instr_temp_aes_block(&instr);
-
- safexcel_instr_insert_digest(&instr, req->sess->digestlen);
-
- cdesc->additional_cdata_size = instr - start;
-}
-
-static void
safexcel_set_token(struct safexcel_request *req)
{
- const struct crypto_session_params *csp;
struct safexcel_cmd_descr *cdesc;
struct safexcel_instr *instr;
struct safexcel_softc *sc;
int ringidx;
- csp = crypto_get_params(req->crp->crp_session);
cdesc = req->cdesc;
sc = req->sc;
ringidx = req->sess->ringidx;
@@ -1797,8 +1786,7 @@ safexcel_set_token(struct safexcel_request *req)
* in the token itself. Otherwise we use an additional token descriptor
* and the embedded instruction space is used to store the IV.
*/
- if (csp->csp_cipher_alg == 0 &&
- csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) {
+ if (req->enc == NULL) {
instr = (void *)cdesc->control_data.token;
} else {
instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr +
@@ -1807,30 +1795,32 @@ safexcel_set_token(struct safexcel_request *req)
cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD;
}
- switch (csp->csp_cipher_alg) {
- case CRYPTO_AES_NIST_GCM_16:
- safexcel_instr_gcm(req, instr, cdesc);
- break;
- case CRYPTO_AES_CCM_16:
- safexcel_instr_ccm(req, instr, cdesc);
- break;
- case CRYPTO_AES_XTS:
- memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN);
- memset(cdesc->control_data.token +
- AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN);
+ if (req->enc != NULL) {
+ switch (req->enc->crd_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ safexcel_instr_gcm(req, instr, cdesc);
+ break;
+ case CRYPTO_AES_CCM_16:
+ safexcel_instr_ccm(req, instr, cdesc);
+ break;
+ case CRYPTO_AES_XTS:
+ memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN);
+ memset(cdesc->control_data.token +
+ AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN);
- safexcel_instr_cipher(req, instr, cdesc);
- break;
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN);
- if (csp->csp_auth_alg != 0)
- safexcel_instr_eta(req, instr, cdesc);
- else
safexcel_instr_cipher(req, instr, cdesc);
- break;
- default:
- switch (csp->csp_auth_alg) {
+ break;
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
+ memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN);
+ if (req->mac != NULL)
+ safexcel_instr_eta(req, instr, cdesc);
+ else
+ safexcel_instr_cipher(req, instr, cdesc);
+ break;
+ }
+ } else {
+ switch (req->mac->crd_alg) {
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_224:
@@ -1843,13 +1833,9 @@ safexcel_set_token(struct safexcel_request *req)
case CRYPTO_SHA2_512_HMAC:
safexcel_instr_sha_hash(req, instr);
break;
- case CRYPTO_AES_NIST_GMAC:
- safexcel_instr_gmac(req, instr, cdesc);
- break;
default:
- panic("unhandled auth request %d", csp->csp_auth_alg);
+ panic("unhandled auth request %d", req->mac->crd_alg);
}
- break;
}
}
@@ -1896,8 +1882,8 @@ safexcel_cmd_descr_add(struct safexcel_ring *ring, boo
struct safexcel_cmd_descr *cdesc;
struct safexcel_cmd_descr_ring *cring;
- KASSERT(full_data_len <= SAFEXCEL_MAX_REQUEST_SIZE,
- ("%s: request length %u too long", __func__, full_data_len));
+ KASSERT(reqlen <= SAFEXCEL_MAX_REQUEST_SIZE,
+ ("%s: request length %u too long", __func__, reqlen));
mtx_assert(&ring->mtx, MA_OWNED);
cring = &ring->cdr;
@@ -1978,13 +1964,14 @@ safexcel_append_segs(bus_dma_segment_t *segs, int nseg
len -= seglen;
start = 0;
}
+
+ KASSERT(len == 0, ("%s: %d residual bytes", __func__, len));
}
static void
safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg,
int error)
{
- const struct crypto_session_params *csp;
struct cryptop *crp;
struct safexcel_cmd_descr *cdesc;
struct safexcel_request *req;
@@ -2002,7 +1989,6 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t
}
crp = req->crp;
- csp = crypto_get_params(crp->crp_session);
sess = req->sess;
ring = &req->sc->sc_ring[sess->ringidx];
@@ -2021,23 +2007,30 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t
*/
sglist_reset(ring->cmd_data);
sglist_reset(ring->res_data);
- if (crp->crp_aad_length != 0) {
+ if (req->mac != NULL && (req->enc == NULL ||
+ req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16 ||
+ req->enc->crd_alg == CRYPTO_AES_CCM_16)) {
safexcel_append_segs(segs, nseg, ring->cmd_data,
- crp->crp_aad_start, crp->crp_aad_length);
+ req->mac->crd_skip, req->mac->crd_len);
}
- safexcel_append_segs(segs, nseg, ring->cmd_data,
- crp->crp_payload_start, crp->crp_payload_length);
- if (csp->csp_cipher_alg != 0) {
+ if (req->enc != NULL) {
+ safexcel_append_segs(segs, nseg, ring->cmd_data,
+ req->enc->crd_skip, req->enc->crd_len);
safexcel_append_segs(segs, nseg, ring->res_data,
- crp->crp_payload_start, crp->crp_payload_length);
+ req->enc->crd_skip, req->enc->crd_len);
}
if (sess->digestlen > 0) {
- if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
+ if (req->enc == NULL ||
+ (req->enc->crd_flags & CRD_F_ENCRYPT) != 0)
+ safexcel_append_segs(segs, nseg, ring->res_data,
+ req->mac->crd_inject, sess->digestlen);
+ else if (req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16 ||
+ req->enc->crd_alg == CRYPTO_AES_CCM_16) {
safexcel_append_segs(segs, nseg, ring->cmd_data,
- crp->crp_digest_start, sess->digestlen);
+ req->mac->crd_inject, sess->digestlen);
} else {
safexcel_append_segs(segs, nseg, ring->res_data,
- crp->crp_digest_start, sess->digestlen);
+ req->mac->crd_inject, sess->digestlen);
}
}
@@ -2097,63 +2090,84 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t
req->rdescs = sg->sg_nseg;
}
+static void
+safexcel_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
+ bus_size_t mapsize __unused, int error)
+{
+ safexcel_create_chain_cb(arg, segs, nseg, error);
+}
+
+#include <sys/uio.h>
static int
safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req)
{
+ struct cryptop *crp;
int error;
req->error = 0;
req->cdescs = req->rdescs = 0;
+ crp = req->crp;
- error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp,
- safexcel_create_chain_cb, req, BUS_DMA_NOWAIT);
+ if ((crp->crp_flags & CRYPTO_F_IOV) != 0) {
+ error = bus_dmamap_load_uio(ring->data_dtag, req->dmap,
+ (struct uio *)crp->crp_buf, safexcel_create_chain_cb2,
+ req, BUS_DMA_NOWAIT);
+ } else if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) {
+ error = bus_dmamap_load_mbuf(ring->data_dtag, req->dmap,
+ (struct mbuf *)crp->crp_buf, safexcel_create_chain_cb2,
+ req, BUS_DMA_NOWAIT);
+ } else {
+ error = bus_dmamap_load(ring->data_dtag, req->dmap,
+ crp->crp_buf, crp->crp_ilen,
+ safexcel_create_chain_cb, req, BUS_DMA_NOWAIT);
+ }
if (error == 0)
req->dmap_loaded = true;
-
- if (req->error != 0)
+ else if (req->error != 0)
error = req->error;
-
return (error);
}
-static bool
-safexcel_probe_cipher(const struct crypto_session_params *csp)
-{
- switch (csp->csp_cipher_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- if (csp->csp_ivlen != AES_BLOCK_LEN)
- return (false);
- break;
- case CRYPTO_AES_XTS:
- if (csp->csp_ivlen != AES_XTS_IV_LEN)
- return (false);
- break;
- default:
- return (false);
- }
-
- return (true);
-}
-
/*
* Determine whether the driver can implement a session with the requested
* parameters.
*/
static int
-safexcel_probesession(device_t dev, const struct crypto_session_params *csp)
+safexcel_probesession(struct cryptoini *enc, struct cryptoini *mac)
{
- switch (csp->csp_mode) {
- case CSP_MODE_CIPHER:
- if (!safexcel_probe_cipher(csp))
- return (EINVAL);
- break;
- case CSP_MODE_DIGEST:
- switch (csp->csp_auth_alg) {
- case CRYPTO_AES_NIST_GMAC:
- if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ if (enc != NULL) {
+ switch (enc->cri_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (mac == NULL ||
+ (mac->cri_alg != CRYPTO_AES_128_NIST_GMAC &&
+ mac->cri_alg != CRYPTO_AES_192_NIST_GMAC &&
+ mac->cri_alg != CRYPTO_AES_256_NIST_GMAC))
return (EINVAL);
break;
+ case CRYPTO_AES_CCM_16:
+ if (mac == NULL ||
+ mac->cri_alg != CRYPTO_AES_CCM_CBC_MAC)
+ return (EINVAL);
+ break;
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
+ if (mac != NULL &&
+ mac->cri_alg != CRYPTO_SHA1_HMAC &&
+ mac->cri_alg != CRYPTO_SHA2_224_HMAC &&
+ mac->cri_alg != CRYPTO_SHA2_256_HMAC &&
+ mac->cri_alg != CRYPTO_SHA2_384_HMAC &&
+ mac->cri_alg != CRYPTO_SHA2_512_HMAC)
+ return (EINVAL);
+ break;
+ case CRYPTO_AES_XTS:
+ if (mac != NULL)
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ } else {
+ switch (mac->cri_alg) {
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_224:
@@ -2168,47 +2182,9 @@ safexcel_probesession(device_t dev, const struct crypt
default:
return (EINVAL);
}
- break;
- case CSP_MODE_AEAD:
- switch (csp->csp_cipher_alg) {
- case CRYPTO_AES_NIST_GCM_16:
- if (csp->csp_ivlen != AES_GCM_IV_LEN)
- return (EINVAL);
- break;
- case CRYPTO_AES_CCM_16:
- if (csp->csp_ivlen != AES_CCM_IV_LEN)
- return (EINVAL);
- break;
- default:
- return (EINVAL);
- }
- break;
- case CSP_MODE_ETA:
- if (!safexcel_probe_cipher(csp))
- return (EINVAL);
- switch (csp->csp_cipher_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- /*
- * The EIP-97 does not support combining AES-XTS with
- * hash operations.
- */
- if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC &&
- csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC &&
- csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC &&
- csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC &&
- csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC)
- return (EINVAL);
- break;
- default:
- return (EINVAL);
- }
- break;
- default:
- return (EINVAL);
}
- return (CRYPTODEV_PROBE_HARDWARE);
+ return (0);
}
/*
@@ -2286,20 +2262,64 @@ safexcel_setkey_hmac_digest(struct auth_hash *ahash, u
}
}
+static void
+safexcel_hmac_init_pad(struct auth_hash *axf, const char *key, int klen,
+ union authctx *auth_ctx, uint8_t padval)
+{
+ uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
+ u_int i;
+
+ memset(hmac_key, 0, sizeof(hmac_key));
+ if (klen > axf->blocksize) {
+ axf->Init(auth_ctx);
+ axf->Update(auth_ctx, key, klen);
+ axf->Final(hmac_key, auth_ctx);
+ klen = axf->hashsize;
+ } else {
+ memcpy(hmac_key, key, klen);
+ }
+
+ for (i = 0; i < axf->blocksize; i++)
+ hmac_key[i] ^= padval;
+
+ axf->Init(auth_ctx);
+ axf->Update(auth_ctx, hmac_key, axf->blocksize);
+ explicit_bzero(hmac_key, sizeof(hmac_key));
+}
+
/*
* Pre-compute the inner and outer digests used in the HMAC algorithm.
*/
static void
-safexcel_setkey_hmac(const struct crypto_session_params *csp,
- struct safexcel_session *sess, const uint8_t *key, int klen)
+safexcel_setkey_hmac(struct safexcel_session *sess, int alg, const uint8_t *key,
+ int klen)
{
union authctx ctx;
struct auth_hash *ahash;
- ahash = crypto_auth_hash(csp);
- hmac_init_ipad(ahash, key, klen, &ctx);
+ switch (alg) {
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list