1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2017 Marvell
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmapool.h>
12 #include <crypto/aead.h>
13 #include <crypto/aes.h>
14 #include <crypto/authenc.h>
15 #include <crypto/des.h>
16 #include <crypto/sha.h>
17 #include <crypto/skcipher.h>
18 #include <crypto/internal/aead.h>
19 #include <crypto/internal/skcipher.h>
23 enum safexcel_cipher_direction {
28 enum safexcel_cipher_alg {
34 struct safexcel_cipher_ctx {
35 struct safexcel_context base;
36 struct safexcel_crypto_priv *priv;
39 enum safexcel_cipher_alg alg;
45 /* All the below is AEAD specific */
48 u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
49 u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
52 struct safexcel_cipher_req {
53 enum safexcel_cipher_direction direction;
54 /* Number of result descriptors associated to the request */
59 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
60 struct safexcel_command_desc *cdesc,
63 struct safexcel_token *token;
66 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
69 offset = DES_BLOCK_SIZE / sizeof(u32);
70 memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE);
71 cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
74 offset = DES3_EDE_BLOCK_SIZE / sizeof(u32);
75 memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE);
76 cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
80 offset = AES_BLOCK_SIZE / sizeof(u32);
81 memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
82 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
87 token = (struct safexcel_token *)(cdesc->control_data.token + offset);
89 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
90 token[0].packet_length = length;
91 token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
92 EIP197_TOKEN_STAT_LAST_HASH;
93 token[0].instructions = EIP197_TOKEN_INS_LAST |
94 EIP197_TOKEN_INS_TYPE_CRYTO |
95 EIP197_TOKEN_INS_TYPE_OUTPUT;
98 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
99 struct safexcel_command_desc *cdesc,
100 enum safexcel_cipher_direction direction,
101 u32 cryptlen, u32 assoclen, u32 digestsize)
103 struct safexcel_token *token;
106 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
107 offset = AES_BLOCK_SIZE / sizeof(u32);
108 memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
110 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
113 token = (struct safexcel_token *)(cdesc->control_data.token + offset);
115 if (direction == SAFEXCEL_DECRYPT)
116 cryptlen -= digestsize;
118 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
119 token[0].packet_length = assoclen;
120 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH |
121 EIP197_TOKEN_INS_TYPE_OUTPUT;
123 token[1].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
124 token[1].packet_length = cryptlen;
125 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
126 token[1].instructions = EIP197_TOKEN_INS_LAST |
127 EIP197_TOKEN_INS_TYPE_CRYTO |
128 EIP197_TOKEN_INS_TYPE_HASH |
129 EIP197_TOKEN_INS_TYPE_OUTPUT;
131 if (direction == SAFEXCEL_ENCRYPT) {
132 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
133 token[2].packet_length = digestsize;
134 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
135 EIP197_TOKEN_STAT_LAST_PACKET;
136 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
137 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
139 token[2].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
140 token[2].packet_length = digestsize;
141 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
142 EIP197_TOKEN_STAT_LAST_PACKET;
143 token[2].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
145 token[3].opcode = EIP197_TOKEN_OPCODE_VERIFY;
146 token[3].packet_length = digestsize |
147 EIP197_TOKEN_HASH_RESULT_VERIFY;
148 token[3].stat = EIP197_TOKEN_STAT_LAST_HASH |
149 EIP197_TOKEN_STAT_LAST_PACKET;
150 token[3].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
154 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
155 const u8 *key, unsigned int len)
157 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
158 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
159 struct safexcel_crypto_priv *priv = ctx->priv;
160 struct crypto_aes_ctx aes;
163 ret = crypto_aes_expand_key(&aes, key, len);
165 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
169 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
170 for (i = 0; i < len / sizeof(u32); i++) {
171 if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
172 ctx->base.needs_inv = true;
178 for (i = 0; i < len / sizeof(u32); i++)
179 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
183 memzero_explicit(&aes, sizeof(aes));
187 static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
190 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
191 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
192 struct safexcel_ahash_export_state istate, ostate;
193 struct safexcel_crypto_priv *priv = ctx->priv;
194 struct crypto_authenc_keys keys;
196 if (crypto_authenc_extractkeys(&keys, key, len) != 0)
199 if (keys.enckeylen > sizeof(ctx->key))
203 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
204 memcmp(ctx->key, keys.enckey, keys.enckeylen))
205 ctx->base.needs_inv = true;
208 switch (ctx->hash_alg) {
209 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
210 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
211 keys.authkeylen, &istate, &ostate))
214 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
215 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
216 keys.authkeylen, &istate, &ostate))
219 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
220 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
221 keys.authkeylen, &istate, &ostate))
224 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
225 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
226 keys.authkeylen, &istate, &ostate))
229 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
230 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
231 keys.authkeylen, &istate, &ostate))
235 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
239 crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
240 CRYPTO_TFM_RES_MASK);
242 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
243 (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
244 memcmp(ctx->opad, ostate.state, ctx->state_sz)))
245 ctx->base.needs_inv = true;
247 /* Now copy the keys into the context */
248 memcpy(ctx->key, keys.enckey, keys.enckeylen);
249 ctx->key_len = keys.enckeylen;
251 memcpy(ctx->ipad, &istate.state, ctx->state_sz);
252 memcpy(ctx->opad, &ostate.state, ctx->state_sz);
254 memzero_explicit(&keys, sizeof(keys));
258 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
259 memzero_explicit(&keys, sizeof(keys));
263 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
264 struct crypto_async_request *async,
265 struct safexcel_cipher_req *sreq,
266 struct safexcel_command_desc *cdesc)
268 struct safexcel_crypto_priv *priv = ctx->priv;
272 if (sreq->direction == SAFEXCEL_ENCRYPT)
273 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
275 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
277 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
279 /* The decryption control type is a combination of the
280 * encryption type and CONTEXT_CONTROL_TYPE_NULL_IN, for all
283 if (sreq->direction == SAFEXCEL_DECRYPT)
284 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_NULL_IN;
287 cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
288 cdesc->control_data.control1 |= ctx->mode;
291 cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
294 if (ctx->alg == SAFEXCEL_DES) {
295 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_DES;
296 } else if (ctx->alg == SAFEXCEL_3DES) {
297 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_3DES;
298 } else if (ctx->alg == SAFEXCEL_AES) {
299 switch (ctx->key_len) {
300 case AES_KEYSIZE_128:
301 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
303 case AES_KEYSIZE_192:
304 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
306 case AES_KEYSIZE_256:
307 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
310 dev_err(priv->dev, "aes keysize not supported: %u\n",
316 ctrl_size = ctx->key_len / sizeof(u32);
318 /* Take in account the ipad+opad digests */
319 ctrl_size += ctx->state_sz / sizeof(u32) * 2;
320 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);
325 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
326 struct crypto_async_request *async,
327 struct scatterlist *src,
328 struct scatterlist *dst,
329 unsigned int cryptlen,
330 struct safexcel_cipher_req *sreq,
331 bool *should_complete, int *ret)
333 struct safexcel_result_desc *rdesc;
338 if (unlikely(!sreq->rdescs))
341 while (sreq->rdescs--) {
342 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
345 "cipher: result: could not retrieve the result descriptor\n");
346 *ret = PTR_ERR(rdesc);
351 *ret = safexcel_rdesc_check_errors(priv, rdesc);
356 safexcel_complete(priv, ring);
359 dma_unmap_sg(priv->dev, src,
360 sg_nents_for_len(src, cryptlen),
363 dma_unmap_sg(priv->dev, src,
364 sg_nents_for_len(src, cryptlen),
366 dma_unmap_sg(priv->dev, dst,
367 sg_nents_for_len(dst, cryptlen),
371 *should_complete = true;
376 static int safexcel_send_req(struct crypto_async_request *base, int ring,
377 struct safexcel_cipher_req *sreq,
378 struct scatterlist *src, struct scatterlist *dst,
379 unsigned int cryptlen, unsigned int assoclen,
380 unsigned int digestsize, u8 *iv, int *commands,
383 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
384 struct safexcel_crypto_priv *priv = ctx->priv;
385 struct safexcel_command_desc *cdesc;
386 struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
387 struct scatterlist *sg;
388 unsigned int totlen = cryptlen + assoclen;
389 int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
393 nr_src = dma_map_sg(priv->dev, src,
394 sg_nents_for_len(src, totlen),
400 nr_src = dma_map_sg(priv->dev, src,
401 sg_nents_for_len(src, totlen),
406 nr_dst = dma_map_sg(priv->dev, dst,
407 sg_nents_for_len(dst, totlen),
410 dma_unmap_sg(priv->dev, src,
411 sg_nents_for_len(src, totlen),
417 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
420 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
421 ctx->ipad, ctx->state_sz);
422 memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
423 ctx->opad, ctx->state_sz);
426 /* command descriptors */
427 for_each_sg(src, sg, nr_src, i) {
428 int len = sg_dma_len(sg);
430 /* Do not overflow the request */
431 if (queued - len < 0)
434 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
435 sg_dma_address(sg), len, totlen,
438 /* No space left in the command descriptor ring */
439 ret = PTR_ERR(cdesc);
445 safexcel_context_control(ctx, base, sreq, cdesc);
447 safexcel_aead_token(ctx, iv, cdesc,
448 sreq->direction, cryptlen,
449 assoclen, digestsize);
451 safexcel_skcipher_token(ctx, iv, cdesc,
460 /* result descriptors */
461 for_each_sg(dst, sg, nr_dst, i) {
462 bool first = !i, last = (i == nr_dst - 1);
463 u32 len = sg_dma_len(sg);
465 rdesc = safexcel_add_rdesc(priv, ring, first, last,
466 sg_dma_address(sg), len);
468 /* No space left in the result descriptor ring */
469 ret = PTR_ERR(rdesc);
477 safexcel_rdr_req_set(priv, ring, first_rdesc, base);
484 for (i = 0; i < n_rdesc; i++)
485 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
487 for (i = 0; i < n_cdesc; i++)
488 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
491 dma_unmap_sg(priv->dev, src,
492 sg_nents_for_len(src, totlen),
495 dma_unmap_sg(priv->dev, src,
496 sg_nents_for_len(src, totlen),
498 dma_unmap_sg(priv->dev, dst,
499 sg_nents_for_len(dst, totlen),
506 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
508 struct crypto_async_request *base,
509 struct safexcel_cipher_req *sreq,
510 bool *should_complete, int *ret)
512 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
513 struct safexcel_result_desc *rdesc;
514 int ndesc = 0, enq_ret;
518 if (unlikely(!sreq->rdescs))
521 while (sreq->rdescs--) {
522 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
525 "cipher: invalidate: could not retrieve the result descriptor\n");
526 *ret = PTR_ERR(rdesc);
531 *ret = safexcel_rdesc_check_errors(priv, rdesc);
536 safexcel_complete(priv, ring);
538 if (ctx->base.exit_inv) {
539 dma_pool_free(priv->context_pool, ctx->base.ctxr,
542 *should_complete = true;
547 ring = safexcel_select_ring(priv);
548 ctx->base.ring = ring;
550 spin_lock_bh(&priv->ring[ring].queue_lock);
551 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
552 spin_unlock_bh(&priv->ring[ring].queue_lock);
554 if (enq_ret != -EINPROGRESS)
557 queue_work(priv->ring[ring].workqueue,
558 &priv->ring[ring].work_data.work);
560 *should_complete = false;
565 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
567 struct crypto_async_request *async,
568 bool *should_complete, int *ret)
570 struct skcipher_request *req = skcipher_request_cast(async);
571 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
574 if (sreq->needs_inv) {
575 sreq->needs_inv = false;
576 err = safexcel_handle_inv_result(priv, ring, async, sreq,
577 should_complete, ret);
579 err = safexcel_handle_req_result(priv, ring, async, req->src,
580 req->dst, req->cryptlen, sreq,
581 should_complete, ret);
587 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
589 struct crypto_async_request *async,
590 bool *should_complete, int *ret)
592 struct aead_request *req = aead_request_cast(async);
593 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
594 struct safexcel_cipher_req *sreq = aead_request_ctx(req);
597 if (sreq->needs_inv) {
598 sreq->needs_inv = false;
599 err = safexcel_handle_inv_result(priv, ring, async, sreq,
600 should_complete, ret);
602 err = safexcel_handle_req_result(priv, ring, async, req->src,
604 req->cryptlen + crypto_aead_authsize(tfm),
605 sreq, should_complete, ret);
611 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
612 int ring, int *commands, int *results)
614 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
615 struct safexcel_crypto_priv *priv = ctx->priv;
618 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
628 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
629 int *commands, int *results)
631 struct skcipher_request *req = skcipher_request_cast(async);
632 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
633 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
634 struct safexcel_crypto_priv *priv = ctx->priv;
637 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
640 ret = safexcel_cipher_send_inv(async, ring, commands, results);
642 ret = safexcel_send_req(async, ring, sreq, req->src,
643 req->dst, req->cryptlen, 0, 0, req->iv,
646 sreq->rdescs = *results;
650 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
651 int *commands, int *results)
653 struct aead_request *req = aead_request_cast(async);
654 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
655 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
656 struct safexcel_cipher_req *sreq = aead_request_ctx(req);
657 struct safexcel_crypto_priv *priv = ctx->priv;
660 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
663 ret = safexcel_cipher_send_inv(async, ring, commands, results);
665 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
666 req->cryptlen, req->assoclen,
667 crypto_aead_authsize(tfm), req->iv,
669 sreq->rdescs = *results;
673 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
674 struct crypto_async_request *base,
675 struct safexcel_cipher_req *sreq,
676 struct safexcel_inv_result *result)
678 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
679 struct safexcel_crypto_priv *priv = ctx->priv;
680 int ring = ctx->base.ring;
682 init_completion(&result->completion);
684 ctx = crypto_tfm_ctx(base->tfm);
685 ctx->base.exit_inv = true;
686 sreq->needs_inv = true;
688 spin_lock_bh(&priv->ring[ring].queue_lock);
689 crypto_enqueue_request(&priv->ring[ring].queue, base);
690 spin_unlock_bh(&priv->ring[ring].queue_lock);
692 queue_work(priv->ring[ring].workqueue,
693 &priv->ring[ring].work_data.work);
695 wait_for_completion(&result->completion);
699 "cipher: sync: invalidate: completion error %d\n",
701 return result->error;
707 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
709 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
710 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
711 struct safexcel_inv_result result = {};
713 memset(req, 0, sizeof(struct skcipher_request));
715 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
716 safexcel_inv_complete, &result);
717 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
719 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
722 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
724 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
725 struct safexcel_cipher_req *sreq = aead_request_ctx(req);
726 struct safexcel_inv_result result = {};
728 memset(req, 0, sizeof(struct aead_request));
730 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
731 safexcel_inv_complete, &result);
732 aead_request_set_tfm(req, __crypto_aead_cast(tfm));
734 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
737 static int safexcel_queue_req(struct crypto_async_request *base,
738 struct safexcel_cipher_req *sreq,
739 enum safexcel_cipher_direction dir, u32 mode,
740 enum safexcel_cipher_alg alg)
742 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
743 struct safexcel_crypto_priv *priv = ctx->priv;
746 sreq->needs_inv = false;
747 sreq->direction = dir;
751 if (ctx->base.ctxr) {
752 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
753 sreq->needs_inv = true;
754 ctx->base.needs_inv = false;
757 ctx->base.ring = safexcel_select_ring(priv);
758 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
759 EIP197_GFP_FLAGS(*base),
760 &ctx->base.ctxr_dma);
765 ring = ctx->base.ring;
767 spin_lock_bh(&priv->ring[ring].queue_lock);
768 ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
769 spin_unlock_bh(&priv->ring[ring].queue_lock);
771 queue_work(priv->ring[ring].workqueue,
772 &priv->ring[ring].work_data.work);
777 static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
779 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
780 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
784 static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
786 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
787 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
791 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
793 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
794 struct safexcel_alg_template *tmpl =
795 container_of(tfm->__crt_alg, struct safexcel_alg_template,
798 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
799 sizeof(struct safexcel_cipher_req));
801 ctx->priv = tmpl->priv;
803 ctx->base.send = safexcel_skcipher_send;
804 ctx->base.handle_result = safexcel_skcipher_handle_result;
808 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
810 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
812 memzero_explicit(ctx->key, sizeof(ctx->key));
814 /* context not allocated, skip invalidation */
818 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
822 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
824 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
825 struct safexcel_crypto_priv *priv = ctx->priv;
828 if (safexcel_cipher_cra_exit(tfm))
831 if (priv->flags & EIP197_TRC_CACHE) {
832 ret = safexcel_skcipher_exit_inv(tfm);
834 dev_warn(priv->dev, "skcipher: invalidation error %d\n",
837 dma_pool_free(priv->context_pool, ctx->base.ctxr,
842 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
844 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
845 struct safexcel_crypto_priv *priv = ctx->priv;
848 if (safexcel_cipher_cra_exit(tfm))
851 if (priv->flags & EIP197_TRC_CACHE) {
852 ret = safexcel_aead_exit_inv(tfm);
854 dev_warn(priv->dev, "aead: invalidation error %d\n",
857 dma_pool_free(priv->context_pool, ctx->base.ctxr,
862 struct safexcel_alg_template safexcel_alg_ecb_aes = {
863 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
864 .engines = EIP97IES | EIP197B | EIP197D,
866 .setkey = safexcel_skcipher_aes_setkey,
867 .encrypt = safexcel_ecb_aes_encrypt,
868 .decrypt = safexcel_ecb_aes_decrypt,
869 .min_keysize = AES_MIN_KEY_SIZE,
870 .max_keysize = AES_MAX_KEY_SIZE,
872 .cra_name = "ecb(aes)",
873 .cra_driver_name = "safexcel-ecb-aes",
875 .cra_flags = CRYPTO_ALG_ASYNC |
876 CRYPTO_ALG_KERN_DRIVER_ONLY,
877 .cra_blocksize = AES_BLOCK_SIZE,
878 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
880 .cra_init = safexcel_skcipher_cra_init,
881 .cra_exit = safexcel_skcipher_cra_exit,
882 .cra_module = THIS_MODULE,
887 static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
889 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
890 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
894 static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
896 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
897 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
901 struct safexcel_alg_template safexcel_alg_cbc_aes = {
902 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
903 .engines = EIP97IES | EIP197B | EIP197D,
905 .setkey = safexcel_skcipher_aes_setkey,
906 .encrypt = safexcel_cbc_aes_encrypt,
907 .decrypt = safexcel_cbc_aes_decrypt,
908 .min_keysize = AES_MIN_KEY_SIZE,
909 .max_keysize = AES_MAX_KEY_SIZE,
910 .ivsize = AES_BLOCK_SIZE,
912 .cra_name = "cbc(aes)",
913 .cra_driver_name = "safexcel-cbc-aes",
915 .cra_flags = CRYPTO_ALG_ASYNC |
916 CRYPTO_ALG_KERN_DRIVER_ONLY,
917 .cra_blocksize = AES_BLOCK_SIZE,
918 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
920 .cra_init = safexcel_skcipher_cra_init,
921 .cra_exit = safexcel_skcipher_cra_exit,
922 .cra_module = THIS_MODULE,
927 static int safexcel_cbc_des_encrypt(struct skcipher_request *req)
929 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
930 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
934 static int safexcel_cbc_des_decrypt(struct skcipher_request *req)
936 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
937 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
941 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
944 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
945 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
946 u32 tmp[DES_EXPKEY_WORDS];
949 if (len != DES_KEY_SIZE) {
950 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
954 ret = des_ekey(tmp, key);
955 if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
956 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
960 /* if context exits and key changed, need to invalidate it */
961 if (ctx->base.ctxr_dma)
962 if (memcmp(ctx->key, key, len))
963 ctx->base.needs_inv = true;
965 memcpy(ctx->key, key, len);
971 struct safexcel_alg_template safexcel_alg_cbc_des = {
972 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
973 .engines = EIP97IES | EIP197B | EIP197D,
975 .setkey = safexcel_des_setkey,
976 .encrypt = safexcel_cbc_des_encrypt,
977 .decrypt = safexcel_cbc_des_decrypt,
978 .min_keysize = DES_KEY_SIZE,
979 .max_keysize = DES_KEY_SIZE,
980 .ivsize = DES_BLOCK_SIZE,
982 .cra_name = "cbc(des)",
983 .cra_driver_name = "safexcel-cbc-des",
985 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
986 CRYPTO_ALG_KERN_DRIVER_ONLY,
987 .cra_blocksize = DES_BLOCK_SIZE,
988 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
990 .cra_init = safexcel_skcipher_cra_init,
991 .cra_exit = safexcel_skcipher_cra_exit,
992 .cra_module = THIS_MODULE,
997 static int safexcel_ecb_des_encrypt(struct skcipher_request *req)
999 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1000 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1004 static int safexcel_ecb_des_decrypt(struct skcipher_request *req)
1006 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1007 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1011 struct safexcel_alg_template safexcel_alg_ecb_des = {
1012 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1013 .engines = EIP97IES | EIP197B | EIP197D,
1015 .setkey = safexcel_des_setkey,
1016 .encrypt = safexcel_ecb_des_encrypt,
1017 .decrypt = safexcel_ecb_des_decrypt,
1018 .min_keysize = DES_KEY_SIZE,
1019 .max_keysize = DES_KEY_SIZE,
1020 .ivsize = DES_BLOCK_SIZE,
1022 .cra_name = "ecb(des)",
1023 .cra_driver_name = "safexcel-ecb-des",
1024 .cra_priority = 300,
1025 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1026 CRYPTO_ALG_KERN_DRIVER_ONLY,
1027 .cra_blocksize = DES_BLOCK_SIZE,
1028 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1030 .cra_init = safexcel_skcipher_cra_init,
1031 .cra_exit = safexcel_skcipher_cra_exit,
1032 .cra_module = THIS_MODULE,
1037 static int safexcel_cbc_des3_ede_encrypt(struct skcipher_request *req)
1039 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1040 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
1044 static int safexcel_cbc_des3_ede_decrypt(struct skcipher_request *req)
1046 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1047 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
1051 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1052 const u8 *key, unsigned int len)
1054 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1055 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1057 if (len != DES3_EDE_KEY_SIZE) {
1058 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1062 /* if context exits and key changed, need to invalidate it */
1063 if (ctx->base.ctxr_dma) {
1064 if (memcmp(ctx->key, key, len))
1065 ctx->base.needs_inv = true;
1068 memcpy(ctx->key, key, len);
1075 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1076 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1077 .engines = EIP97IES | EIP197B | EIP197D,
1079 .setkey = safexcel_des3_ede_setkey,
1080 .encrypt = safexcel_cbc_des3_ede_encrypt,
1081 .decrypt = safexcel_cbc_des3_ede_decrypt,
1082 .min_keysize = DES3_EDE_KEY_SIZE,
1083 .max_keysize = DES3_EDE_KEY_SIZE,
1084 .ivsize = DES3_EDE_BLOCK_SIZE,
1086 .cra_name = "cbc(des3_ede)",
1087 .cra_driver_name = "safexcel-cbc-des3_ede",
1088 .cra_priority = 300,
1089 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1090 CRYPTO_ALG_KERN_DRIVER_ONLY,
1091 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1092 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1094 .cra_init = safexcel_skcipher_cra_init,
1095 .cra_exit = safexcel_skcipher_cra_exit,
1096 .cra_module = THIS_MODULE,
1101 static int safexcel_ecb_des3_ede_encrypt(struct skcipher_request *req)
1103 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1104 SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1108 static int safexcel_ecb_des3_ede_decrypt(struct skcipher_request *req)
1110 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1111 SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1115 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1116 .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1117 .engines = EIP97IES | EIP197B | EIP197D,
1119 .setkey = safexcel_des3_ede_setkey,
1120 .encrypt = safexcel_ecb_des3_ede_encrypt,
1121 .decrypt = safexcel_ecb_des3_ede_decrypt,
1122 .min_keysize = DES3_EDE_KEY_SIZE,
1123 .max_keysize = DES3_EDE_KEY_SIZE,
1124 .ivsize = DES3_EDE_BLOCK_SIZE,
1126 .cra_name = "ecb(des3_ede)",
1127 .cra_driver_name = "safexcel-ecb-des3_ede",
1128 .cra_priority = 300,
1129 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1130 CRYPTO_ALG_KERN_DRIVER_ONLY,
1131 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1132 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1134 .cra_init = safexcel_skcipher_cra_init,
1135 .cra_exit = safexcel_skcipher_cra_exit,
1136 .cra_module = THIS_MODULE,
1141 static int safexcel_aead_encrypt(struct aead_request *req)
1143 struct safexcel_cipher_req *creq = aead_request_ctx(req);
1145 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT,
1146 CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
1149 static int safexcel_aead_decrypt(struct aead_request *req)
1151 struct safexcel_cipher_req *creq = aead_request_ctx(req);
1153 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT,
1154 CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
1157 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1159 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1160 struct safexcel_alg_template *tmpl =
1161 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1164 crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1165 sizeof(struct safexcel_cipher_req));
1167 ctx->priv = tmpl->priv;
1170 ctx->base.send = safexcel_aead_send;
1171 ctx->base.handle_result = safexcel_aead_handle_result;
1175 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1177 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1179 safexcel_aead_cra_init(tfm);
1180 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1181 ctx->state_sz = SHA1_DIGEST_SIZE;
1185 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1186 .type = SAFEXCEL_ALG_TYPE_AEAD,
1187 .engines = EIP97IES | EIP197B | EIP197D,
1189 .setkey = safexcel_aead_aes_setkey,
1190 .encrypt = safexcel_aead_encrypt,
1191 .decrypt = safexcel_aead_decrypt,
1192 .ivsize = AES_BLOCK_SIZE,
1193 .maxauthsize = SHA1_DIGEST_SIZE,
1195 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1196 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1197 .cra_priority = 300,
1198 .cra_flags = CRYPTO_ALG_ASYNC |
1199 CRYPTO_ALG_KERN_DRIVER_ONLY,
1200 .cra_blocksize = AES_BLOCK_SIZE,
1201 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1203 .cra_init = safexcel_aead_sha1_cra_init,
1204 .cra_exit = safexcel_aead_cra_exit,
1205 .cra_module = THIS_MODULE,
1210 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1212 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1214 safexcel_aead_cra_init(tfm);
1215 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1216 ctx->state_sz = SHA256_DIGEST_SIZE;
1220 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1221 .type = SAFEXCEL_ALG_TYPE_AEAD,
1222 .engines = EIP97IES | EIP197B | EIP197D,
1224 .setkey = safexcel_aead_aes_setkey,
1225 .encrypt = safexcel_aead_encrypt,
1226 .decrypt = safexcel_aead_decrypt,
1227 .ivsize = AES_BLOCK_SIZE,
1228 .maxauthsize = SHA256_DIGEST_SIZE,
1230 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1231 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1232 .cra_priority = 300,
1233 .cra_flags = CRYPTO_ALG_ASYNC |
1234 CRYPTO_ALG_KERN_DRIVER_ONLY,
1235 .cra_blocksize = AES_BLOCK_SIZE,
1236 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1238 .cra_init = safexcel_aead_sha256_cra_init,
1239 .cra_exit = safexcel_aead_cra_exit,
1240 .cra_module = THIS_MODULE,
1245 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1247 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1249 safexcel_aead_cra_init(tfm);
1250 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1251 ctx->state_sz = SHA256_DIGEST_SIZE;
1255 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1256 .type = SAFEXCEL_ALG_TYPE_AEAD,
1257 .engines = EIP97IES | EIP197B | EIP197D,
1259 .setkey = safexcel_aead_aes_setkey,
1260 .encrypt = safexcel_aead_encrypt,
1261 .decrypt = safexcel_aead_decrypt,
1262 .ivsize = AES_BLOCK_SIZE,
1263 .maxauthsize = SHA224_DIGEST_SIZE,
1265 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1266 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1267 .cra_priority = 300,
1268 .cra_flags = CRYPTO_ALG_ASYNC |
1269 CRYPTO_ALG_KERN_DRIVER_ONLY,
1270 .cra_blocksize = AES_BLOCK_SIZE,
1271 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1273 .cra_init = safexcel_aead_sha224_cra_init,
1274 .cra_exit = safexcel_aead_cra_exit,
1275 .cra_module = THIS_MODULE,
1280 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1282 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1284 safexcel_aead_cra_init(tfm);
1285 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1286 ctx->state_sz = SHA512_DIGEST_SIZE;
1290 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1291 .type = SAFEXCEL_ALG_TYPE_AEAD,
1292 .engines = EIP97IES | EIP197B | EIP197D,
1294 .setkey = safexcel_aead_aes_setkey,
1295 .encrypt = safexcel_aead_encrypt,
1296 .decrypt = safexcel_aead_decrypt,
1297 .ivsize = AES_BLOCK_SIZE,
1298 .maxauthsize = SHA512_DIGEST_SIZE,
1300 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1301 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1302 .cra_priority = 300,
1303 .cra_flags = CRYPTO_ALG_ASYNC |
1304 CRYPTO_ALG_KERN_DRIVER_ONLY,
1305 .cra_blocksize = AES_BLOCK_SIZE,
1306 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1308 .cra_init = safexcel_aead_sha512_cra_init,
1309 .cra_exit = safexcel_aead_cra_exit,
1310 .cra_module = THIS_MODULE,
1315 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1317 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1319 safexcel_aead_cra_init(tfm);
1320 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1321 ctx->state_sz = SHA512_DIGEST_SIZE;
1325 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1326 .type = SAFEXCEL_ALG_TYPE_AEAD,
1327 .engines = EIP97IES | EIP197B | EIP197D,
1329 .setkey = safexcel_aead_aes_setkey,
1330 .encrypt = safexcel_aead_encrypt,
1331 .decrypt = safexcel_aead_decrypt,
1332 .ivsize = AES_BLOCK_SIZE,
1333 .maxauthsize = SHA384_DIGEST_SIZE,
1335 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1336 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1337 .cra_priority = 300,
1338 .cra_flags = CRYPTO_ALG_ASYNC |
1339 CRYPTO_ALG_KERN_DRIVER_ONLY,
1340 .cra_blocksize = AES_BLOCK_SIZE,
1341 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1343 .cra_init = safexcel_aead_sha384_cra_init,
1344 .cra_exit = safexcel_aead_cra_exit,
1345 .cra_module = THIS_MODULE,