1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/module.h>
4 #include <linux/slab.h>
5 #include <linux/crypto.h>
6 #include <crypto/internal/aead.h>
7 #include <crypto/internal/cipher.h>
8 #include <crypto/internal/skcipher.h>
9 #include <crypto/aes.h>
10 #include <crypto/sha1.h>
11 #include <crypto/sha2.h>
12 #include <crypto/hash.h>
13 #include <crypto/hmac.h>
14 #include <crypto/algapi.h>
15 #include <crypto/authenc.h>
16 #include <crypto/scatterwalk.h>
17 #include <crypto/xts.h>
18 #include <linux/dma-mapping.h>
19 #include "adf_accel_devices.h"
20 #include "adf_transport.h"
21 #include "adf_common_drv.h"
22 #include "qat_crypto.h"
23 #include "icp_qat_hw.h"
24 #include "icp_qat_fw.h"
25 #include "icp_qat_fw_la.h"
27 #define QAT_AES_HW_CONFIG_ENC(alg, mode) \
28 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
29 ICP_QAT_HW_CIPHER_NO_CONVERT, \
30 ICP_QAT_HW_CIPHER_ENCRYPT)
32 #define QAT_AES_HW_CONFIG_DEC(alg, mode) \
33 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
34 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
35 ICP_QAT_HW_CIPHER_DECRYPT)
37 #define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \
38 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
39 ICP_QAT_HW_CIPHER_NO_CONVERT, \
40 ICP_QAT_HW_CIPHER_DECRYPT)
42 #define HW_CAP_AES_V2(accel_dev) \
43 (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
44 ICP_ACCEL_CAPABILITIES_AES_V2)
46 static DEFINE_MUTEX(algs_lock);
47 static unsigned int active_devs;
55 struct qat_alg_buf_list {
59 struct qat_alg_buf bufers[];
60 } __packed __aligned(64);
62 /* Common content descriptor */
65 struct qat_enc { /* Encrypt content desc */
66 struct icp_qat_hw_cipher_algo_blk cipher;
67 struct icp_qat_hw_auth_algo_blk hash;
69 struct qat_dec { /* Decrypt content desc */
70 struct icp_qat_hw_auth_algo_blk hash;
71 struct icp_qat_hw_cipher_algo_blk cipher;
76 struct qat_alg_aead_ctx {
77 struct qat_alg_cd *enc_cd;
78 struct qat_alg_cd *dec_cd;
79 dma_addr_t enc_cd_paddr;
80 dma_addr_t dec_cd_paddr;
81 struct icp_qat_fw_la_bulk_req enc_fw_req;
82 struct icp_qat_fw_la_bulk_req dec_fw_req;
83 struct crypto_shash *hash_tfm;
84 enum icp_qat_hw_auth_algo qat_hash_alg;
85 struct qat_crypto_instance *inst;
87 struct sha1_state sha1;
88 struct sha256_state sha256;
89 struct sha512_state sha512;
91 char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
92 char opad[SHA512_BLOCK_SIZE];
95 struct qat_alg_skcipher_ctx {
96 struct icp_qat_hw_cipher_algo_blk *enc_cd;
97 struct icp_qat_hw_cipher_algo_blk *dec_cd;
98 dma_addr_t enc_cd_paddr;
99 dma_addr_t dec_cd_paddr;
100 struct icp_qat_fw_la_bulk_req enc_fw_req;
101 struct icp_qat_fw_la_bulk_req dec_fw_req;
102 struct qat_crypto_instance *inst;
103 struct crypto_skcipher *ftfm;
104 struct crypto_cipher *tweak;
109 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
111 switch (qat_hash_alg) {
112 case ICP_QAT_HW_AUTH_ALGO_SHA1:
113 return ICP_QAT_HW_SHA1_STATE1_SZ;
114 case ICP_QAT_HW_AUTH_ALGO_SHA256:
115 return ICP_QAT_HW_SHA256_STATE1_SZ;
116 case ICP_QAT_HW_AUTH_ALGO_SHA512:
117 return ICP_QAT_HW_SHA512_STATE1_SZ;
124 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
125 struct qat_alg_aead_ctx *ctx,
127 unsigned int auth_keylen)
129 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
130 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
131 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
132 __be32 *hash_state_out;
133 __be64 *hash512_state_out;
136 memset(ctx->ipad, 0, block_size);
137 memset(ctx->opad, 0, block_size);
138 shash->tfm = ctx->hash_tfm;
140 if (auth_keylen > block_size) {
141 int ret = crypto_shash_digest(shash, auth_key,
142 auth_keylen, ctx->ipad);
146 memcpy(ctx->opad, ctx->ipad, digest_size);
148 memcpy(ctx->ipad, auth_key, auth_keylen);
149 memcpy(ctx->opad, auth_key, auth_keylen);
152 for (i = 0; i < block_size; i++) {
153 char *ipad_ptr = ctx->ipad + i;
154 char *opad_ptr = ctx->opad + i;
155 *ipad_ptr ^= HMAC_IPAD_VALUE;
156 *opad_ptr ^= HMAC_OPAD_VALUE;
159 if (crypto_shash_init(shash))
162 if (crypto_shash_update(shash, ctx->ipad, block_size))
165 hash_state_out = (__be32 *)hash->sha.state1;
166 hash512_state_out = (__be64 *)hash_state_out;
168 switch (ctx->qat_hash_alg) {
169 case ICP_QAT_HW_AUTH_ALGO_SHA1:
170 if (crypto_shash_export(shash, &ctx->sha1))
172 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
173 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
175 case ICP_QAT_HW_AUTH_ALGO_SHA256:
176 if (crypto_shash_export(shash, &ctx->sha256))
178 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
179 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
181 case ICP_QAT_HW_AUTH_ALGO_SHA512:
182 if (crypto_shash_export(shash, &ctx->sha512))
184 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
185 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
191 if (crypto_shash_init(shash))
194 if (crypto_shash_update(shash, ctx->opad, block_size))
197 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
201 hash_state_out = (__be32 *)(hash->sha.state1 + offset);
202 hash512_state_out = (__be64 *)hash_state_out;
204 switch (ctx->qat_hash_alg) {
205 case ICP_QAT_HW_AUTH_ALGO_SHA1:
206 if (crypto_shash_export(shash, &ctx->sha1))
208 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
209 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
211 case ICP_QAT_HW_AUTH_ALGO_SHA256:
212 if (crypto_shash_export(shash, &ctx->sha256))
214 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
215 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
217 case ICP_QAT_HW_AUTH_ALGO_SHA512:
218 if (crypto_shash_export(shash, &ctx->sha512))
220 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
221 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
226 memzero_explicit(ctx->ipad, block_size);
227 memzero_explicit(ctx->opad, block_size);
231 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
234 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
235 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
236 header->comn_req_flags =
237 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
238 QAT_COMN_PTR_TYPE_SGL);
239 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
240 ICP_QAT_FW_LA_PARTIAL_NONE);
241 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
242 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
243 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
244 ICP_QAT_FW_LA_NO_PROTO);
245 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
246 ICP_QAT_FW_LA_NO_UPDATE_STATE);
249 static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
251 struct crypto_authenc_keys *keys,
254 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
255 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
256 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
257 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
258 struct icp_qat_hw_auth_algo_blk *hash =
259 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
260 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
261 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
262 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
263 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
264 void *ptr = &req_tmpl->cd_ctrl;
265 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
266 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
269 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
270 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
271 hash->sha.inner_setup.auth_config.config =
272 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
273 ctx->qat_hash_alg, digestsize);
274 hash->sha.inner_setup.auth_counter.counter =
275 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
277 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
281 qat_alg_init_common_hdr(header);
282 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
283 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
284 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
285 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
286 ICP_QAT_FW_LA_RET_AUTH_RES);
287 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
288 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
289 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
290 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
292 /* Cipher CD config setup */
293 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
294 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
295 cipher_cd_ctrl->cipher_cfg_offset = 0;
296 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
297 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
298 /* Auth CD config setup */
299 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
300 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
301 hash_cd_ctrl->inner_res_sz = digestsize;
302 hash_cd_ctrl->final_sz = digestsize;
304 switch (ctx->qat_hash_alg) {
305 case ICP_QAT_HW_AUTH_ALGO_SHA1:
306 hash_cd_ctrl->inner_state1_sz =
307 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
308 hash_cd_ctrl->inner_state2_sz =
309 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
311 case ICP_QAT_HW_AUTH_ALGO_SHA256:
312 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
313 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
315 case ICP_QAT_HW_AUTH_ALGO_SHA512:
316 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
317 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
322 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
323 ((sizeof(struct icp_qat_hw_auth_setup) +
324 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
325 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
326 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
330 static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
332 struct crypto_authenc_keys *keys,
335 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
336 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
337 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
338 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
339 struct icp_qat_hw_cipher_algo_blk *cipher =
340 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
341 sizeof(struct icp_qat_hw_auth_setup) +
342 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
343 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
344 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
345 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
346 void *ptr = &req_tmpl->cd_ctrl;
347 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
348 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
349 struct icp_qat_fw_la_auth_req_params *auth_param =
350 (struct icp_qat_fw_la_auth_req_params *)
351 ((char *)&req_tmpl->serv_specif_rqpars +
352 sizeof(struct icp_qat_fw_la_cipher_req_params));
355 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
356 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
357 hash->sha.inner_setup.auth_config.config =
358 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
361 hash->sha.inner_setup.auth_counter.counter =
362 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
364 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
368 qat_alg_init_common_hdr(header);
369 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
370 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
371 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
372 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
373 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
374 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
375 ICP_QAT_FW_LA_CMP_AUTH_RES);
376 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
377 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
379 /* Cipher CD config setup */
380 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
381 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
382 cipher_cd_ctrl->cipher_cfg_offset =
383 (sizeof(struct icp_qat_hw_auth_setup) +
384 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
385 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
386 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
388 /* Auth CD config setup */
389 hash_cd_ctrl->hash_cfg_offset = 0;
390 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
391 hash_cd_ctrl->inner_res_sz = digestsize;
392 hash_cd_ctrl->final_sz = digestsize;
394 switch (ctx->qat_hash_alg) {
395 case ICP_QAT_HW_AUTH_ALGO_SHA1:
396 hash_cd_ctrl->inner_state1_sz =
397 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
398 hash_cd_ctrl->inner_state2_sz =
399 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
401 case ICP_QAT_HW_AUTH_ALGO_SHA256:
402 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
403 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
405 case ICP_QAT_HW_AUTH_ALGO_SHA512:
406 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
407 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
413 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
414 ((sizeof(struct icp_qat_hw_auth_setup) +
415 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
416 auth_param->auth_res_sz = digestsize;
417 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
418 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
422 static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
423 struct icp_qat_fw_la_bulk_req *req,
424 struct icp_qat_hw_cipher_algo_blk *cd,
425 const u8 *key, unsigned int keylen)
427 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
428 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
429 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
430 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
431 int mode = ctx->mode;
433 qat_alg_init_common_hdr(header);
434 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
435 cd_pars->u.s.content_desc_params_sz =
436 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
438 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
439 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
440 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
442 /* Store both XTS keys in CD, only the first key is sent
443 * to the HW, the second key is used for tweak calculation
445 memcpy(cd->ucs_aes.key, key, keylen);
447 } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
448 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
449 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
450 keylen = round_up(keylen, 16);
451 memcpy(cd->ucs_aes.key, key, keylen);
453 memcpy(cd->aes.key, key, keylen);
456 /* Cipher CD config setup */
457 cd_ctrl->cipher_key_sz = keylen >> 3;
458 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
459 cd_ctrl->cipher_cfg_offset = 0;
460 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
461 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
464 static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
465 int alg, const u8 *key,
466 unsigned int keylen, int mode)
468 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
469 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
470 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
472 qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);
473 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
474 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
477 static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
480 struct crypto_aes_ctx aes_expanded;
484 aes_expandkey(&aes_expanded, key_forward, keylen);
485 if (keylen == AES_KEYSIZE_128) {
487 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
488 memcpy(key_reverse, key, AES_BLOCK_SIZE);
490 /* AES_KEYSIZE_256 */
492 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
493 memcpy(key_reverse, key, AES_BLOCK_SIZE);
494 memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
499 static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
500 int alg, const u8 *key,
501 unsigned int keylen, int mode)
503 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
504 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
505 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
506 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
508 qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);
509 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
511 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
512 /* Key reversing not supported, set no convert */
513 dec_cd->aes.cipher_config.val =
514 QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);
516 /* In-place key reversal */
517 qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2,
518 dec_cd->ucs_aes.key);
519 } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {
520 dec_cd->aes.cipher_config.val =
521 QAT_AES_HW_CONFIG_DEC(alg, mode);
523 dec_cd->aes.cipher_config.val =
524 QAT_AES_HW_CONFIG_ENC(alg, mode);
528 static int qat_alg_validate_key(int key_len, int *alg, int mode)
530 if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
532 case AES_KEYSIZE_128:
533 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
535 case AES_KEYSIZE_192:
536 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
538 case AES_KEYSIZE_256:
539 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
546 case AES_KEYSIZE_128 << 1:
547 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
549 case AES_KEYSIZE_256 << 1:
550 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
559 static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
560 unsigned int keylen, int mode)
562 struct crypto_authenc_keys keys;
565 if (crypto_authenc_extractkeys(&keys, key, keylen))
568 if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
571 if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
574 if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
577 memzero_explicit(&keys, sizeof(keys));
580 memzero_explicit(&keys, sizeof(keys));
583 memzero_explicit(&keys, sizeof(keys));
587 static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
594 if (qat_alg_validate_key(keylen, &alg, mode))
597 qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
598 qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
602 static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,
605 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
607 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
608 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
609 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
610 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
612 return qat_alg_aead_init_sessions(tfm, key, keylen,
613 ICP_QAT_HW_CIPHER_CBC_MODE);
616 static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,
619 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
620 struct qat_crypto_instance *inst = NULL;
621 int node = get_current_node();
625 inst = qat_crypto_get_instance_node(node);
628 dev = &GET_DEV(inst->accel_dev);
630 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
637 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
645 ret = qat_alg_aead_init_sessions(tfm, key, keylen,
646 ICP_QAT_HW_CIPHER_CBC_MODE);
653 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
654 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
655 ctx->dec_cd, ctx->dec_cd_paddr);
658 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
659 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
660 ctx->enc_cd, ctx->enc_cd_paddr);
664 qat_crypto_put_instance(inst);
668 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,
671 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
674 return qat_alg_aead_rekey(tfm, key, keylen);
676 return qat_alg_aead_newkey(tfm, key, keylen);
679 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
680 struct qat_crypto_request *qat_req)
682 struct device *dev = &GET_DEV(inst->accel_dev);
683 struct qat_alg_buf_list *bl = qat_req->buf.bl;
684 struct qat_alg_buf_list *blout = qat_req->buf.blout;
685 dma_addr_t blp = qat_req->buf.blp;
686 dma_addr_t blpout = qat_req->buf.bloutp;
687 size_t sz = qat_req->buf.sz;
688 size_t sz_out = qat_req->buf.sz_out;
691 for (i = 0; i < bl->num_bufs; i++)
692 dma_unmap_single(dev, bl->bufers[i].addr,
693 bl->bufers[i].len, DMA_BIDIRECTIONAL);
695 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
698 /* If out of place operation dma unmap only data */
699 int bufless = blout->num_bufs - blout->num_mapped_bufs;
701 for (i = bufless; i < blout->num_bufs; i++) {
702 dma_unmap_single(dev, blout->bufers[i].addr,
703 blout->bufers[i].len,
706 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
711 static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
712 struct scatterlist *sgl,
713 struct scatterlist *sglout,
714 struct qat_crypto_request *qat_req)
716 struct device *dev = &GET_DEV(inst->accel_dev);
718 int n = sg_nents(sgl);
719 struct qat_alg_buf_list *bufl;
720 struct qat_alg_buf_list *buflout = NULL;
721 dma_addr_t blp = DMA_MAPPING_ERROR;
722 dma_addr_t bloutp = DMA_MAPPING_ERROR;
723 struct scatterlist *sg;
724 size_t sz_out, sz = struct_size(bufl, bufers, n + 1);
729 bufl = kzalloc_node(sz, GFP_ATOMIC,
730 dev_to_node(&GET_DEV(inst->accel_dev)));
734 for_each_sg(sgl, sg, n, i)
735 bufl->bufers[i].addr = DMA_MAPPING_ERROR;
737 for_each_sg(sgl, sg, n, i) {
743 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
746 bufl->bufers[y].len = sg->length;
747 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
751 bufl->num_bufs = sg_nctr;
752 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
753 if (unlikely(dma_mapping_error(dev, blp)))
755 qat_req->buf.bl = bufl;
756 qat_req->buf.blp = blp;
757 qat_req->buf.sz = sz;
758 /* Handle out of place operation */
760 struct qat_alg_buf *bufers;
762 n = sg_nents(sglout);
763 sz_out = struct_size(buflout, bufers, n + 1);
765 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
766 dev_to_node(&GET_DEV(inst->accel_dev)));
767 if (unlikely(!buflout))
770 bufers = buflout->bufers;
771 for_each_sg(sglout, sg, n, i)
772 bufers[i].addr = DMA_MAPPING_ERROR;
774 for_each_sg(sglout, sg, n, i) {
780 bufers[y].addr = dma_map_single(dev, sg_virt(sg),
783 if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
785 bufers[y].len = sg->length;
788 buflout->num_bufs = sg_nctr;
789 buflout->num_mapped_bufs = sg_nctr;
790 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
791 if (unlikely(dma_mapping_error(dev, bloutp)))
793 qat_req->buf.blout = buflout;
794 qat_req->buf.bloutp = bloutp;
795 qat_req->buf.sz_out = sz_out;
797 /* Otherwise set the src and dst to the same address */
798 qat_req->buf.bloutp = qat_req->buf.blp;
799 qat_req->buf.sz_out = 0;
804 if (!dma_mapping_error(dev, bloutp))
805 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
807 n = sg_nents(sglout);
808 for (i = 0; i < n; i++)
809 if (!dma_mapping_error(dev, buflout->bufers[i].addr))
810 dma_unmap_single(dev, buflout->bufers[i].addr,
811 buflout->bufers[i].len,
816 if (!dma_mapping_error(dev, blp))
817 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
820 for (i = 0; i < n; i++)
821 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
822 dma_unmap_single(dev, bufl->bufers[i].addr,
828 dev_err(dev, "Failed to map buf for dma\n");
832 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
833 struct qat_crypto_request *qat_req)
835 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
836 struct qat_crypto_instance *inst = ctx->inst;
837 struct aead_request *areq = qat_req->aead_req;
838 u8 stat_filed = qat_resp->comn_resp.comn_status;
839 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
841 qat_alg_free_bufl(inst, qat_req);
842 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
844 areq->base.complete(&areq->base, res);
847 static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)
849 struct skcipher_request *sreq = qat_req->skcipher_req;
854 memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);
856 iv_lo = be64_to_cpu(qat_req->iv_lo);
857 iv_hi = be64_to_cpu(qat_req->iv_hi);
860 iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);
861 if (iv_lo < iv_lo_prev)
864 qat_req->iv_lo = cpu_to_be64(iv_lo);
865 qat_req->iv_hi = cpu_to_be64(iv_hi);
868 static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)
870 struct skcipher_request *sreq = qat_req->skcipher_req;
871 int offset = sreq->cryptlen - AES_BLOCK_SIZE;
872 struct scatterlist *sgl;
874 if (qat_req->encryption)
879 scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0);
882 static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
884 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
885 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
888 case ICP_QAT_HW_CIPHER_CTR_MODE:
889 qat_alg_update_iv_ctr_mode(qat_req);
891 case ICP_QAT_HW_CIPHER_CBC_MODE:
892 qat_alg_update_iv_cbc_mode(qat_req);
894 case ICP_QAT_HW_CIPHER_XTS_MODE:
897 dev_warn(dev, "Unsupported IV update for cipher mode %d\n",
902 static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
903 struct qat_crypto_request *qat_req)
905 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
906 struct qat_crypto_instance *inst = ctx->inst;
907 struct skcipher_request *sreq = qat_req->skcipher_req;
908 u8 stat_filed = qat_resp->comn_resp.comn_status;
909 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
911 qat_alg_free_bufl(inst, qat_req);
912 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
915 if (qat_req->encryption)
916 qat_alg_update_iv(qat_req);
918 memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
920 sreq->base.complete(&sreq->base, res);
923 void qat_alg_callback(void *resp)
925 struct icp_qat_fw_la_resp *qat_resp = resp;
926 struct qat_crypto_request *qat_req =
927 (void *)(__force long)qat_resp->opaque_data;
929 qat_req->cb(qat_resp, qat_req);
932 static int qat_alg_aead_dec(struct aead_request *areq)
934 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
935 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
936 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
937 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
938 struct icp_qat_fw_la_cipher_req_params *cipher_param;
939 struct icp_qat_fw_la_auth_req_params *auth_param;
940 struct icp_qat_fw_la_bulk_req *msg;
941 int digst_size = crypto_aead_authsize(aead_tfm);
945 cipher_len = areq->cryptlen - digst_size;
946 if (cipher_len % AES_BLOCK_SIZE != 0)
949 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
954 *msg = ctx->dec_fw_req;
955 qat_req->aead_ctx = ctx;
956 qat_req->aead_req = areq;
957 qat_req->cb = qat_aead_alg_callback;
958 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
959 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
960 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
961 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
962 cipher_param->cipher_length = cipher_len;
963 cipher_param->cipher_offset = areq->assoclen;
964 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
965 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
966 auth_param->auth_off = 0;
967 auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
969 ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
970 } while (ret == -EAGAIN && ctr++ < 10);
972 if (ret == -EAGAIN) {
973 qat_alg_free_bufl(ctx->inst, qat_req);
979 static int qat_alg_aead_enc(struct aead_request *areq)
981 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
982 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
983 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
984 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
985 struct icp_qat_fw_la_cipher_req_params *cipher_param;
986 struct icp_qat_fw_la_auth_req_params *auth_param;
987 struct icp_qat_fw_la_bulk_req *msg;
991 if (areq->cryptlen % AES_BLOCK_SIZE != 0)
994 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
999 *msg = ctx->enc_fw_req;
1000 qat_req->aead_ctx = ctx;
1001 qat_req->aead_req = areq;
1002 qat_req->cb = qat_aead_alg_callback;
1003 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1004 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1005 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1006 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1007 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
1009 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
1010 cipher_param->cipher_length = areq->cryptlen;
1011 cipher_param->cipher_offset = areq->assoclen;
1013 auth_param->auth_off = 0;
1014 auth_param->auth_len = areq->assoclen + areq->cryptlen;
1017 ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
1018 } while (ret == -EAGAIN && ctr++ < 10);
1020 if (ret == -EAGAIN) {
1021 qat_alg_free_bufl(ctx->inst, qat_req);
1024 return -EINPROGRESS;
1027 static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
1028 const u8 *key, unsigned int keylen,
1031 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
1032 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
1033 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
1034 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
1036 return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
1039 static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
1040 const u8 *key, unsigned int keylen,
1043 struct qat_crypto_instance *inst = NULL;
1045 int node = get_current_node();
1048 inst = qat_crypto_get_instance_node(node);
1051 dev = &GET_DEV(inst->accel_dev);
1053 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
1058 goto out_free_instance;
1060 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
1068 ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
1075 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
1076 dma_free_coherent(dev, sizeof(*ctx->dec_cd),
1077 ctx->dec_cd, ctx->dec_cd_paddr);
1080 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
1081 dma_free_coherent(dev, sizeof(*ctx->enc_cd),
1082 ctx->enc_cd, ctx->enc_cd_paddr);
1086 qat_crypto_put_instance(inst);
1090 static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
1091 const u8 *key, unsigned int keylen,
1094 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1099 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
1101 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
1104 static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
1105 const u8 *key, unsigned int keylen)
1107 return qat_alg_skcipher_setkey(tfm, key, keylen,
1108 ICP_QAT_HW_CIPHER_CBC_MODE);
1111 static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
1112 const u8 *key, unsigned int keylen)
1114 return qat_alg_skcipher_setkey(tfm, key, keylen,
1115 ICP_QAT_HW_CIPHER_CTR_MODE);
1118 static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
1119 const u8 *key, unsigned int keylen)
1121 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1124 ret = xts_verify_key(tfm, key, keylen);
1128 if (keylen >> 1 == AES_KEYSIZE_192) {
1129 ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen);
1133 ctx->fallback = true;
1138 ctx->fallback = false;
1140 ret = qat_alg_skcipher_setkey(tfm, key, keylen,
1141 ICP_QAT_HW_CIPHER_XTS_MODE);
1145 if (HW_CAP_AES_V2(ctx->inst->accel_dev))
1146 ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2),
1152 static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)
1154 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1155 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
1156 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
1157 u8 *iv = qat_req->skcipher_req->iv;
1159 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1161 if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)
1162 crypto_cipher_encrypt_one(ctx->tweak,
1163 (u8 *)cipher_param->u.cipher_IV_array,
1166 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
1169 static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
1171 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1172 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1173 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1174 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1175 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1176 struct icp_qat_fw_la_bulk_req *msg;
1179 if (req->cryptlen == 0)
1182 ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
1186 msg = &qat_req->req;
1187 *msg = ctx->enc_fw_req;
1188 qat_req->skcipher_ctx = ctx;
1189 qat_req->skcipher_req = req;
1190 qat_req->cb = qat_skcipher_alg_callback;
1191 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1192 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1193 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1194 qat_req->encryption = true;
1195 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1196 cipher_param->cipher_length = req->cryptlen;
1197 cipher_param->cipher_offset = 0;
1199 qat_alg_set_req_iv(qat_req);
1202 ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
1203 } while (ret == -EAGAIN && ctr++ < 10);
1205 if (ret == -EAGAIN) {
1206 qat_alg_free_bufl(ctx->inst, qat_req);
1209 return -EINPROGRESS;
1212 static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
1214 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1217 return qat_alg_skcipher_encrypt(req);
1220 static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)
1222 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1223 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1224 struct skcipher_request *nreq = skcipher_request_ctx(req);
1226 if (req->cryptlen < XTS_BLOCK_SIZE)
1229 if (ctx->fallback) {
1230 memcpy(nreq, req, sizeof(*req));
1231 skcipher_request_set_tfm(nreq, ctx->ftfm);
1232 return crypto_skcipher_encrypt(nreq);
1235 return qat_alg_skcipher_encrypt(req);
1238 static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1240 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1241 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1242 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1243 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1244 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1245 struct icp_qat_fw_la_bulk_req *msg;
1248 if (req->cryptlen == 0)
1251 ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
1255 msg = &qat_req->req;
1256 *msg = ctx->dec_fw_req;
1257 qat_req->skcipher_ctx = ctx;
1258 qat_req->skcipher_req = req;
1259 qat_req->cb = qat_skcipher_alg_callback;
1260 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1261 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1262 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1263 qat_req->encryption = false;
1264 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1265 cipher_param->cipher_length = req->cryptlen;
1266 cipher_param->cipher_offset = 0;
1268 qat_alg_set_req_iv(qat_req);
1269 qat_alg_update_iv(qat_req);
1272 ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
1273 } while (ret == -EAGAIN && ctr++ < 10);
1275 if (ret == -EAGAIN) {
1276 qat_alg_free_bufl(ctx->inst, qat_req);
1279 return -EINPROGRESS;
1282 static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1284 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1287 return qat_alg_skcipher_decrypt(req);
1290 static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
1292 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1293 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1294 struct skcipher_request *nreq = skcipher_request_ctx(req);
1296 if (req->cryptlen < XTS_BLOCK_SIZE)
1299 if (ctx->fallback) {
1300 memcpy(nreq, req, sizeof(*req));
1301 skcipher_request_set_tfm(nreq, ctx->ftfm);
1302 return crypto_skcipher_decrypt(nreq);
1305 return qat_alg_skcipher_decrypt(req);
1308 static int qat_alg_aead_init(struct crypto_aead *tfm,
1309 enum icp_qat_hw_auth_algo hash,
1310 const char *hash_name)
1312 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1314 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1315 if (IS_ERR(ctx->hash_tfm))
1316 return PTR_ERR(ctx->hash_tfm);
1317 ctx->qat_hash_alg = hash;
1318 crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1322 static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1324 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1327 static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1329 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1332 static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1334 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1337 static void qat_alg_aead_exit(struct crypto_aead *tfm)
1339 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1340 struct qat_crypto_instance *inst = ctx->inst;
1343 crypto_free_shash(ctx->hash_tfm);
1348 dev = &GET_DEV(inst->accel_dev);
1350 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1351 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1352 ctx->enc_cd, ctx->enc_cd_paddr);
1355 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1356 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1357 ctx->dec_cd, ctx->dec_cd_paddr);
1359 qat_crypto_put_instance(inst);
1362 static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1364 crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1368 static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
1370 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1373 ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0,
1374 CRYPTO_ALG_NEED_FALLBACK);
1375 if (IS_ERR(ctx->ftfm))
1376 return PTR_ERR(ctx->ftfm);
1378 ctx->tweak = crypto_alloc_cipher("aes", 0, 0);
1379 if (IS_ERR(ctx->tweak)) {
1380 crypto_free_skcipher(ctx->ftfm);
1381 return PTR_ERR(ctx->tweak);
1384 reqsize = max(sizeof(struct qat_crypto_request),
1385 sizeof(struct skcipher_request) +
1386 crypto_skcipher_reqsize(ctx->ftfm));
1387 crypto_skcipher_set_reqsize(tfm, reqsize);
1392 static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1394 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1395 struct qat_crypto_instance *inst = ctx->inst;
1401 dev = &GET_DEV(inst->accel_dev);
1403 memset(ctx->enc_cd, 0,
1404 sizeof(struct icp_qat_hw_cipher_algo_blk));
1405 dma_free_coherent(dev,
1406 sizeof(struct icp_qat_hw_cipher_algo_blk),
1407 ctx->enc_cd, ctx->enc_cd_paddr);
1410 memset(ctx->dec_cd, 0,
1411 sizeof(struct icp_qat_hw_cipher_algo_blk));
1412 dma_free_coherent(dev,
1413 sizeof(struct icp_qat_hw_cipher_algo_blk),
1414 ctx->dec_cd, ctx->dec_cd_paddr);
1416 qat_crypto_put_instance(inst);
1419 static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)
1421 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1424 crypto_free_skcipher(ctx->ftfm);
1427 crypto_free_cipher(ctx->tweak);
1429 qat_alg_skcipher_exit_tfm(tfm);
1432 static struct aead_alg qat_aeads[] = { {
1434 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1435 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1436 .cra_priority = 4001,
1437 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1438 .cra_blocksize = AES_BLOCK_SIZE,
1439 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1440 .cra_module = THIS_MODULE,
1442 .init = qat_alg_aead_sha1_init,
1443 .exit = qat_alg_aead_exit,
1444 .setkey = qat_alg_aead_setkey,
1445 .decrypt = qat_alg_aead_dec,
1446 .encrypt = qat_alg_aead_enc,
1447 .ivsize = AES_BLOCK_SIZE,
1448 .maxauthsize = SHA1_DIGEST_SIZE,
1451 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1452 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1453 .cra_priority = 4001,
1454 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1455 .cra_blocksize = AES_BLOCK_SIZE,
1456 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1457 .cra_module = THIS_MODULE,
1459 .init = qat_alg_aead_sha256_init,
1460 .exit = qat_alg_aead_exit,
1461 .setkey = qat_alg_aead_setkey,
1462 .decrypt = qat_alg_aead_dec,
1463 .encrypt = qat_alg_aead_enc,
1464 .ivsize = AES_BLOCK_SIZE,
1465 .maxauthsize = SHA256_DIGEST_SIZE,
1468 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1469 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1470 .cra_priority = 4001,
1471 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1472 .cra_blocksize = AES_BLOCK_SIZE,
1473 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1474 .cra_module = THIS_MODULE,
1476 .init = qat_alg_aead_sha512_init,
1477 .exit = qat_alg_aead_exit,
1478 .setkey = qat_alg_aead_setkey,
1479 .decrypt = qat_alg_aead_dec,
1480 .encrypt = qat_alg_aead_enc,
1481 .ivsize = AES_BLOCK_SIZE,
1482 .maxauthsize = SHA512_DIGEST_SIZE,
1485 static struct skcipher_alg qat_skciphers[] = { {
1486 .base.cra_name = "cbc(aes)",
1487 .base.cra_driver_name = "qat_aes_cbc",
1488 .base.cra_priority = 4001,
1489 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1490 .base.cra_blocksize = AES_BLOCK_SIZE,
1491 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1492 .base.cra_alignmask = 0,
1493 .base.cra_module = THIS_MODULE,
1495 .init = qat_alg_skcipher_init_tfm,
1496 .exit = qat_alg_skcipher_exit_tfm,
1497 .setkey = qat_alg_skcipher_cbc_setkey,
1498 .decrypt = qat_alg_skcipher_blk_decrypt,
1499 .encrypt = qat_alg_skcipher_blk_encrypt,
1500 .min_keysize = AES_MIN_KEY_SIZE,
1501 .max_keysize = AES_MAX_KEY_SIZE,
1502 .ivsize = AES_BLOCK_SIZE,
1504 .base.cra_name = "ctr(aes)",
1505 .base.cra_driver_name = "qat_aes_ctr",
1506 .base.cra_priority = 4001,
1507 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1508 .base.cra_blocksize = 1,
1509 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1510 .base.cra_alignmask = 0,
1511 .base.cra_module = THIS_MODULE,
1513 .init = qat_alg_skcipher_init_tfm,
1514 .exit = qat_alg_skcipher_exit_tfm,
1515 .setkey = qat_alg_skcipher_ctr_setkey,
1516 .decrypt = qat_alg_skcipher_decrypt,
1517 .encrypt = qat_alg_skcipher_encrypt,
1518 .min_keysize = AES_MIN_KEY_SIZE,
1519 .max_keysize = AES_MAX_KEY_SIZE,
1520 .ivsize = AES_BLOCK_SIZE,
1522 .base.cra_name = "xts(aes)",
1523 .base.cra_driver_name = "qat_aes_xts",
1524 .base.cra_priority = 4001,
1525 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1526 CRYPTO_ALG_ALLOCATES_MEMORY,
1527 .base.cra_blocksize = AES_BLOCK_SIZE,
1528 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1529 .base.cra_alignmask = 0,
1530 .base.cra_module = THIS_MODULE,
1532 .init = qat_alg_skcipher_init_xts_tfm,
1533 .exit = qat_alg_skcipher_exit_xts_tfm,
1534 .setkey = qat_alg_skcipher_xts_setkey,
1535 .decrypt = qat_alg_skcipher_xts_decrypt,
1536 .encrypt = qat_alg_skcipher_xts_encrypt,
1537 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1538 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1539 .ivsize = AES_BLOCK_SIZE,
1542 int qat_algs_register(void)
1546 mutex_lock(&algs_lock);
1547 if (++active_devs != 1)
1550 ret = crypto_register_skciphers(qat_skciphers,
1551 ARRAY_SIZE(qat_skciphers));
1555 ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1560 mutex_unlock(&algs_lock);
1564 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1568 void qat_algs_unregister(void)
1570 mutex_lock(&algs_lock);
1571 if (--active_devs != 0)
1574 crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1575 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1578 mutex_unlock(&algs_lock);