1 /* SPDX-License-Identifier: GPL-2.0 */
5 #include <linux/dma-mapping.h>
6 #include <crypto/aes.h>
8 #include "nitrox_dev.h"
10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
13 typedef void (*sereq_completion_t)(void *req, int err);
16 * struct gphdr - General purpose Header
17 * @param0: first parameter.
18 * @param1: second parameter.
19 * @param2: third parameter.
20 * @param3: fourth parameter.
22 * Params tell the iv and enc/dec data offsets.
32 * struct se_req_ctrl - SE request information.
33 * @arg: Minor number of the opcode
34 * @ctxc: Context control.
35 * @unca: Uncertainity enabled.
36 * @info: Additional information for SE cores.
37 * @ctxl: Context length in bytes.
38 * @uddl: User defined data length
57 * struct se_crypto_request - SE crypto request structure.
58 * @opcode: Request opcode (enc/dec)
59 * @flags: flags from crypto subsystem
60 * @ctx_handle: Crypto context handle.
62 * @ctrl: Request Information.
64 * @comp: completion address
68 struct se_crypto_request {
75 union se_req_ctrl ctrl;
79 struct scatterlist *src;
80 struct scatterlist *dst;
84 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
90 /* IV from Input data */
91 #define IV_FROM_DPTR 1
94 * cipher opcodes for firmware
125 * struct crypto_keys - Crypto keys
126 * @key: Encryption key or KEY1 for AES-XTS
127 * @iv: Encryption IV or Tweak for AES-XTS
131 u8 key[AES_MAX_KEY_SIZE];
132 u8 key1[AES_MAX_KEY_SIZE];
134 u8 iv[AES_BLOCK_SIZE];
138 * struct auth_keys - Authentication keys
139 * @ipad: IPAD or KEY2 for AES-XTS
140 * @opad: OPAD or AUTH KEY if auth_input_type = 1
154 #if defined(__BIG_ENDIAN_BITFIELD)
160 u64 reserved_49_51 : 3;
161 u64 auth_input_type: 1;
163 u64 reserved_0_39 : 40;
165 u64 reserved_0_39 : 40;
167 u64 auth_input_type: 1;
168 u64 reserved_49_51 : 3;
178 * struct flexi_crypto_context - Crypto context
179 * @cipher_type: Encryption cipher type
180 * @aes_keylen: AES key length
181 * @iv_source: Encryption IV source
182 * @hash_type: Authentication type
183 * @auth_input_type: Authentication input type
184 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
185 * 0 - Authentication OPAD/IPAD
186 * @mac_len: mac length
187 * @crypto: Crypto keys
188 * @auth: Authentication keys
190 struct flexi_crypto_context {
191 union fc_ctx_flags flags;
192 struct crypto_keys crypto;
193 struct auth_keys auth;
196 struct crypto_ctx_hdr {
197 struct dma_pool *pool;
202 struct nitrox_crypto_ctx {
203 struct nitrox_device *ndev;
206 struct flexi_crypto_context *fctx;
208 struct crypto_ctx_hdr *chdr;
209 sereq_completion_t callback;
212 struct nitrox_kcrypt_request {
213 struct se_crypto_request creq;
220 * struct nitrox_aead_rctx - AEAD request context
221 * @nkreq: Base request context
222 * @cryptlen: Encryption/Decryption data length
223 * @assoclen: AAD length
224 * @srclen: Input buffer length
225 * @dstlen: Output buffer length
227 * @ivsize: IV data length
228 * @flags: AEAD req flags
229 * @ctx_handle: Device context handle
230 * @src: Source sglist
231 * @dst: Destination sglist
232 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
234 struct nitrox_aead_rctx {
235 struct nitrox_kcrypt_request nkreq;
236 unsigned int cryptlen;
237 unsigned int assoclen;
244 struct scatterlist *src;
245 struct scatterlist *dst;
250 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
251 * @base: AEAD request context
252 * @src: Source sglist
253 * @dst: Destination sglist
256 struct nitrox_rfc4106_rctx {
257 struct nitrox_aead_rctx base;
258 struct scatterlist src[3];
259 struct scatterlist dst[3];
264 * struct pkt_instr_hdr - Packet Instruction Header
266 * When [G] is set and [GSZ] != 0, the instruction is
267 * indirect gather instruction.
268 * When [G] is set and [GSZ] = 0, the instruction is
269 * direct gather instruction.
270 * @gsz: Number of pointers in the indirect gather list
271 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
272 * and adds them to the packet after the pkt_instr_hdr but before any UDD
273 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
275 * @fsz: The number of front data bytes directly included in the
277 * @tlen: The length of the input packet in bytes, include:
279 * - Inline context bytes if any,
281 * - packet payload bytes
283 union pkt_instr_hdr {
287 #if defined(__BIG_ENDIAN_BITFIELD)
312 * struct pkt_hdr - Packet Input Header
313 * @opcode: Request opcode (Major)
314 * @arg: Request opcode (Minor)
315 * @ctxc: Context control.
316 * @unca: When set [UNC] is the uncertainty count for an input packet.
317 * The hardware uses uncertainty counts to predict
318 * output buffer use and avoid deadlock.
319 * @info: Not used by input hardware. Available for use
320 * during SE processing.
321 * @destport: The expected destination port/ring/channel for the packet.
322 * @unc: Uncertainty count for an input packet.
323 * @grp: SE group that will process the input packet.
324 * @ctxl: Context Length in 64-bit words.
325 * @uddl: User-defined data (UDD) length in bytes.
326 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
332 #if defined(__BIG_ENDIAN_BITFIELD)
366 * struct slc_store_info - Solicited Paceket Output Store Information.
367 * @ssz: The number of scatterlist pointers for the solicited output port
369 * @rptr: The result pointer for the solicited output port packet.
370 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
371 * host that is large enough to hold the entire output packet.
372 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
373 * sglist components at [RPTR] on the remote host.
375 union slc_store_info {
379 #if defined(__BIG_ENDIAN_BITFIELD)
393 * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
394 * @dptr0 : Input pointer points to buffer in remote host.
395 * @ih: Packet Instruction Header (8 bytes)
396 * @irh: Packet Input Header (16 bytes)
397 * @slc: Solicited Packet Output Store Information (16 bytes)
400 * 64-Byte Instruction Format
402 struct nps_pkt_instr {
404 union pkt_instr_hdr ih;
406 union slc_store_info slc;
411 * struct aqmq_command_s - The 32 byte command for AE processing.
412 * @opcode: Request opcode
413 * @param1: Request control parameter 1
414 * @param2: Request control parameter 2
415 * @dlen: Input length
416 * @dptr: Input pointer points to buffer in remote host
417 * @rptr: Result pointer points to buffer in remote host
418 * @grp: AQM Group (0..7)
419 * @cptr: Context pointer
421 struct aqmq_command_s {
430 #if defined(__BIG_ENDIAN_BITFIELD)
441 * struct ctx_hdr - Book keeping data about the crypto context
442 * @pool: Pool used to allocate crypto context
443 * @dma: Base DMA address of the crypto context
444 * @ctx_dma: Actual usable crypto context for NITROX
447 struct dma_pool *pool;
453 * struct sglist_component - SG list component format
454 * @len0: The number of bytes at [PTR0] on the remote host.
455 * @len1: The number of bytes at [PTR1] on the remote host.
456 * @len2: The number of bytes at [PTR2] on the remote host.
457 * @len3: The number of bytes at [PTR3] on the remote host.
458 * @dma0: First pointer point to buffer in remote host.
459 * @dma1: Second pointer point to buffer in remote host.
460 * @dma2: Third pointer point to buffer in remote host.
461 * @dma3: Fourth pointer point to buffer in remote host.
463 struct nitrox_sgcomp {
469 * strutct nitrox_sgtable - SG list information
470 * @sgmap_cnt: Number of buffers mapped
471 * @total_bytes: Total bytes in sglist.
472 * @sgcomp_len: Total sglist components length.
473 * @sgcomp_dma: DMA address of sglist component.
474 * @sg: crypto request buffer.
475 * @sgcomp: sglist component for NITROX.
477 struct nitrox_sgtable {
481 dma_addr_t sgcomp_dma;
482 struct scatterlist *sg;
483 struct nitrox_sgcomp *sgcomp;
486 /* Response Header Length */
488 /* Completion bytes Length */
496 typedef void (*completion_t)(void *arg, int err);
499 * struct nitrox_softreq - Represents the NIROX Request.
500 * @response: response list entry
501 * @backlog: Backlog list entry
502 * @ndev: Device used to submit the request
503 * @cmdq: Command queue for submission
504 * @resp: Response headers
505 * @instr: 64B instruction
506 * @in: SG table for input
507 * @out SG table for output
508 * @tstamp: Request submitted time in jiffies
509 * @callback: callback after request completion/timeout
510 * @cb_arg: callback argument
512 struct nitrox_softreq {
513 struct list_head response;
514 struct list_head backlog;
520 struct nitrox_device *ndev;
521 struct nitrox_cmdq *cmdq;
523 struct nps_pkt_instr instr;
524 struct resp_hdr resp;
525 struct nitrox_sgtable in;
526 struct nitrox_sgtable out;
528 unsigned long tstamp;
530 completion_t callback;
534 static inline int flexi_aes_keylen(int keylen)
539 case AES_KEYSIZE_128:
542 case AES_KEYSIZE_192:
545 case AES_KEYSIZE_256:
549 aes_keylen = -EINVAL;
555 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
559 size = sizeof(struct scatterlist) * nents;
562 return kzalloc(size, gfp);
566 * create_single_sg - Point SG entry to the data
567 * @sg: Destination SG list
569 * @buflen: Data length
571 * Returns next free entry in the destination SG list
573 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
574 void *buf, int buflen)
576 sg_set_buf(sg, buf, buflen);
582 * create_multi_sg - Create multiple sg entries with buflen data length from
584 * @to_sg: Destination SG list
585 * @from_sg: Source SG list
586 * @buflen: Data length
588 * Returns next free entry in the destination SG list
590 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
591 struct scatterlist *from_sg,
594 struct scatterlist *sg = to_sg;
597 for (; buflen && from_sg; buflen -= sglen) {
598 sglen = from_sg->length;
602 sg_set_buf(sg, sg_virt(from_sg), sglen);
603 from_sg = sg_next(from_sg);
610 static inline void set_orh_value(u64 *orh)
612 WRITE_ONCE(*orh, PENDING_SIG);
615 static inline void set_comp_value(u64 *comp)
617 WRITE_ONCE(*comp, PENDING_SIG);
620 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
621 int nents, int ivsize)
623 struct se_crypto_request *creq = &nkreq->creq;
625 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
632 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
634 memcpy(dst, src, size);
637 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
639 return (struct scatterlist *)(iv + ivsize);
642 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
643 int nents, int ivsize,
644 struct scatterlist *src, int buflen)
646 char *iv = nkreq->src;
647 struct scatterlist *sg;
648 struct se_crypto_request *creq = &nkreq->creq;
650 creq->src = nitrox_creq_src_sg(iv, ivsize);
652 sg_init_table(sg, nents);
655 * +----+----------------+
656 * | IV | SRC sg entries |
657 * +----+----------------+
661 sg = create_single_sg(sg, iv, ivsize);
663 create_multi_sg(sg, src, buflen);
666 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
669 int extralen = ORH_HLEN + COMP_HLEN;
670 struct se_crypto_request *creq = &nkreq->creq;
672 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
679 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
681 struct se_crypto_request *creq = &nkreq->creq;
683 creq->orh = (u64 *)(nkreq->dst);
684 set_orh_value(creq->orh);
687 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
689 struct se_crypto_request *creq = &nkreq->creq;
691 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
692 set_comp_value(creq->comp);
695 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
697 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
700 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
701 int nents, int ivsize,
702 struct scatterlist *dst, int buflen)
704 struct se_crypto_request *creq = &nkreq->creq;
705 struct scatterlist *sg;
706 char *iv = nkreq->src;
708 creq->dst = nitrox_creq_dst_sg(nkreq->dst);
710 sg_init_table(sg, nents);
713 * +-----+----+----------------+-----------------+
714 * | ORH | IV | DST sg entries | COMPLETION Bytes|
715 * +-----+----+----------------+-----------------+
719 sg = create_single_sg(sg, creq->orh, ORH_HLEN);
721 sg = create_single_sg(sg, iv, ivsize);
723 sg = create_multi_sg(sg, dst, buflen);
724 /* COMPLETION Bytes */
725 create_single_sg(sg, creq->comp, COMP_HLEN);
728 #endif /* __NITROX_REQ_H */