1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/workqueue.h>
4 #include <crypto/internal/skcipher.h>
6 #include "nitrox_dev.h"
7 #include "nitrox_req.h"
8 #include "nitrox_csr.h"
9 #include "nitrox_req.h"
12 #define MIN_UDD_LEN 16
13 /* PKT_IN_HDR + SLC_STORE_INFO */
15 /* Base destination port for the solicited requests */
16 #define SOLICIT_BASE_DPORT 256
17 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
19 #define REQ_NOT_POSTED 1
24 * Response codes from SE microcode
26 * Completion with no error
27 * 0x43 - ERR_GC_DATA_LEN_INVALID
28 * Invalid Data length if Encryption Data length is
29 * less than 16 bytes for AES-XTS and AES-CTS.
30 * 0x45 - ERR_GC_CTX_LEN_INVALID
31 * Invalid context length: CTXL != 23 words.
32 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
33 * DOCSIS support is enabled with other than
34 * AES/DES-CBC mode encryption.
35 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
36 * Authentication offset is other than 0 with
37 * Encryption IV source = 0.
38 * Authentication offset is other than 8 (DES)/16 (AES)
39 * with Encryption IV source = 1
40 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
41 * CRC32 is enabled for other than DOCSIS encryption.
42 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
43 * Invalid flag options in AES-CCM IV.
46 static inline int incr_index(int index, int count, int max)
48 if ((index + count) >= max)
49 index = index + count - max;
57 * dma_free_sglist - unmap and free the sg lists.
61 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
63 struct nitrox_device *ndev = sr->ndev;
64 struct device *dev = DEV(ndev);
65 struct nitrox_sglist *sglist;
68 sglist = sr->in.sglist;
73 dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
74 /* unmpa src sglist */
75 dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
76 /* unamp gather component */
77 dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
82 sr->in.map_bufs_cnt = 0;
86 sglist = sr->out.sglist;
91 dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
93 /* unmap dst sglist */
95 dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
98 /* unmap completion */
99 dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
101 /* unmap scatter component */
102 dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
103 kfree(sr->out.sglist);
104 kfree(sr->out.sgcomp);
105 sr->out.sglist = NULL;
107 sr->out.map_bufs_cnt = 0;
110 static void softreq_destroy(struct nitrox_softreq *sr)
112 softreq_unmap_sgbufs(sr);
117 * create_sg_component - create SG componets for N5 device.
118 * @sr: Request structure
120 * @nr_comp: total number of components required
122 * Component structure
124 * 63 48 47 32 31 16 15 0
125 * --------------------------------------
126 * | LEN0 | LEN1 | LEN2 | LEN3 |
127 * |-------------------------------------
129 * --------------------------------------
131 * --------------------------------------
133 * --------------------------------------
135 * --------------------------------------
137 * Returns 0 if success or a negative errno code on error.
139 static int create_sg_component(struct nitrox_softreq *sr,
140 struct nitrox_sgtable *sgtbl, int map_nents)
142 struct nitrox_device *ndev = sr->ndev;
143 struct nitrox_sgcomp *sgcomp;
144 struct nitrox_sglist *sglist;
149 nr_sgcomp = roundup(map_nents, 4) / 4;
151 /* each component holds 4 dma pointers */
152 sz_comp = nr_sgcomp * sizeof(*sgcomp);
153 sgcomp = kzalloc(sz_comp, sr->gfp);
157 sgtbl->sgcomp = sgcomp;
158 sgtbl->nr_sgcomp = nr_sgcomp;
160 sglist = sgtbl->sglist;
161 /* populate device sg component */
162 for (i = 0; i < nr_sgcomp; i++) {
163 for (j = 0; j < 4; j++) {
164 sgcomp->len[j] = cpu_to_be16(sglist->len);
165 sgcomp->dma[j] = cpu_to_be64(sglist->dma);
170 /* map the device sg component */
171 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
172 if (dma_mapping_error(DEV(ndev), dma)) {
173 kfree(sgtbl->sgcomp);
174 sgtbl->sgcomp = NULL;
179 sgtbl->len = sz_comp;
185 * dma_map_inbufs - DMA map input sglist and creates sglist component
187 * @sr: Request structure
188 * @req: Crypto request structre
190 * Returns 0 if successful or a negative errno code on error.
192 static int dma_map_inbufs(struct nitrox_softreq *sr,
193 struct se_crypto_request *req)
195 struct device *dev = DEV(sr->ndev);
196 struct scatterlist *sg = req->src;
197 struct nitrox_sglist *glist;
198 int i, nents, ret = 0;
202 nents = sg_nents(req->src);
204 /* creater gather list IV and src entries */
205 sz = roundup((1 + nents), 4) * sizeof(*glist);
206 glist = kzalloc(sz, sr->gfp);
210 sr->in.sglist = glist;
212 dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
213 if (dma_mapping_error(dev, dma)) {
218 sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
219 /* map src entries */
220 nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
225 sr->in.buf = req->src;
227 /* store the mappings */
228 glist->len = req->ivsize;
231 sr->in.total_bytes += req->ivsize;
233 for_each_sg(req->src, sg, nents, i) {
234 glist->len = sg_dma_len(sg);
235 glist->dma = sg_dma_address(sg);
236 sr->in.total_bytes += glist->len;
239 /* roundup map count to align with entires in sg component */
240 sr->in.map_bufs_cnt = (1 + nents);
242 /* create NITROX gather component */
243 ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
250 dma_unmap_sg(dev, req->src, nents, sr->in.dir);
251 sr->in.map_bufs_cnt = 0;
253 dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
255 kfree(sr->in.sglist);
256 sr->in.sglist = NULL;
260 static int dma_map_outbufs(struct nitrox_softreq *sr,
261 struct se_crypto_request *req)
263 struct device *dev = DEV(sr->ndev);
264 struct nitrox_sglist *glist = sr->in.sglist;
265 struct nitrox_sglist *slist;
266 struct scatterlist *sg;
267 int i, nents, map_bufs_cnt, ret = 0;
270 nents = sg_nents(req->dst);
272 /* create scatter list ORH, IV, dst entries and Completion header */
273 sz = roundup((3 + nents), 4) * sizeof(*slist);
274 slist = kzalloc(sz, sr->gfp);
278 sr->out.sglist = slist;
279 sr->out.dir = DMA_BIDIRECTIONAL;
281 sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
283 if (dma_mapping_error(dev, sr->resp.orh_dma)) {
289 sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
290 COMP_HLEN, sr->out.dir);
291 if (dma_mapping_error(dev, sr->resp.completion_dma)) {
296 sr->inplace = (req->src == req->dst) ? true : false;
299 nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
305 sr->out.buf = req->dst;
307 /* store the mappings */
309 slist->len = ORH_HLEN;
310 slist->dma = sr->resp.orh_dma;
313 /* copy the glist mappings */
315 nents = sr->in.map_bufs_cnt - 1;
316 map_bufs_cnt = sr->in.map_bufs_cnt;
317 while (map_bufs_cnt--) {
318 slist->len = glist->len;
319 slist->dma = glist->dma;
324 /* copy iv mapping */
325 slist->len = glist->len;
326 slist->dma = glist->dma;
328 /* copy remaining maps */
329 for_each_sg(req->dst, sg, nents, i) {
330 slist->len = sg_dma_len(sg);
331 slist->dma = sg_dma_address(sg);
337 slist->len = COMP_HLEN;
338 slist->dma = sr->resp.completion_dma;
340 sr->out.map_bufs_cnt = (3 + nents);
342 ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
344 goto outcomp_map_err;
350 dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
351 sr->out.map_bufs_cnt = 0;
354 dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
355 sr->resp.completion_dma = 0;
357 dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
358 sr->resp.orh_dma = 0;
360 kfree(sr->out.sglist);
361 sr->out.sglist = NULL;
365 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
366 struct se_crypto_request *creq)
370 ret = dma_map_inbufs(sr, creq);
374 ret = dma_map_outbufs(sr, creq);
376 softreq_unmap_sgbufs(sr);
381 static inline void backlog_list_add(struct nitrox_softreq *sr,
382 struct nitrox_cmdq *cmdq)
384 INIT_LIST_HEAD(&sr->backlog);
386 spin_lock_bh(&cmdq->backlog_lock);
387 list_add_tail(&sr->backlog, &cmdq->backlog_head);
388 atomic_inc(&cmdq->backlog_count);
389 atomic_set(&sr->status, REQ_BACKLOG);
390 spin_unlock_bh(&cmdq->backlog_lock);
393 static inline void response_list_add(struct nitrox_softreq *sr,
394 struct nitrox_cmdq *cmdq)
396 INIT_LIST_HEAD(&sr->response);
398 spin_lock_bh(&cmdq->response_lock);
399 list_add_tail(&sr->response, &cmdq->response_head);
400 spin_unlock_bh(&cmdq->response_lock);
403 static inline void response_list_del(struct nitrox_softreq *sr,
404 struct nitrox_cmdq *cmdq)
406 spin_lock_bh(&cmdq->response_lock);
407 list_del(&sr->response);
408 spin_unlock_bh(&cmdq->response_lock);
411 static struct nitrox_softreq *
412 get_first_response_entry(struct nitrox_cmdq *cmdq)
414 return list_first_entry_or_null(&cmdq->response_head,
415 struct nitrox_softreq, response);
418 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
420 if (atomic_inc_return(&cmdq->pending_count) > qlen) {
421 atomic_dec(&cmdq->pending_count);
422 /* sync with other cpus */
423 smp_mb__after_atomic();
430 * post_se_instr - Post SE instruction to Packet Input ring
431 * @sr: Request structure
433 * Returns 0 if successful or a negative error code,
434 * if no space in ring.
436 static void post_se_instr(struct nitrox_softreq *sr,
437 struct nitrox_cmdq *cmdq)
439 struct nitrox_device *ndev = sr->ndev;
443 spin_lock_bh(&cmdq->cmdq_lock);
445 idx = cmdq->write_idx;
446 /* copy the instruction */
447 ent = cmdq->head + (idx * cmdq->instr_size);
448 memcpy(ent, &sr->instr, cmdq->instr_size);
450 atomic_set(&sr->status, REQ_POSTED);
451 response_list_add(sr, cmdq);
452 sr->tstamp = jiffies;
453 /* flush the command queue updates */
456 /* Ring doorbell with count 1 */
457 writeq(1, cmdq->dbell_csr_addr);
458 /* orders the doorbell rings */
461 cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
463 spin_unlock_bh(&cmdq->cmdq_lock);
466 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
468 struct nitrox_device *ndev = cmdq->ndev;
469 struct nitrox_softreq *sr, *tmp;
472 if (!atomic_read(&cmdq->backlog_count))
475 spin_lock_bh(&cmdq->backlog_lock);
477 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
478 struct skcipher_request *skreq;
480 /* submit until space available */
481 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
485 /* delete from backlog list */
486 list_del(&sr->backlog);
487 atomic_dec(&cmdq->backlog_count);
488 /* sync with other cpus */
489 smp_mb__after_atomic();
492 /* post the command */
493 post_se_instr(sr, cmdq);
495 /* backlog requests are posted, wakeup with -EINPROGRESS */
496 skcipher_request_complete(skreq, -EINPROGRESS);
498 spin_unlock_bh(&cmdq->backlog_lock);
503 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
505 struct nitrox_cmdq *cmdq = sr->cmdq;
506 struct nitrox_device *ndev = sr->ndev;
508 /* try to post backlog requests */
509 post_backlog_cmds(cmdq);
511 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
512 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
514 /* add to backlog list */
515 backlog_list_add(sr, cmdq);
518 post_se_instr(sr, cmdq);
524 * nitrox_se_request - Send request to SE core
525 * @ndev: NITROX device
526 * @req: Crypto request
528 * Returns 0 on success, or a negative error code.
530 int nitrox_process_se_request(struct nitrox_device *ndev,
531 struct se_crypto_request *req,
532 completion_t callback,
533 struct skcipher_request *skreq)
535 struct nitrox_softreq *sr;
536 dma_addr_t ctx_handle = 0;
539 if (!nitrox_ready(ndev))
542 sr = kzalloc(sizeof(*sr), req->gfp);
547 sr->flags = req->flags;
549 sr->callback = callback;
552 atomic_set(&sr->status, REQ_NOT_POSTED);
554 WRITE_ONCE(sr->resp.orh, PENDING_SIG);
555 WRITE_ONCE(sr->resp.completion, PENDING_SIG);
557 ret = softreq_map_iobuf(sr, req);
563 /* get the context handle */
564 if (req->ctx_handle) {
568 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
569 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
570 ctx_handle = hdr->ctx_dma;
573 /* select the queue */
574 qno = smp_processor_id() % ndev->nr_queues;
576 sr->cmdq = &ndev->pkt_cmdqs[qno];
579 * 64-Byte Instruction Format
581 * ----------------------
583 * ----------------------
584 * | PKT_IN_INSTR_HDR | 8 bytes
585 * ----------------------
586 * | PKT_IN_HDR | 16 bytes
587 * ----------------------
588 * | SLC_INFO | 16 bytes
589 * ----------------------
590 * | Front data | 16 bytes
591 * ----------------------
594 /* fill the packet instruction */
596 sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
599 sr->instr.ih.value = 0;
600 sr->instr.ih.s.g = 1;
601 sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
602 sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
603 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
604 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
605 sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
608 sr->instr.irh.value[0] = 0;
609 sr->instr.irh.s.uddl = MIN_UDD_LEN;
610 /* context length in 64-bit words */
611 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
612 /* offset from solicit base port 256 */
613 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
614 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
615 sr->instr.irh.s.arg = req->ctrl.s.arg;
616 sr->instr.irh.s.opcode = req->opcode;
617 sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
620 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
623 sr->instr.slc.value[0] = 0;
624 sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
625 sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
628 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
631 * No conversion for front data,
632 * It goes into payload
633 * put GP Header in front data
635 sr->instr.fdata[0] = *((u64 *)&req->gph);
636 sr->instr.fdata[1] = 0;
638 ret = nitrox_enqueue_request(sr);
649 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
651 return time_after_eq(jiffies, (tstamp + timeout));
654 void backlog_qflush_work(struct work_struct *work)
656 struct nitrox_cmdq *cmdq;
658 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
659 post_backlog_cmds(cmdq);
663 * process_request_list - process completed requests
665 * @qno: queue to operate
667 * Returns the number of responses processed.
669 static void process_response_list(struct nitrox_cmdq *cmdq)
671 struct nitrox_device *ndev = cmdq->ndev;
672 struct nitrox_softreq *sr;
673 struct skcipher_request *skreq;
674 completion_t callback;
675 int req_completed = 0, err = 0, budget;
677 /* check all pending requests */
678 budget = atomic_read(&cmdq->pending_count);
680 while (req_completed < budget) {
681 sr = get_first_response_entry(cmdq);
685 if (atomic_read(&sr->status) != REQ_POSTED)
688 /* check orh and completion bytes updates */
689 if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
690 /* request not completed, check for timeout */
691 if (!cmd_timeout(sr->tstamp, ndev->timeout))
693 dev_err_ratelimited(DEV(ndev),
694 "Request timeout, orh 0x%016llx\n",
695 READ_ONCE(sr->resp.orh));
697 atomic_dec(&cmdq->pending_count);
698 /* sync with other cpus */
699 smp_mb__after_atomic();
700 /* remove from response list */
701 response_list_del(sr, cmdq);
703 callback = sr->callback;
707 err = READ_ONCE(sr->resp.orh) & 0xff;
711 callback(skreq, err);
718 * pkt_slc_resp_handler - post processing of SE responses
720 void pkt_slc_resp_handler(unsigned long data)
722 struct bh_data *bh = (void *)(uintptr_t)(data);
723 struct nitrox_cmdq *cmdq = bh->cmdq;
724 union nps_pkt_slc_cnts pkt_slc_cnts;
726 /* read completion count */
727 pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
728 /* resend the interrupt if more work to do */
729 pkt_slc_cnts.s.resend = 1;
731 process_response_list(cmdq);
734 * clear the interrupt with resend bit enabled,
735 * MSI-X interrupt generates if Completion count > Threshold
737 writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
738 /* order the writes */
741 if (atomic_read(&cmdq->backlog_count))
742 schedule_work(&cmdq->backlog_qflush);