drivers/crypto/cavium/nitrox/nitrox_lib.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/cpumask.h>
   3 #include <linux/dma-mapping.h>
   4 #include <linux/dmapool.h>
   5 #include <linux/delay.h>
   6 #include <linux/gfp.h>
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/pci_regs.h>
  10 #include <linux/vmalloc.h>
  11 #include <linux/pci.h>
  12
  13 #include "nitrox_dev.h"
  14 #include "nitrox_common.h"
  15 #include "nitrox_req.h"
  16 #include "nitrox_csr.h"
  17
  18 #define CRYPTO_CTX_SIZE 256
  19
  20 /* packet inuput ring alignments */
  21 #define PKTIN_Q_ALIGN_BYTES 16
  22 /* AQM Queue input alignments */
  23 #define AQM_Q_ALIGN_BYTES 32
  24
  25 static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
  26 {
  27         struct nitrox_device *ndev = cmdq->ndev;
  28
  29         cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
  30         cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize,
  31                                                 &cmdq->unalign_dma,
  32                                                 GFP_KERNEL);
  33         if (!cmdq->unalign_base)
  34                 return -ENOMEM;
  35
  36         cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
  37         cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
  38         cmdq->write_idx = 0;
  39
  40         spin_lock_init(&cmdq->cmd_qlock);
  41         spin_lock_init(&cmdq->resp_qlock);
  42         spin_lock_init(&cmdq->backlog_qlock);
  43
  44         INIT_LIST_HEAD(&cmdq->response_head);
  45         INIT_LIST_HEAD(&cmdq->backlog_head);
  46         INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
  47
  48         atomic_set(&cmdq->pending_count, 0);
  49         atomic_set(&cmdq->backlog_count, 0);
  50         return 0;
  51 }
  52
  53 static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
  54 {
  55         cmdq->write_idx = 0;
  56         atomic_set(&cmdq->pending_count, 0);
  57         atomic_set(&cmdq->backlog_count, 0);
  58 }
  59
  60 static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
  61 {
  62         struct nitrox_device *ndev;
  63
  64         if (!cmdq)
  65                 return;
  66
  67         if (!cmdq->unalign_base)
  68                 return;
  69
  70         ndev = cmdq->ndev;
  71         cancel_work_sync(&cmdq->backlog_qflush);
  72
  73         dma_free_coherent(DEV(ndev), cmdq->qsize,
  74                           cmdq->unalign_base, cmdq->unalign_dma);
  75         nitrox_cmdq_reset(cmdq);
  76
  77         cmdq->dbell_csr_addr = NULL;
  78         cmdq->compl_cnt_csr_addr = NULL;
  79         cmdq->unalign_base = NULL;
  80         cmdq->base = NULL;
  81         cmdq->unalign_dma = 0;
  82         cmdq->dma = 0;
  83         cmdq->qsize = 0;
  84         cmdq->instr_size = 0;
  85 }
  86
  87 static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
  88 {
  89         int i;
  90
  91         for (i = 0; i < ndev->nr_queues; i++) {
  92                 nitrox_cmdq_cleanup(ndev->aqmq[i]);
  93                 kfree_sensitive(ndev->aqmq[i]);
  94                 ndev->aqmq[i] = NULL;
  95         }
  96 }
  97
  98 static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
  99 {
 100         int i, err;
 101
 102         for (i = 0; i < ndev->nr_queues; i++) {
 103                 struct nitrox_cmdq *cmdq;
 104                 u64 offset;
 105
 106                 cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
 107                 if (!cmdq) {
 108                         err = -ENOMEM;
 109                         goto aqmq_fail;
 110                 }
 111
 112                 cmdq->ndev = ndev;
 113                 cmdq->qno = i;
 114                 cmdq->instr_size = sizeof(struct aqmq_command_s);
 115
 116                 /* AQM Queue Doorbell Counter Register Address */
 117                 offset = AQMQ_DRBLX(i);
 118                 cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
 119                 /* AQM Queue Commands Completed Count Register Address */
 120                 offset = AQMQ_CMD_CNTX(i);
 121                 cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
 122
 123                 err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
 124                 if (err) {
 125                         kfree_sensitive(cmdq);
 126                         goto aqmq_fail;
 127                 }
 128                 ndev->aqmq[i] = cmdq;
 129         }
 130
 131         return 0;
 132
 133 aqmq_fail:
 134         nitrox_free_aqm_queues(ndev);
 135         return err;
 136 }
 137
 138 static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
 139 {
 140         int i;
 141
 142         for (i = 0; i < ndev->nr_queues; i++) {
 143                 struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
 144
 145                 nitrox_cmdq_cleanup(cmdq);
 146         }
 147         kfree(ndev->pkt_inq);
 148         ndev->pkt_inq = NULL;
 149 }
 150
 151 static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
 152 {
 153         int i, err;
 154
 155         ndev->pkt_inq = kcalloc_node(ndev->nr_queues,
 156                                      sizeof(struct nitrox_cmdq),
 157                                      GFP_KERNEL, ndev->node);
 158         if (!ndev->pkt_inq)
 159                 return -ENOMEM;
 160
 161         for (i = 0; i < ndev->nr_queues; i++) {
 162                 struct nitrox_cmdq *cmdq;
 163                 u64 offset;
 164
 165                 cmdq = &ndev->pkt_inq[i];
 166                 cmdq->ndev = ndev;
 167                 cmdq->qno = i;
 168                 cmdq->instr_size = sizeof(struct nps_pkt_instr);
 169
 170                 /* packet input ring doorbell address */
 171                 offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
 172                 cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
 173                 /* packet solicit port completion count address */
 174                 offset = NPS_PKT_SLC_CNTSX(i);
 175                 cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
 176
 177                 err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
 178                 if (err)
 179                         goto pktq_fail;
 180         }
 181         return 0;
 182
 183 pktq_fail:
 184         nitrox_free_pktin_queues(ndev);
 185         return err;
 186 }
 187
 188 static int create_crypto_dma_pool(struct nitrox_device *ndev)
 189 {
 190         size_t size;
 191
 192         /* Crypto context pool, 16 byte aligned */
 193         size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
 194         ndev->ctx_pool = dma_pool_create("nitrox-context",
 195                                          DEV(ndev), size, 16, 0);
 196         if (!ndev->ctx_pool)
 197                 return -ENOMEM;
 198
 199         return 0;
 200 }
 201
 202 static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
 203 {
 204         if (!ndev->ctx_pool)
 205                 return;
 206
 207         dma_pool_destroy(ndev->ctx_pool);
 208         ndev->ctx_pool = NULL;
 209 }
 210
 211 /*
 212  * crypto_alloc_context - Allocate crypto context from pool
 213  * @ndev: NITROX Device
 214  */
 215 void *crypto_alloc_context(struct nitrox_device *ndev)
 216 {
 217         struct ctx_hdr *ctx;
 218         struct crypto_ctx_hdr *chdr;
 219         void *vaddr;
 220         dma_addr_t dma;
 221
 222         chdr = kmalloc(sizeof(*chdr), GFP_KERNEL);
 223         if (!chdr)
 224                 return NULL;
 225
 226         vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma);
 227         if (!vaddr) {
 228                 kfree(chdr);
 229                 return NULL;
 230         }
 231
 232         /* fill meta data */
 233         ctx = vaddr;
 234         ctx->pool = ndev->ctx_pool;
 235         ctx->dma = dma;
 236         ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
 237
 238         chdr->pool = ndev->ctx_pool;
 239         chdr->dma = dma;
 240         chdr->vaddr = vaddr;
 241
 242         return chdr;
 243 }
 244
 245 /**
 246  * crypto_free_context - Free crypto context to pool
 247  * @ctx: context to free
 248  */
 249 void crypto_free_context(void *ctx)
 250 {
 251         struct crypto_ctx_hdr *ctxp;
 252
 253         if (!ctx)
 254                 return;
 255
 256         ctxp = ctx;
 257         dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma);
 258         kfree(ctxp);
 259 }
 260
 261 /**
 262  * nitrox_common_sw_init - allocate software resources.
 263  * @ndev: NITROX device
 264  *
 265  * Allocates crypto context pools and command queues etc.
 266  *
 267  * Return: 0 on success, or a negative error code on error.
 268  */
 269 int nitrox_common_sw_init(struct nitrox_device *ndev)
 270 {
 271         int err = 0;
 272
 273         /* per device crypto context pool */
 274         err = create_crypto_dma_pool(ndev);
 275         if (err)
 276                 return err;
 277
 278         err = nitrox_alloc_pktin_queues(ndev);
 279         if (err)
 280                 destroy_crypto_dma_pool(ndev);
 281
 282         err = nitrox_alloc_aqm_queues(ndev);
 283         if (err) {
 284                 nitrox_free_pktin_queues(ndev);
 285                 destroy_crypto_dma_pool(ndev);
 286         }
 287
 288         return err;
 289 }
 290
 291 /**
 292  * nitrox_common_sw_cleanup - free software resources.
 293  * @ndev: NITROX device
 294  */
 295 void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
 296 {
 297         nitrox_free_aqm_queues(ndev);
 298         nitrox_free_pktin_queues(ndev);
 299         destroy_crypto_dma_pool(ndev);
 300 }