drivers/crypto/cavium/nitrox/nitrox_isr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/pci.h>
   3 #include <linux/printk.h>
   4 #include <linux/slab.h>
   5
   6 #include "nitrox_dev.h"
   7 #include "nitrox_csr.h"
   8 #include "nitrox_common.h"
   9 #include "nitrox_hal.h"
  10 #include "nitrox_isr.h"
  11 #include "nitrox_mbx.h"
  12
  13 /*
  14  * One vector for each type of ring
  15  *  - NPS packet ring, AQMQ ring and ZQMQ ring
  16  */
  17 #define NR_RING_VECTORS 3
  18 #define NR_NON_RING_VECTORS 1
  19 /* base entry for packet ring/port */
  20 #define PKT_RING_MSIX_BASE 0
  21 #define NON_RING_MSIX_BASE 192
  22
  23 /**
  24  * nps_pkt_slc_isr - IRQ handler for NPS solicit port
  25  * @irq: irq number
  26  * @data: argument
  27  */
  28 static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
  29 {
  30         struct nitrox_q_vector *qvec = data;
  31         union nps_pkt_slc_cnts slc_cnts;
  32         struct nitrox_cmdq *cmdq = qvec->cmdq;
  33
  34         slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
  35         /* New packet on SLC output port */
  36         if (slc_cnts.s.slc_int)
  37                 tasklet_hi_schedule(&qvec->resp_tasklet);
  38
  39         return IRQ_HANDLED;
  40 }
  41
  42 static void clear_nps_core_err_intr(struct nitrox_device *ndev)
  43 {
  44         u64 value;
  45
  46         /* Write 1 to clear */
  47         value = nitrox_read_csr(ndev, NPS_CORE_INT);
  48         nitrox_write_csr(ndev, NPS_CORE_INT, value);
  49
  50         dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT  0x%016llx\n", value);
  51 }
  52
  53 static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
  54 {
  55         union nps_pkt_int pkt_int;
  56         unsigned long value, offset;
  57         int i;
  58
  59         pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
  60         dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT  0x%016llx\n",
  61                             pkt_int.value);
  62
  63         if (pkt_int.s.slc_err) {
  64                 offset = NPS_PKT_SLC_ERR_TYPE;
  65                 value = nitrox_read_csr(ndev, offset);
  66                 nitrox_write_csr(ndev, offset, value);
  67                 dev_err_ratelimited(DEV(ndev),
  68                                     "NPS_PKT_SLC_ERR_TYPE  0x%016lx\n", value);
  69
  70                 offset = NPS_PKT_SLC_RERR_LO;
  71                 value = nitrox_read_csr(ndev, offset);
  72                 nitrox_write_csr(ndev, offset, value);
  73                 /* enable the solicit ports */
  74                 for_each_set_bit(i, &value, BITS_PER_LONG)
  75                         enable_pkt_solicit_port(ndev, i);
  76
  77                 dev_err_ratelimited(DEV(ndev),
  78                                     "NPS_PKT_SLC_RERR_LO  0x%016lx\n", value);
  79
  80                 offset = NPS_PKT_SLC_RERR_HI;
  81                 value = nitrox_read_csr(ndev, offset);
  82                 nitrox_write_csr(ndev, offset, value);
  83                 dev_err_ratelimited(DEV(ndev),
  84                                     "NPS_PKT_SLC_RERR_HI  0x%016lx\n", value);
  85         }
  86
  87         if (pkt_int.s.in_err) {
  88                 offset = NPS_PKT_IN_ERR_TYPE;
  89                 value = nitrox_read_csr(ndev, offset);
  90                 nitrox_write_csr(ndev, offset, value);
  91                 dev_err_ratelimited(DEV(ndev),
  92                                     "NPS_PKT_IN_ERR_TYPE  0x%016lx\n", value);
  93                 offset = NPS_PKT_IN_RERR_LO;
  94                 value = nitrox_read_csr(ndev, offset);
  95                 nitrox_write_csr(ndev, offset, value);
  96                 /* enable the input ring */
  97                 for_each_set_bit(i, &value, BITS_PER_LONG)
  98                         enable_pkt_input_ring(ndev, i);
  99
 100                 dev_err_ratelimited(DEV(ndev),
 101                                     "NPS_PKT_IN_RERR_LO  0x%016lx\n", value);
 102
 103                 offset = NPS_PKT_IN_RERR_HI;
 104                 value = nitrox_read_csr(ndev, offset);
 105                 nitrox_write_csr(ndev, offset, value);
 106                 dev_err_ratelimited(DEV(ndev),
 107                                     "NPS_PKT_IN_RERR_HI  0x%016lx\n", value);
 108         }
 109 }
 110
 111 static void clear_pom_err_intr(struct nitrox_device *ndev)
 112 {
 113         u64 value;
 114
 115         value = nitrox_read_csr(ndev, POM_INT);
 116         nitrox_write_csr(ndev, POM_INT, value);
 117         dev_err_ratelimited(DEV(ndev), "POM_INT  0x%016llx\n", value);
 118 }
 119
 120 static void clear_pem_err_intr(struct nitrox_device *ndev)
 121 {
 122         u64 value;
 123
 124         value = nitrox_read_csr(ndev, PEM0_INT);
 125         nitrox_write_csr(ndev, PEM0_INT, value);
 126         dev_err_ratelimited(DEV(ndev), "PEM(0)_INT  0x%016llx\n", value);
 127 }
 128
 129 static void clear_lbc_err_intr(struct nitrox_device *ndev)
 130 {
 131         union lbc_int lbc_int;
 132         u64 value, offset;
 133         int i;
 134
 135         lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
 136         dev_err_ratelimited(DEV(ndev), "LBC_INT  0x%016llx\n", lbc_int.value);
 137
 138         if (lbc_int.s.dma_rd_err) {
 139                 for (i = 0; i < NR_CLUSTERS; i++) {
 140                         offset = EFL_CORE_VF_ERR_INT0X(i);
 141                         value = nitrox_read_csr(ndev, offset);
 142                         nitrox_write_csr(ndev, offset, value);
 143                         offset = EFL_CORE_VF_ERR_INT1X(i);
 144                         value = nitrox_read_csr(ndev, offset);
 145                         nitrox_write_csr(ndev, offset, value);
 146                 }
 147         }
 148
 149         if (lbc_int.s.cam_soft_err) {
 150                 dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
 151                 invalidate_lbc(ndev);
 152         }
 153
 154         if (lbc_int.s.pref_dat_len_mismatch_err) {
 155                 offset = LBC_PLM_VF1_64_INT;
 156                 value = nitrox_read_csr(ndev, offset);
 157                 nitrox_write_csr(ndev, offset, value);
 158                 offset = LBC_PLM_VF65_128_INT;
 159                 value = nitrox_read_csr(ndev, offset);
 160                 nitrox_write_csr(ndev, offset, value);
 161         }
 162
 163         if (lbc_int.s.rd_dat_len_mismatch_err) {
 164                 offset = LBC_ELM_VF1_64_INT;
 165                 value = nitrox_read_csr(ndev, offset);
 166                 nitrox_write_csr(ndev, offset, value);
 167                 offset = LBC_ELM_VF65_128_INT;
 168                 value = nitrox_read_csr(ndev, offset);
 169                 nitrox_write_csr(ndev, offset, value);
 170         }
 171         nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
 172 }
 173
 174 static void clear_efl_err_intr(struct nitrox_device *ndev)
 175 {
 176         int i;
 177
 178         for (i = 0; i < NR_CLUSTERS; i++) {
 179                 union efl_core_int core_int;
 180                 u64 value, offset;
 181
 182                 offset = EFL_CORE_INTX(i);
 183                 core_int.value = nitrox_read_csr(ndev, offset);
 184                 nitrox_write_csr(ndev, offset, core_int.value);
 185                 dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT  0x%016llx\n",
 186                                     i, core_int.value);
 187                 if (core_int.s.se_err) {
 188                         offset = EFL_CORE_SE_ERR_INTX(i);
 189                         value = nitrox_read_csr(ndev, offset);
 190                         nitrox_write_csr(ndev, offset, value);
 191                 }
 192         }
 193 }
 194
 195 static void clear_bmi_err_intr(struct nitrox_device *ndev)
 196 {
 197         u64 value;
 198
 199         value = nitrox_read_csr(ndev, BMI_INT);
 200         nitrox_write_csr(ndev, BMI_INT, value);
 201         dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
 202 }
 203
 204 static void nps_core_int_tasklet(unsigned long data)
 205 {
 206         struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
 207         struct nitrox_device *ndev = qvec->ndev;
 208
 209         /* if pf mode do queue recovery */
 210         if (ndev->mode == __NDEV_MODE_PF) {
 211         } else {
 212                 /**
 213                  * if VF(s) enabled communicate the error information
 214                  * to VF(s)
 215                  */
 216         }
 217 }
 218
 219 /*
 220  * nps_core_int_isr - interrupt handler for NITROX errors and
 221  *   mailbox communication
 222  */
 223 static irqreturn_t nps_core_int_isr(int irq, void *data)
 224 {
 225         struct nitrox_q_vector *qvec = data;
 226         struct nitrox_device *ndev = qvec->ndev;
 227         union nps_core_int_active core_int;
 228
 229         core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
 230
 231         if (core_int.s.nps_core)
 232                 clear_nps_core_err_intr(ndev);
 233
 234         if (core_int.s.nps_pkt)
 235                 clear_nps_pkt_err_intr(ndev);
 236
 237         if (core_int.s.pom)
 238                 clear_pom_err_intr(ndev);
 239
 240         if (core_int.s.pem)
 241                 clear_pem_err_intr(ndev);
 242
 243         if (core_int.s.lbc)
 244                 clear_lbc_err_intr(ndev);
 245
 246         if (core_int.s.efl)
 247                 clear_efl_err_intr(ndev);
 248
 249         if (core_int.s.bmi)
 250                 clear_bmi_err_intr(ndev);
 251
 252         /* Mailbox interrupt */
 253         if (core_int.s.mbox)
 254                 nitrox_pf2vf_mbox_handler(ndev);
 255
 256         /* If more work callback the ISR, set resend */
 257         core_int.s.resend = 1;
 258         nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
 259
 260         return IRQ_HANDLED;
 261 }
 262
 263 void nitrox_unregister_interrupts(struct nitrox_device *ndev)
 264 {
 265         struct pci_dev *pdev = ndev->pdev;
 266         int i;
 267
 268         for (i = 0; i < ndev->num_vecs; i++) {
 269                 struct nitrox_q_vector *qvec;
 270                 int vec;
 271
 272                 qvec = ndev->qvec + i;
 273                 if (!qvec->valid)
 274                         continue;
 275
 276                 /* get the vector number */
 277                 vec = pci_irq_vector(pdev, i);
 278                 irq_set_affinity_hint(vec, NULL);
 279                 free_irq(vec, qvec);
 280
 281                 tasklet_disable(&qvec->resp_tasklet);
 282                 tasklet_kill(&qvec->resp_tasklet);
 283                 qvec->valid = false;
 284         }
 285         kfree(ndev->qvec);
 286         ndev->qvec = NULL;
 287         pci_free_irq_vectors(pdev);
 288 }
 289
 290 int nitrox_register_interrupts(struct nitrox_device *ndev)
 291 {
 292         struct pci_dev *pdev = ndev->pdev;
 293         struct nitrox_q_vector *qvec;
 294         int nr_vecs, vec, cpu;
 295         int ret, i;
 296
 297         /*
 298          * PF MSI-X vectors
 299          *
 300          * Entry 0: NPS PKT ring 0
 301          * Entry 1: AQMQ ring 0
 302          * Entry 2: ZQM ring 0
 303          * Entry 3: NPS PKT ring 1
 304          * Entry 4: AQMQ ring 1
 305          * Entry 5: ZQM ring 1
 306          * ....
 307          * Entry 192: NPS_CORE_INT_ACTIVE
 308          */
 309         nr_vecs = pci_msix_vec_count(pdev);
 310         if (nr_vecs < 0) {
 311                 dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs);
 312                 return nr_vecs;
 313         }
 314
 315         /* Enable MSI-X */
 316         ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
 317         if (ret < 0) {
 318                 dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
 319                 return ret;
 320         }
 321         ndev->num_vecs = nr_vecs;
 322
 323         ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL);
 324         if (!ndev->qvec) {
 325                 pci_free_irq_vectors(pdev);
 326                 return -ENOMEM;
 327         }
 328
 329         /* request irqs for packet rings/ports */
 330         for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
 331                 qvec = &ndev->qvec[i];
 332
 333                 qvec->ring = i / NR_RING_VECTORS;
 334                 if (qvec->ring >= ndev->nr_queues)
 335                         break;
 336
 337                 qvec->cmdq = &ndev->pkt_inq[qvec->ring];
 338                 snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
 339                 /* get the vector number */
 340                 vec = pci_irq_vector(pdev, i);
 341                 ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
 342                 if (ret) {
 343                         dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
 344                                 qvec->ring);
 345                         goto irq_fail;
 346                 }
 347                 cpu = qvec->ring % num_online_cpus();
 348                 irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 349
 350                 tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
 351                              (unsigned long)qvec);
 352                 qvec->valid = true;
 353         }
 354
 355         /* request irqs for non ring vectors */
 356         i = NON_RING_MSIX_BASE;
 357         qvec = &ndev->qvec[i];
 358         qvec->ndev = ndev;
 359
 360         snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
 361         /* get the vector number */
 362         vec = pci_irq_vector(pdev, i);
 363         ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
 364         if (ret) {
 365                 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
 366                 goto irq_fail;
 367         }
 368         cpu = num_online_cpus();
 369         irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 370
 371         tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
 372                      (unsigned long)qvec);
 373         qvec->valid = true;
 374
 375         return 0;
 376
 377 irq_fail:
 378         nitrox_unregister_interrupts(ndev);
 379         return ret;
 380 }
 381
 382 void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
 383 {
 384         struct pci_dev *pdev = ndev->pdev;
 385         int i;
 386
 387         for (i = 0; i < ndev->num_vecs; i++) {
 388                 struct nitrox_q_vector *qvec;
 389                 int vec;
 390
 391                 qvec = ndev->qvec + i;
 392                 if (!qvec->valid)
 393                         continue;
 394
 395                 vec = ndev->iov.msix.vector;
 396                 irq_set_affinity_hint(vec, NULL);
 397                 free_irq(vec, qvec);
 398
 399                 tasklet_disable(&qvec->resp_tasklet);
 400                 tasklet_kill(&qvec->resp_tasklet);
 401                 qvec->valid = false;
 402         }
 403         kfree(ndev->qvec);
 404         ndev->qvec = NULL;
 405         pci_disable_msix(pdev);
 406 }
 407
 408 int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
 409 {
 410         struct pci_dev *pdev = ndev->pdev;
 411         struct nitrox_q_vector *qvec;
 412         int vec, cpu;
 413         int ret;
 414
 415         /**
 416          * only non ring vectors i.e Entry 192 is available
 417          * for PF in SR-IOV mode.
 418          */
 419         ndev->iov.msix.entry = NON_RING_MSIX_BASE;
 420         ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS);
 421         if (ret) {
 422                 dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
 423                         NON_RING_MSIX_BASE);
 424                 return ret;
 425         }
 426
 427         qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL);
 428         if (!qvec) {
 429                 pci_disable_msix(pdev);
 430                 return -ENOMEM;
 431         }
 432         qvec->ndev = ndev;
 433
 434         ndev->qvec = qvec;
 435         ndev->num_vecs = NR_NON_RING_VECTORS;
 436         snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d",
 437                  NON_RING_MSIX_BASE);
 438
 439         vec = ndev->iov.msix.vector;
 440         ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
 441         if (ret) {
 442                 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
 443                         NON_RING_MSIX_BASE);
 444                 goto iov_irq_fail;
 445         }
 446         cpu = num_online_cpus();
 447         irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 448
 449         tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
 450                      (unsigned long)qvec);
 451         qvec->valid = true;
 452
 453         return 0;
 454
 455 iov_irq_fail:
 456         nitrox_sriov_unregister_interrupts(ndev);
 457         return ret;
 458 }