drivers/crypto/qat/qat_common/adf_isr.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/kernel.h>
   4 #include <linux/init.h>
   5 #include <linux/types.h>
   6 #include <linux/pci.h>
   7 #include <linux/slab.h>
   8 #include <linux/errno.h>
   9 #include <linux/interrupt.h>
  10 #include "adf_accel_devices.h"
  11 #include "adf_common_drv.h"
  12 #include "adf_cfg.h"
  13 #include "adf_cfg_strings.h"
  14 #include "adf_cfg_common.h"
  15 #include "adf_transport_access_macros.h"
  16 #include "adf_transport_internal.h"
  17
  18 #define ADF_MAX_NUM_VFS 32
  19
  20 static int adf_enable_msix(struct adf_accel_dev *accel_dev)
  21 {
  22         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
  23         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  24         u32 msix_num_entries = hw_data->num_banks + 1;
  25         int ret;
  26
  27         if (hw_data->set_msix_rttable)
  28                 hw_data->set_msix_rttable(accel_dev);
  29
  30         ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries,
  31                                     msix_num_entries, PCI_IRQ_MSIX);
  32         if (unlikely(ret < 0)) {
  33                 dev_err(&GET_DEV(accel_dev),
  34                         "Failed to allocate %d MSI-X vectors\n",
  35                         msix_num_entries);
  36                 return ret;
  37         }
  38         return 0;
  39 }
  40
  41 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
  42 {
  43         pci_free_irq_vectors(pci_dev_info->pci_dev);
  44 }
  45
  46 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
  47 {
  48         struct adf_etr_bank_data *bank = bank_ptr;
  49         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
  50
  51         csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,
  52                                             0);
  53         tasklet_hi_schedule(&bank->resp_handler);
  54         return IRQ_HANDLED;
  55 }
  56
  57 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
  58 {
  59         struct adf_accel_dev *accel_dev = dev_ptr;
  60
  61 #ifdef CONFIG_PCI_IOV
  62         /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
  63         if (accel_dev->pf.vf_info) {
  64                 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  65                 struct adf_bar *pmisc =
  66                         &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
  67                 void __iomem *pmisc_addr = pmisc->virt_addr;
  68                 unsigned long vf_mask;
  69
  70                 /* Get the interrupt sources triggered by VFs */
  71                 vf_mask = hw_data->get_vf2pf_sources(pmisc_addr);
  72
  73                 if (vf_mask) {
  74                         struct adf_accel_vf_info *vf_info;
  75                         bool irq_handled = false;
  76                         int i;
  77
  78                         /* Disable VF2PF interrupts for VFs with pending ints */
  79                         adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
  80
  81                         /*
  82                          * Handle VF2PF interrupt unless the VF is malicious and
  83                          * is attempting to flood the host OS with VF2PF interrupts.
  84                          */
  85                         for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
  86                                 vf_info = accel_dev->pf.vf_info + i;
  87
  88                                 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
  89                                         dev_info(&GET_DEV(accel_dev),
  90                                                  "Too many ints from VF%d\n",
  91                                                   vf_info->vf_nr + 1);
  92                                         continue;
  93                                 }
  94
  95                                 adf_schedule_vf2pf_handler(vf_info);
  96                                 irq_handled = true;
  97                         }
  98
  99                         if (irq_handled)
 100                                 return IRQ_HANDLED;
 101                 }
 102         }
 103 #endif /* CONFIG_PCI_IOV */
 104
 105         dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
 106                 accel_dev->accel_id);
 107
 108         return IRQ_NONE;
 109 }
 110
 111 static void adf_free_irqs(struct adf_accel_dev *accel_dev)
 112 {
 113         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
 114         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 115         struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
 116         struct adf_etr_data *etr_data = accel_dev->transport;
 117         int clust_irq = hw_data->num_banks;
 118         int irq, i = 0;
 119
 120         if (pci_dev_info->msix_entries.num_entries > 1) {
 121                 for (i = 0; i < hw_data->num_banks; i++) {
 122                         if (irqs[i].enabled) {
 123                                 irq = pci_irq_vector(pci_dev_info->pci_dev, i);
 124                                 irq_set_affinity_hint(irq, NULL);
 125                                 free_irq(irq, &etr_data->banks[i]);
 126                         }
 127                 }
 128         }
 129
 130         if (irqs[i].enabled) {
 131                 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
 132                 free_irq(irq, accel_dev);
 133         }
 134 }
 135
 136 static int adf_request_irqs(struct adf_accel_dev *accel_dev)
 137 {
 138         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
 139         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 140         struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
 141         struct adf_etr_data *etr_data = accel_dev->transport;
 142         int clust_irq = hw_data->num_banks;
 143         int ret, irq, i = 0;
 144         char *name;
 145
 146         /* Request msix irq for all banks unless SR-IOV enabled */
 147         if (!accel_dev->pf.vf_info) {
 148                 for (i = 0; i < hw_data->num_banks; i++) {
 149                         struct adf_etr_bank_data *bank = &etr_data->banks[i];
 150                         unsigned int cpu, cpus = num_online_cpus();
 151
 152                         name = irqs[i].name;
 153                         snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
 154                                  "qat%d-bundle%d", accel_dev->accel_id, i);
 155                         irq = pci_irq_vector(pci_dev_info->pci_dev, i);
 156                         if (unlikely(irq < 0)) {
 157                                 dev_err(&GET_DEV(accel_dev),
 158                                         "Failed to get IRQ number of device vector %d - %s\n",
 159                                         i, name);
 160                                 ret = irq;
 161                                 goto err;
 162                         }
 163                         ret = request_irq(irq, adf_msix_isr_bundle, 0,
 164                                           &name[0], bank);
 165                         if (ret) {
 166                                 dev_err(&GET_DEV(accel_dev),
 167                                         "Failed to allocate IRQ %d for %s\n",
 168                                         irq, name);
 169                                 goto err;
 170                         }
 171
 172                         cpu = ((accel_dev->accel_id * hw_data->num_banks) +
 173                                i) % cpus;
 174                         irq_set_affinity_hint(irq, get_cpu_mask(cpu));
 175                         irqs[i].enabled = true;
 176                 }
 177         }
 178
 179         /* Request msix irq for AE */
 180         name = irqs[i].name;
 181         snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
 182                  "qat%d-ae-cluster", accel_dev->accel_id);
 183         irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
 184         if (unlikely(irq < 0)) {
 185                 dev_err(&GET_DEV(accel_dev),
 186                         "Failed to get IRQ number of device vector %d - %s\n",
 187                         i, name);
 188                 ret = irq;
 189                 goto err;
 190         }
 191         ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev);
 192         if (ret) {
 193                 dev_err(&GET_DEV(accel_dev),
 194                         "Failed to allocate IRQ %d for %s\n", irq, name);
 195                 goto err;
 196         }
 197         irqs[i].enabled = true;
 198         return ret;
 199 err:
 200         adf_free_irqs(accel_dev);
 201         return ret;
 202 }
 203
 204 static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev)
 205 {
 206         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 207         u32 msix_num_entries = 1;
 208         struct adf_irq *irqs;
 209
 210         /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
 211         if (!accel_dev->pf.vf_info)
 212                 msix_num_entries += hw_data->num_banks;
 213
 214         irqs = kzalloc_node(msix_num_entries * sizeof(*irqs),
 215                             GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
 216         if (!irqs)
 217                 return -ENOMEM;
 218
 219         accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
 220         accel_dev->accel_pci_dev.msix_entries.irqs = irqs;
 221         return 0;
 222 }
 223
 224 static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev)
 225 {
 226         kfree(accel_dev->accel_pci_dev.msix_entries.irqs);
 227         accel_dev->accel_pci_dev.msix_entries.irqs = NULL;
 228 }
 229
 230 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
 231 {
 232         struct adf_etr_data *priv_data = accel_dev->transport;
 233         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 234         int i;
 235
 236         for (i = 0; i < hw_data->num_banks; i++)
 237                 tasklet_init(&priv_data->banks[i].resp_handler,
 238                              adf_response_handler,
 239                              (unsigned long)&priv_data->banks[i]);
 240         return 0;
 241 }
 242
 243 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
 244 {
 245         struct adf_etr_data *priv_data = accel_dev->transport;
 246         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 247         int i;
 248
 249         for (i = 0; i < hw_data->num_banks; i++) {
 250                 tasklet_disable(&priv_data->banks[i].resp_handler);
 251                 tasklet_kill(&priv_data->banks[i].resp_handler);
 252         }
 253 }
 254
 255 /**
 256  * adf_isr_resource_free() - Free IRQ for acceleration device
 257  * @accel_dev:  Pointer to acceleration device.
 258  *
 259  * Function frees interrupts for acceleration device.
 260  */
 261 void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
 262 {
 263         adf_free_irqs(accel_dev);
 264         adf_cleanup_bh(accel_dev);
 265         adf_disable_msix(&accel_dev->accel_pci_dev);
 266         adf_isr_free_msix_vectors_data(accel_dev);
 267 }
 268 EXPORT_SYMBOL_GPL(adf_isr_resource_free);
 269
 270 /**
 271  * adf_isr_resource_alloc() - Allocate IRQ for acceleration device
 272  * @accel_dev:  Pointer to acceleration device.
 273  *
 274  * Function allocates interrupts for acceleration device.
 275  *
 276  * Return: 0 on success, error code otherwise.
 277  */
 278 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
 279 {
 280         int ret;
 281
 282         ret = adf_isr_alloc_msix_vectors_data(accel_dev);
 283         if (ret)
 284                 goto err_out;
 285
 286         ret = adf_enable_msix(accel_dev);
 287         if (ret)
 288                 goto err_free_msix_table;
 289
 290         ret = adf_setup_bh(accel_dev);
 291         if (ret)
 292                 goto err_disable_msix;
 293
 294         ret = adf_request_irqs(accel_dev);
 295         if (ret)
 296                 goto err_cleanup_bh;
 297
 298         return 0;
 299
 300 err_cleanup_bh:
 301         adf_cleanup_bh(accel_dev);
 302
 303 err_disable_msix:
 304         adf_disable_msix(&accel_dev->accel_pci_dev);
 305
 306 err_free_msix_table:
 307         adf_isr_free_msix_vectors_data(accel_dev);
 308
 309 err_out:
 310         return ret;
 311 }
 312 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);