drivers/crypto/qat/qat_common/adf_isr.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/kernel.h>
   4 #include <linux/init.h>
   5 #include <linux/types.h>
   6 #include <linux/pci.h>
   7 #include <linux/slab.h>
   8 #include <linux/errno.h>
   9 #include <linux/interrupt.h>
  10 #include "adf_accel_devices.h"
  11 #include "adf_common_drv.h"
  12 #include "adf_cfg.h"
  13 #include "adf_cfg_strings.h"
  14 #include "adf_cfg_common.h"
  15 #include "adf_transport_access_macros.h"
  16 #include "adf_transport_internal.h"
  17
  18 #define ADF_MAX_NUM_VFS 32
  19
  20 static int adf_enable_msix(struct adf_accel_dev *accel_dev)
  21 {
  22         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
  23         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  24         u32 msix_num_entries = 1;
  25
  26         /* If SR-IOV is disabled, add entries for each bank */
  27         if (!accel_dev->pf.vf_info) {
  28                 int i;
  29
  30                 msix_num_entries += hw_data->num_banks;
  31                 for (i = 0; i < msix_num_entries; i++)
  32                         pci_dev_info->msix_entries.entries[i].entry = i;
  33         } else {
  34                 pci_dev_info->msix_entries.entries[0].entry =
  35                         hw_data->num_banks;
  36         }
  37
  38         if (pci_enable_msix_exact(pci_dev_info->pci_dev,
  39                                   pci_dev_info->msix_entries.entries,
  40                                   msix_num_entries)) {
  41                 dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n");
  42                 return -EFAULT;
  43         }
  44         return 0;
  45 }
  46
  47 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
  48 {
  49         pci_disable_msix(pci_dev_info->pci_dev);
  50 }
  51
  52 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
  53 {
  54         struct adf_etr_bank_data *bank = bank_ptr;
  55
  56         WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
  57         tasklet_hi_schedule(&bank->resp_handler);
  58         return IRQ_HANDLED;
  59 }
  60
  61 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
  62 {
  63         struct adf_accel_dev *accel_dev = dev_ptr;
  64
  65 #ifdef CONFIG_PCI_IOV
  66         /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
  67         if (accel_dev->pf.vf_info) {
  68                 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  69                 struct adf_bar *pmisc =
  70                         &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
  71                 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
  72                 unsigned long vf_mask;
  73
  74                 /* Get the interrupt sources triggered by VFs */
  75                 vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) &
  76                             0x0000FFFF) << 16) |
  77                           ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) &
  78                             0x01FFFE00) >> 9);
  79
  80                 if (vf_mask) {
  81                         struct adf_accel_vf_info *vf_info;
  82                         bool irq_handled = false;
  83                         int i;
  84
  85                         /* Disable VF2PF interrupts for VFs with pending ints */
  86                         adf_disable_vf2pf_interrupts(accel_dev, vf_mask);
  87
  88                         /*
  89                          * Schedule tasklets to handle VF2PF interrupt BHs
  90                          * unless the VF is malicious and is attempting to
  91                          * flood the host OS with VF2PF interrupts.
  92                          */
  93                         for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
  94                                 vf_info = accel_dev->pf.vf_info + i;
  95
  96                                 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
  97                                         dev_info(&GET_DEV(accel_dev),
  98                                                  "Too many ints from VF%d\n",
  99                                                   vf_info->vf_nr + 1);
 100                                         continue;
 101                                 }
 102
 103                                 /* Tasklet will re-enable ints from this VF */
 104                                 tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet);
 105                                 irq_handled = true;
 106                         }
 107
 108                         if (irq_handled)
 109                                 return IRQ_HANDLED;
 110                 }
 111         }
 112 #endif /* CONFIG_PCI_IOV */
 113
 114         dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
 115                 accel_dev->accel_id);
 116
 117         return IRQ_NONE;
 118 }
 119
 120 static int adf_request_irqs(struct adf_accel_dev *accel_dev)
 121 {
 122         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
 123         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 124         struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
 125         struct adf_etr_data *etr_data = accel_dev->transport;
 126         int ret, i = 0;
 127         char *name;
 128
 129         /* Request msix irq for all banks unless SR-IOV enabled */
 130         if (!accel_dev->pf.vf_info) {
 131                 for (i = 0; i < hw_data->num_banks; i++) {
 132                         struct adf_etr_bank_data *bank = &etr_data->banks[i];
 133                         unsigned int cpu, cpus = num_online_cpus();
 134
 135                         name = *(pci_dev_info->msix_entries.names + i);
 136                         snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
 137                                  "qat%d-bundle%d", accel_dev->accel_id, i);
 138                         ret = request_irq(msixe[i].vector,
 139                                           adf_msix_isr_bundle, 0, name, bank);
 140                         if (ret) {
 141                                 dev_err(&GET_DEV(accel_dev),
 142                                         "failed to enable irq %d for %s\n",
 143                                         msixe[i].vector, name);
 144                                 return ret;
 145                         }
 146
 147                         cpu = ((accel_dev->accel_id * hw_data->num_banks) +
 148                                i) % cpus;
 149                         irq_set_affinity_hint(msixe[i].vector,
 150                                               get_cpu_mask(cpu));
 151                 }
 152         }
 153
 154         /* Request msix irq for AE */
 155         name = *(pci_dev_info->msix_entries.names + i);
 156         snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
 157                  "qat%d-ae-cluster", accel_dev->accel_id);
 158         ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev);
 159         if (ret) {
 160                 dev_err(&GET_DEV(accel_dev),
 161                         "failed to enable irq %d, for %s\n",
 162                         msixe[i].vector, name);
 163                 return ret;
 164         }
 165         return ret;
 166 }
 167
 168 static void adf_free_irqs(struct adf_accel_dev *accel_dev)
 169 {
 170         struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
 171         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 172         struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
 173         struct adf_etr_data *etr_data = accel_dev->transport;
 174         int i = 0;
 175
 176         if (pci_dev_info->msix_entries.num_entries > 1) {
 177                 for (i = 0; i < hw_data->num_banks; i++) {
 178                         irq_set_affinity_hint(msixe[i].vector, NULL);
 179                         free_irq(msixe[i].vector, &etr_data->banks[i]);
 180                 }
 181         }
 182         irq_set_affinity_hint(msixe[i].vector, NULL);
 183         free_irq(msixe[i].vector, accel_dev);
 184 }
 185
 186 static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev)
 187 {
 188         int i;
 189         char **names;
 190         struct msix_entry *entries;
 191         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 192         u32 msix_num_entries = 1;
 193
 194         /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
 195         if (!accel_dev->pf.vf_info)
 196                 msix_num_entries += hw_data->num_banks;
 197
 198         entries = kcalloc_node(msix_num_entries, sizeof(*entries),
 199                                GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
 200         if (!entries)
 201                 return -ENOMEM;
 202
 203         names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL);
 204         if (!names) {
 205                 kfree(entries);
 206                 return -ENOMEM;
 207         }
 208         for (i = 0; i < msix_num_entries; i++) {
 209                 *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
 210                 if (!(*(names + i)))
 211                         goto err;
 212         }
 213         accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
 214         accel_dev->accel_pci_dev.msix_entries.entries = entries;
 215         accel_dev->accel_pci_dev.msix_entries.names = names;
 216         return 0;
 217 err:
 218         for (i = 0; i < msix_num_entries; i++)
 219                 kfree(*(names + i));
 220         kfree(entries);
 221         kfree(names);
 222         return -ENOMEM;
 223 }
 224
 225 static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev)
 226 {
 227         char **names = accel_dev->accel_pci_dev.msix_entries.names;
 228         int i;
 229
 230         kfree(accel_dev->accel_pci_dev.msix_entries.entries);
 231         for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++)
 232                 kfree(*(names + i));
 233         kfree(names);
 234 }
 235
 236 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
 237 {
 238         struct adf_etr_data *priv_data = accel_dev->transport;
 239         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 240         int i;
 241
 242         for (i = 0; i < hw_data->num_banks; i++)
 243                 tasklet_init(&priv_data->banks[i].resp_handler,
 244                              adf_response_handler,
 245                              (unsigned long)&priv_data->banks[i]);
 246         return 0;
 247 }
 248
 249 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
 250 {
 251         struct adf_etr_data *priv_data = accel_dev->transport;
 252         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 253         int i;
 254
 255         for (i = 0; i < hw_data->num_banks; i++) {
 256                 tasklet_disable(&priv_data->banks[i].resp_handler);
 257                 tasklet_kill(&priv_data->banks[i].resp_handler);
 258         }
 259 }
 260
 261 /**
 262  * adf_isr_resource_free() - Free IRQ for acceleration device
 263  * @accel_dev:  Pointer to acceleration device.
 264  *
 265  * Function frees interrupts for acceleration device.
 266  */
 267 void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
 268 {
 269         adf_free_irqs(accel_dev);
 270         adf_cleanup_bh(accel_dev);
 271         adf_disable_msix(&accel_dev->accel_pci_dev);
 272         adf_isr_free_msix_entry_table(accel_dev);
 273 }
 274 EXPORT_SYMBOL_GPL(adf_isr_resource_free);
 275
 276 /**
 277  * adf_isr_resource_alloc() - Allocate IRQ for acceleration device
 278  * @accel_dev:  Pointer to acceleration device.
 279  *
 280  * Function allocates interrupts for acceleration device.
 281  *
 282  * Return: 0 on success, error code otherwise.
 283  */
 284 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
 285 {
 286         int ret;
 287
 288         ret = adf_isr_alloc_msix_entry_table(accel_dev);
 289         if (ret)
 290                 goto err_out;
 291
 292         ret = adf_enable_msix(accel_dev);
 293         if (ret)
 294                 goto err_free_msix_table;
 295
 296         ret = adf_setup_bh(accel_dev);
 297         if (ret)
 298                 goto err_disable_msix;
 299
 300         ret = adf_request_irqs(accel_dev);
 301         if (ret)
 302                 goto err_cleanup_bh;
 303
 304         return 0;
 305
 306 err_cleanup_bh:
 307         adf_cleanup_bh(accel_dev);
 308
 309 err_disable_msix:
 310         adf_disable_msix(&accel_dev->accel_pci_dev);
 311
 312 err_free_msix_table:
 313         adf_isr_free_msix_entry_table(accel_dev);
 314
 315 err_out:
 316         return ret;
 317 }
 318 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);