drivers/crypto/qat/qat_common/qat_crypto.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/module.h>
   4 #include <linux/slab.h>
   5 #include "adf_accel_devices.h"
   6 #include "adf_common_drv.h"
   7 #include "adf_transport.h"
   8 #include "adf_transport_access_macros.h"
   9 #include "adf_cfg.h"
  10 #include "adf_cfg_strings.h"
  11 #include "qat_crypto.h"
  12 #include "icp_qat_fw.h"
  13
  14 #define SEC ADF_KERNEL_SEC
  15
  16 static struct service_hndl qat_crypto;
  17
  18 void qat_crypto_put_instance(struct qat_crypto_instance *inst)
  19 {
  20         atomic_dec(&inst->refctr);
  21         adf_dev_put(inst->accel_dev);
  22 }
  23
  24 static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev)
  25 {
  26         struct qat_crypto_instance *inst, *tmp;
  27         int i;
  28
  29         list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) {
  30                 for (i = 0; i < atomic_read(&inst->refctr); i++)
  31                         qat_crypto_put_instance(inst);
  32
  33                 if (inst->sym_tx)
  34                         adf_remove_ring(inst->sym_tx);
  35
  36                 if (inst->sym_rx)
  37                         adf_remove_ring(inst->sym_rx);
  38
  39                 if (inst->pke_tx)
  40                         adf_remove_ring(inst->pke_tx);
  41
  42                 if (inst->pke_rx)
  43                         adf_remove_ring(inst->pke_rx);
  44
  45                 list_del(&inst->list);
  46                 kfree(inst);
  47         }
  48         return 0;
  49 }
  50
  51 struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
  52 {
  53         struct adf_accel_dev *accel_dev = NULL, *tmp_dev;
  54         struct qat_crypto_instance *inst = NULL, *tmp_inst;
  55         unsigned long best = ~0;
  56
  57         list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
  58                 unsigned long ctr;
  59
  60                 if ((node == dev_to_node(&GET_DEV(tmp_dev)) ||
  61                      dev_to_node(&GET_DEV(tmp_dev)) < 0) &&
  62                     adf_dev_started(tmp_dev) &&
  63                     !list_empty(&tmp_dev->crypto_list)) {
  64                         ctr = atomic_read(&tmp_dev->ref_count);
  65                         if (best > ctr) {
  66                                 accel_dev = tmp_dev;
  67                                 best = ctr;
  68                         }
  69                 }
  70         }
  71
  72         if (!accel_dev) {
  73                 pr_info("QAT: Could not find a device on node %d\n", node);
  74                 /* Get any started device */
  75                 list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
  76                         if (adf_dev_started(tmp_dev) &&
  77                             !list_empty(&tmp_dev->crypto_list)) {
  78                                 accel_dev = tmp_dev;
  79                                 break;
  80                         }
  81                 }
  82         }
  83
  84         if (!accel_dev)
  85                 return NULL;
  86
  87         best = ~0;
  88         list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) {
  89                 unsigned long ctr;
  90
  91                 ctr = atomic_read(&tmp_inst->refctr);
  92                 if (best > ctr) {
  93                         inst = tmp_inst;
  94                         best = ctr;
  95                 }
  96         }
  97         if (inst) {
  98                 if (adf_dev_get(accel_dev)) {
  99                         dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");
 100                         return NULL;
 101                 }
 102                 atomic_inc(&inst->refctr);
 103         }
 104         return inst;
 105 }
 106
 107 /**
 108  * qat_crypto_dev_config() - create dev config required to create crypto inst.
 109  *
 110  * @accel_dev: Pointer to acceleration device.
 111  *
 112  * Function creates device configuration required to create crypto instances
 113  *
 114  * Return: 0 on success, error code otherwise.
 115  */
 116 int qat_crypto_dev_config(struct adf_accel_dev *accel_dev)
 117 {
 118         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 119         int banks = GET_MAX_BANKS(accel_dev);
 120         int cpus = num_online_cpus();
 121         unsigned long val;
 122         int instances;
 123         int ret;
 124         int i;
 125
 126         if (adf_hw_dev_has_crypto(accel_dev))
 127                 instances = min(cpus, banks);
 128         else
 129                 instances = 0;
 130
 131         ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
 132         if (ret)
 133                 goto err;
 134
 135         ret = adf_cfg_section_add(accel_dev, "Accelerator0");
 136         if (ret)
 137                 goto err;
 138
 139         /* Temporarily set the number of crypto instances to zero to avoid
 140          * registering the crypto algorithms.
 141          * This will be removed when the algorithms will support the
 142          * CRYPTO_TFM_REQ_MAY_BACKLOG flag
 143          */
 144         instances = 0;
 145
 146         for (i = 0; i < instances; i++) {
 147                 val = i;
 148                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
 149                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 150                                                   key, &val, ADF_DEC);
 151                 if (ret)
 152                         goto err;
 153
 154                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i);
 155                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 156                                                   key, &val, ADF_DEC);
 157                 if (ret)
 158                         goto err;
 159
 160                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
 161                          i);
 162                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 163                                                   key, &val, ADF_DEC);
 164                 if (ret)
 165                         goto err;
 166
 167                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
 168                 val = 128;
 169                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 170                                                   key, &val, ADF_DEC);
 171                 if (ret)
 172                         goto err;
 173
 174                 val = 512;
 175                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
 176                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 177                                                   key, &val, ADF_DEC);
 178                 if (ret)
 179                         goto err;
 180
 181                 val = 0;
 182                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
 183                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 184                                                   key, &val, ADF_DEC);
 185                 if (ret)
 186                         goto err;
 187
 188                 val = 2;
 189                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
 190                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 191                                                   key, &val, ADF_DEC);
 192                 if (ret)
 193                         goto err;
 194
 195                 val = 8;
 196                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
 197                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 198                                                   key, &val, ADF_DEC);
 199                 if (ret)
 200                         goto err;
 201
 202                 val = 10;
 203                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
 204                 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 205                                                   key, &val, ADF_DEC);
 206                 if (ret)
 207                         goto err;
 208
 209                 val = ADF_COALESCING_DEF_TIME;
 210                 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
 211                 ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
 212                                                   key, &val, ADF_DEC);
 213                 if (ret)
 214                         goto err;
 215         }
 216
 217         val = i;
 218         ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY,
 219                                           &val, ADF_DEC);
 220         if (ret)
 221                 goto err;
 222
 223         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 224         return 0;
 225 err:
 226         dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
 227         return ret;
 228 }
 229 EXPORT_SYMBOL_GPL(qat_crypto_dev_config);
 230
 231 static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
 232 {
 233         unsigned long num_inst, num_msg_sym, num_msg_asym;
 234         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 235         char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
 236         unsigned long sym_bank, asym_bank;
 237         struct qat_crypto_instance *inst;
 238         int msg_size;
 239         int ret;
 240         int i;
 241
 242         INIT_LIST_HEAD(&accel_dev->crypto_list);
 243         ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val);
 244         if (ret)
 245                 return ret;
 246
 247         ret = kstrtoul(val, 0, &num_inst);
 248         if (ret)
 249                 return ret;
 250
 251         for (i = 0; i < num_inst; i++) {
 252                 inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,
 253                                     dev_to_node(&GET_DEV(accel_dev)));
 254                 if (!inst) {
 255                         ret = -ENOMEM;
 256                         goto err;
 257                 }
 258
 259                 list_add_tail(&inst->list, &accel_dev->crypto_list);
 260                 inst->id = i;
 261                 atomic_set(&inst->refctr, 0);
 262                 inst->accel_dev = accel_dev;
 263
 264                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i);
 265                 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
 266                 if (ret)
 267                         goto err;
 268
 269                 ret = kstrtoul(val, 10, &sym_bank);
 270                 if (ret)
 271                         goto err;
 272
 273                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
 274                 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
 275                 if (ret)
 276                         goto err;
 277
 278                 ret = kstrtoul(val, 10, &asym_bank);
 279                 if (ret)
 280                         goto err;
 281
 282                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
 283                 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
 284                 if (ret)
 285                         goto err;
 286
 287                 ret = kstrtoul(val, 10, &num_msg_sym);
 288                 if (ret)
 289                         goto err;
 290
 291                 num_msg_sym = num_msg_sym >> 1;
 292
 293                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
 294                 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
 295                 if (ret)
 296                         goto err;
 297
 298                 ret = kstrtoul(val, 10, &num_msg_asym);
 299                 if (ret)
 300                         goto err;
 301                 num_msg_asym = num_msg_asym >> 1;
 302
 303                 msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;
 304                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
 305                 ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
 306                                       msg_size, key, NULL, 0, &inst->sym_tx);
 307                 if (ret)
 308                         goto err;
 309
 310                 msg_size = msg_size >> 1;
 311                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
 312                 ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
 313                                       msg_size, key, NULL, 0, &inst->pke_tx);
 314                 if (ret)
 315                         goto err;
 316
 317                 msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;
 318                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
 319                 ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
 320                                       msg_size, key, qat_alg_callback, 0,
 321                                       &inst->sym_rx);
 322                 if (ret)
 323                         goto err;
 324
 325                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
 326                 ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
 327                                       msg_size, key, qat_alg_asym_callback, 0,
 328                                       &inst->pke_rx);
 329                 if (ret)
 330                         goto err;
 331         }
 332         return 0;
 333 err:
 334         qat_crypto_free_instances(accel_dev);
 335         return ret;
 336 }
 337
 338 static int qat_crypto_init(struct adf_accel_dev *accel_dev)
 339 {
 340         if (qat_crypto_create_instances(accel_dev))
 341                 return -EFAULT;
 342
 343         return 0;
 344 }
 345
 346 static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev)
 347 {
 348         return qat_crypto_free_instances(accel_dev);
 349 }
 350
 351 static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev,
 352                                     enum adf_event event)
 353 {
 354         int ret;
 355
 356         switch (event) {
 357         case ADF_EVENT_INIT:
 358                 ret = qat_crypto_init(accel_dev);
 359                 break;
 360         case ADF_EVENT_SHUTDOWN:
 361                 ret = qat_crypto_shutdown(accel_dev);
 362                 break;
 363         case ADF_EVENT_RESTARTING:
 364         case ADF_EVENT_RESTARTED:
 365         case ADF_EVENT_START:
 366         case ADF_EVENT_STOP:
 367         default:
 368                 ret = 0;
 369         }
 370         return ret;
 371 }
 372
 373 int qat_crypto_register(void)
 374 {
 375         memset(&qat_crypto, 0, sizeof(qat_crypto));
 376         qat_crypto.event_hld = qat_crypto_event_handler;
 377         qat_crypto.name = "qat_crypto";
 378         return adf_service_register(&qat_crypto);
 379 }
 380
 381 int qat_crypto_unregister(void)
 382 {
 383         return adf_service_unregister(&qat_crypto);
 384 }