drivers/crypto/qat/qat_common/adf_init.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/mutex.h>
   4 #include <linux/list.h>
   5 #include <linux/bitops.h>
   6 #include <linux/delay.h>
   7 #include "adf_accel_devices.h"
   8 #include "adf_cfg.h"
   9 #include "adf_common_drv.h"
  10
  11 static LIST_HEAD(service_table);
  12 static DEFINE_MUTEX(service_lock);
  13
  14 static void adf_service_add(struct service_hndl *service)
  15 {
  16         mutex_lock(&service_lock);
  17         list_add(&service->list, &service_table);
  18         mutex_unlock(&service_lock);
  19 }
  20
  21 int adf_service_register(struct service_hndl *service)
  22 {
  23         memset(service->init_status, 0, sizeof(service->init_status));
  24         memset(service->start_status, 0, sizeof(service->start_status));
  25         adf_service_add(service);
  26         return 0;
  27 }
  28
  29 static void adf_service_remove(struct service_hndl *service)
  30 {
  31         mutex_lock(&service_lock);
  32         list_del(&service->list);
  33         mutex_unlock(&service_lock);
  34 }
  35
  36 int adf_service_unregister(struct service_hndl *service)
  37 {
  38         int i;
  39
  40         for (i = 0; i < ARRAY_SIZE(service->init_status); i++) {
  41                 if (service->init_status[i] || service->start_status[i]) {
  42                         pr_err("QAT: Could not remove active service\n");
  43                         return -EFAULT;
  44                 }
  45         }
  46         adf_service_remove(service);
  47         return 0;
  48 }
  49
  50 /**
  51  * adf_dev_init() - Init data structures and services for the given accel device
  52  * @accel_dev: Pointer to acceleration device.
  53  *
  54  * Initialize the ring data structures and the admin comms and arbitration
  55  * services.
  56  *
  57  * Return: 0 on success, error code otherwise.
  58  */
  59 int adf_dev_init(struct adf_accel_dev *accel_dev)
  60 {
  61         struct service_hndl *service;
  62         struct list_head *list_itr;
  63         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  64         int ret;
  65
  66         if (!hw_data) {
  67                 dev_err(&GET_DEV(accel_dev),
  68                         "Failed to init device - hw_data not set\n");
  69                 return -EFAULT;
  70         }
  71
  72         if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
  73                 dev_err(&GET_DEV(accel_dev), "Device not configured\n");
  74                 return -EFAULT;
  75         }
  76
  77         if (adf_init_etr_data(accel_dev)) {
  78                 dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
  79                 return -EFAULT;
  80         }
  81
  82         if (hw_data->init_device && hw_data->init_device(accel_dev)) {
  83                 dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n");
  84                 return -EFAULT;
  85         }
  86
  87         if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
  88                 dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
  89                 return -EFAULT;
  90         }
  91
  92         if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
  93                 dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
  94                 return -EFAULT;
  95         }
  96
  97         if (adf_ae_init(accel_dev)) {
  98                 dev_err(&GET_DEV(accel_dev),
  99                         "Failed to initialise Acceleration Engine\n");
 100                 return -EFAULT;
 101         }
 102         set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
 103
 104         if (adf_ae_fw_load(accel_dev)) {
 105                 dev_err(&GET_DEV(accel_dev),
 106                         "Failed to load acceleration FW\n");
 107                 return -EFAULT;
 108         }
 109         set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
 110
 111         if (hw_data->alloc_irq(accel_dev)) {
 112                 dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
 113                 return -EFAULT;
 114         }
 115         set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
 116
 117         hw_data->enable_ints(accel_dev);
 118         hw_data->enable_error_correction(accel_dev);
 119
 120         ret = hw_data->enable_pfvf_comms(accel_dev);
 121         if (ret)
 122                 return ret;
 123
 124         /*
 125          * Subservice initialisation is divided into two stages: init and start.
 126          * This is to facilitate any ordering dependencies between services
 127          * prior to starting any of the accelerators.
 128          */
 129         list_for_each(list_itr, &service_table) {
 130                 service = list_entry(list_itr, struct service_hndl, list);
 131                 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
 132                         dev_err(&GET_DEV(accel_dev),
 133                                 "Failed to initialise service %s\n",
 134                                 service->name);
 135                         return -EFAULT;
 136                 }
 137                 set_bit(accel_dev->accel_id, service->init_status);
 138         }
 139
 140         return 0;
 141 }
 142 EXPORT_SYMBOL_GPL(adf_dev_init);
 143
 144 /**
 145  * adf_dev_start() - Start acceleration service for the given accel device
 146  * @accel_dev:    Pointer to acceleration device.
 147  *
 148  * Function notifies all the registered services that the acceleration device
 149  * is ready to be used.
 150  * To be used by QAT device specific drivers.
 151  *
 152  * Return: 0 on success, error code otherwise.
 153  */
 154 int adf_dev_start(struct adf_accel_dev *accel_dev)
 155 {
 156         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 157         struct service_hndl *service;
 158         struct list_head *list_itr;
 159
 160         set_bit(ADF_STATUS_STARTING, &accel_dev->status);
 161
 162         if (adf_ae_start(accel_dev)) {
 163                 dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
 164                 return -EFAULT;
 165         }
 166         set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
 167
 168         if (hw_data->send_admin_init(accel_dev)) {
 169                 dev_err(&GET_DEV(accel_dev), "Failed to send init message\n");
 170                 return -EFAULT;
 171         }
 172
 173         /* Set ssm watch dog timer */
 174         if (hw_data->set_ssm_wdtimer)
 175                 hw_data->set_ssm_wdtimer(accel_dev);
 176
 177         list_for_each(list_itr, &service_table) {
 178                 service = list_entry(list_itr, struct service_hndl, list);
 179                 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
 180                         dev_err(&GET_DEV(accel_dev),
 181                                 "Failed to start service %s\n",
 182                                 service->name);
 183                         return -EFAULT;
 184                 }
 185                 set_bit(accel_dev->accel_id, service->start_status);
 186         }
 187
 188         clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
 189         set_bit(ADF_STATUS_STARTED, &accel_dev->status);
 190
 191         if (!list_empty(&accel_dev->crypto_list) &&
 192             (qat_algs_register() || qat_asym_algs_register())) {
 193                 dev_err(&GET_DEV(accel_dev),
 194                         "Failed to register crypto algs\n");
 195                 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
 196                 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
 197                 return -EFAULT;
 198         }
 199         return 0;
 200 }
 201 EXPORT_SYMBOL_GPL(adf_dev_start);
 202
 203 /**
 204  * adf_dev_stop() - Stop acceleration service for the given accel device
 205  * @accel_dev:    Pointer to acceleration device.
 206  *
 207  * Function notifies all the registered services that the acceleration device
 208  * is shuting down.
 209  * To be used by QAT device specific drivers.
 210  *
 211  * Return: void
 212  */
 213 void adf_dev_stop(struct adf_accel_dev *accel_dev)
 214 {
 215         struct service_hndl *service;
 216         struct list_head *list_itr;
 217         bool wait = false;
 218         int ret;
 219
 220         if (!adf_dev_started(accel_dev) &&
 221             !test_bit(ADF_STATUS_STARTING, &accel_dev->status))
 222                 return;
 223
 224         clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
 225         clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
 226
 227         if (!list_empty(&accel_dev->crypto_list)) {
 228                 qat_algs_unregister();
 229                 qat_asym_algs_unregister();
 230         }
 231
 232         list_for_each(list_itr, &service_table) {
 233                 service = list_entry(list_itr, struct service_hndl, list);
 234                 if (!test_bit(accel_dev->accel_id, service->start_status))
 235                         continue;
 236                 ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
 237                 if (!ret) {
 238                         clear_bit(accel_dev->accel_id, service->start_status);
 239                 } else if (ret == -EAGAIN) {
 240                         wait = true;
 241                         clear_bit(accel_dev->accel_id, service->start_status);
 242                 }
 243         }
 244
 245         if (wait)
 246                 msleep(100);
 247
 248         if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
 249                 if (adf_ae_stop(accel_dev))
 250                         dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
 251                 else
 252                         clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
 253         }
 254 }
 255 EXPORT_SYMBOL_GPL(adf_dev_stop);
 256
 257 /**
 258  * adf_dev_shutdown() - shutdown acceleration services and data strucutures
 259  * @accel_dev: Pointer to acceleration device
 260  *
 261  * Cleanup the ring data structures and the admin comms and arbitration
 262  * services.
 263  */
 264 void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
 265 {
 266         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 267         struct service_hndl *service;
 268         struct list_head *list_itr;
 269
 270         if (!hw_data) {
 271                 dev_err(&GET_DEV(accel_dev),
 272                         "QAT: Failed to shutdown device - hw_data not set\n");
 273                 return;
 274         }
 275
 276         if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
 277                 adf_ae_fw_release(accel_dev);
 278                 clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
 279         }
 280
 281         if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
 282                 if (adf_ae_shutdown(accel_dev))
 283                         dev_err(&GET_DEV(accel_dev),
 284                                 "Failed to shutdown Accel Engine\n");
 285                 else
 286                         clear_bit(ADF_STATUS_AE_INITIALISED,
 287                                   &accel_dev->status);
 288         }
 289
 290         list_for_each(list_itr, &service_table) {
 291                 service = list_entry(list_itr, struct service_hndl, list);
 292                 if (!test_bit(accel_dev->accel_id, service->init_status))
 293                         continue;
 294                 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
 295                         dev_err(&GET_DEV(accel_dev),
 296                                 "Failed to shutdown service %s\n",
 297                                 service->name);
 298                 else
 299                         clear_bit(accel_dev->accel_id, service->init_status);
 300         }
 301
 302         hw_data->disable_iov(accel_dev);
 303
 304         if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
 305                 hw_data->free_irq(accel_dev);
 306                 clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
 307         }
 308
 309         /* Delete configuration only if not restarting */
 310         if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
 311                 adf_cfg_del_all(accel_dev);
 312
 313         if (hw_data->exit_arb)
 314                 hw_data->exit_arb(accel_dev);
 315
 316         if (hw_data->exit_admin_comms)
 317                 hw_data->exit_admin_comms(accel_dev);
 318
 319         adf_cleanup_etr_data(accel_dev);
 320         adf_dev_restore(accel_dev);
 321 }
 322 EXPORT_SYMBOL_GPL(adf_dev_shutdown);
 323
 324 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
 325 {
 326         struct service_hndl *service;
 327         struct list_head *list_itr;
 328
 329         list_for_each(list_itr, &service_table) {
 330                 service = list_entry(list_itr, struct service_hndl, list);
 331                 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
 332                         dev_err(&GET_DEV(accel_dev),
 333                                 "Failed to restart service %s.\n",
 334                                 service->name);
 335         }
 336         return 0;
 337 }
 338
 339 int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
 340 {
 341         struct service_hndl *service;
 342         struct list_head *list_itr;
 343
 344         list_for_each(list_itr, &service_table) {
 345                 service = list_entry(list_itr, struct service_hndl, list);
 346                 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
 347                         dev_err(&GET_DEV(accel_dev),
 348                                 "Failed to restart service %s.\n",
 349                                 service->name);
 350         }
 351         return 0;
 352 }