drivers/scsi/snic/snic_io.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright 2014 Cisco Systems, Inc.  All rights reserved.
   3
   4 #include <linux/errno.h>
   5 #include <linux/pci.h>
   6 #include <linux/slab.h>
   7
   8 #include <linux/interrupt.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/spinlock.h>
  11 #include <linux/mempool.h>
  12 #include <scsi/scsi_tcq.h>
  13
  14 #include "snic_io.h"
  15 #include "snic.h"
  16 #include "cq_enet_desc.h"
  17 #include "snic_fwint.h"
  18
  19 static void
  20 snic_wq_cmpl_frame_send(struct vnic_wq *wq,
  21                             struct cq_desc *cq_desc,
  22                             struct vnic_wq_buf *buf,
  23                             void *opaque)
  24 {
  25         struct snic *snic = svnic_dev_priv(wq->vdev);
  26
  27         SNIC_BUG_ON(buf->os_buf == NULL);
  28
  29         if (snic_log_level & SNIC_DESC_LOGGING)
  30                 SNIC_HOST_INFO(snic->shost,
  31                                "Ack received for snic_host_req %p.\n",
  32                                buf->os_buf);
  33
  34         SNIC_TRC(snic->shost->host_no, 0, 0,
  35                  ((ulong)(buf->os_buf) - sizeof(struct snic_req_info)), 0, 0,
  36                  0);
  37
  38         buf->os_buf = NULL;
  39 }
  40
  41 static int
  42 snic_wq_cmpl_handler_cont(struct vnic_dev *vdev,
  43                           struct cq_desc *cq_desc,
  44                           u8 type,
  45                           u16 q_num,
  46                           u16 cmpl_idx,
  47                           void *opaque)
  48 {
  49         struct snic *snic = svnic_dev_priv(vdev);
  50         unsigned long flags;
  51
  52         SNIC_BUG_ON(q_num != 0);
  53
  54         spin_lock_irqsave(&snic->wq_lock[q_num], flags);
  55         svnic_wq_service(&snic->wq[q_num],
  56                          cq_desc,
  57                          cmpl_idx,
  58                          snic_wq_cmpl_frame_send,
  59                          NULL);
  60         spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
  61
  62         return 0;
  63 } /* end of snic_cmpl_handler_cont */
  64
  65 int
  66 snic_wq_cmpl_handler(struct snic *snic, int work_to_do)
  67 {
  68         unsigned int work_done = 0;
  69         unsigned int i;
  70
  71         snic->s_stats.misc.last_ack_time = jiffies;
  72         for (i = 0; i < snic->wq_count; i++) {
  73                 work_done += svnic_cq_service(&snic->cq[i],
  74                                               work_to_do,
  75                                               snic_wq_cmpl_handler_cont,
  76                                               NULL);
  77         }
  78
  79         return work_done;
  80 } /* end of snic_wq_cmpl_handler */
  81
  82 void
  83 snic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
  84 {
  85
  86         struct snic_host_req *req = buf->os_buf;
  87         struct snic *snic = svnic_dev_priv(wq->vdev);
  88         struct snic_req_info *rqi = NULL;
  89         unsigned long flags;
  90
  91         dma_unmap_single(&snic->pdev->dev, buf->dma_addr, buf->len,
  92                          DMA_TO_DEVICE);
  93
  94         rqi = req_to_rqi(req);
  95         spin_lock_irqsave(&snic->spl_cmd_lock, flags);
  96         if (list_empty(&rqi->list)) {
  97                 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
  98                 goto end;
  99         }
 100
 101         SNIC_BUG_ON(rqi->list.next == NULL); /* if not added to spl_cmd_list */
 102         list_del_init(&rqi->list);
 103         spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
 104
 105         if (rqi->sge_va) {
 106                 snic_pci_unmap_rsp_buf(snic, rqi);
 107                 kfree((void *)rqi->sge_va);
 108                 rqi->sge_va = 0;
 109         }
 110         snic_req_free(snic, rqi);
 111         SNIC_HOST_INFO(snic->shost, "snic_free_wq_buf .. freed.\n");
 112
 113 end:
 114         return;
 115 }
 116
 117 /* Criteria to select work queue in multi queue mode */
 118 static int
 119 snic_select_wq(struct snic *snic)
 120 {
 121         /* No multi queue support for now */
 122         BUILD_BUG_ON(SNIC_WQ_MAX > 1);
 123
 124         return 0;
 125 }
 126
 127 static int
 128 snic_wqdesc_avail(struct snic *snic, int q_num, int req_type)
 129 {
 130         int nr_wqdesc = snic->config.wq_enet_desc_count;
 131
 132         if (q_num > 0) {
 133                 /*
 134                  * Multi Queue case, additional care is required.
 135                  * Per WQ active requests need to be maintained.
 136                  */
 137                 SNIC_HOST_INFO(snic->shost, "desc_avail: Multi Queue case.\n");
 138                 SNIC_BUG_ON(q_num > 0);
 139
 140                 return -1;
 141         }
 142
 143         nr_wqdesc -= atomic64_read(&snic->s_stats.fw.actv_reqs);
 144
 145         return ((req_type == SNIC_REQ_HBA_RESET) ? nr_wqdesc : nr_wqdesc - 1);
 146 }
 147
 148 int
 149 snic_queue_wq_desc(struct snic *snic, void *os_buf, u16 len)
 150 {
 151         dma_addr_t pa = 0;
 152         unsigned long flags;
 153         struct snic_fw_stats *fwstats = &snic->s_stats.fw;
 154         struct snic_host_req *req = (struct snic_host_req *) os_buf;
 155         long act_reqs;
 156         long desc_avail = 0;
 157         int q_num = 0;
 158
 159         snic_print_desc(__func__, os_buf, len);
 160
 161         /* Map request buffer */
 162         pa = dma_map_single(&snic->pdev->dev, os_buf, len, DMA_TO_DEVICE);
 163         if (dma_mapping_error(&snic->pdev->dev, pa)) {
 164                 SNIC_HOST_ERR(snic->shost, "qdesc: PCI DMA Mapping Fail.\n");
 165
 166                 return -ENOMEM;
 167         }
 168
 169         req->req_pa = (ulong)pa;
 170
 171         q_num = snic_select_wq(snic);
 172
 173         spin_lock_irqsave(&snic->wq_lock[q_num], flags);
 174         desc_avail = snic_wqdesc_avail(snic, q_num, req->hdr.type);
 175         if (desc_avail <= 0) {
 176                 dma_unmap_single(&snic->pdev->dev, pa, len, DMA_TO_DEVICE);
 177                 req->req_pa = 0;
 178                 spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
 179                 atomic64_inc(&snic->s_stats.misc.wq_alloc_fail);
 180                 SNIC_DBG("host = %d, WQ is Full\n", snic->shost->host_no);
 181
 182                 return -ENOMEM;
 183         }
 184
 185         snic_queue_wq_eth_desc(&snic->wq[q_num], os_buf, pa, len, 0, 0, 1);
 186         /*
 187          * Update stats
 188          * note: when multi queue enabled, fw actv_reqs should be per queue.
 189          */
 190         act_reqs = atomic64_inc_return(&fwstats->actv_reqs);
 191         spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
 192
 193         if (act_reqs > atomic64_read(&fwstats->max_actv_reqs))
 194                 atomic64_set(&fwstats->max_actv_reqs, act_reqs);
 195
 196         return 0;
 197 } /* end of snic_queue_wq_desc() */
 198
 199 /*
 200  * snic_handle_untagged_req: Adds snic specific requests to spl_cmd_list.
 201  * Purpose : Used during driver unload to clean up the requests.
 202  */
 203 void
 204 snic_handle_untagged_req(struct snic *snic, struct snic_req_info *rqi)
 205 {
 206         unsigned long flags;
 207
 208         INIT_LIST_HEAD(&rqi->list);
 209
 210         spin_lock_irqsave(&snic->spl_cmd_lock, flags);
 211         list_add_tail(&rqi->list, &snic->spl_cmd_list);
 212         spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
 213 }
 214
 215 /*
 216  * snic_req_init:
 217  * Allocates snic_req_info + snic_host_req + sgl data, and initializes.
 218  */
 219 struct snic_req_info *
 220 snic_req_init(struct snic *snic, int sg_cnt)
 221 {
 222         u8 typ;
 223         struct snic_req_info *rqi = NULL;
 224
 225         typ = (sg_cnt <= SNIC_REQ_CACHE_DFLT_SGL) ?
 226                 SNIC_REQ_CACHE_DFLT_SGL : SNIC_REQ_CACHE_MAX_SGL;
 227
 228         rqi = mempool_alloc(snic->req_pool[typ], GFP_ATOMIC);
 229         if (!rqi) {
 230                 atomic64_inc(&snic->s_stats.io.alloc_fail);
 231                 SNIC_HOST_ERR(snic->shost,
 232                               "Failed to allocate memory from snic req pool id = %d\n",
 233                               typ);
 234                 return rqi;
 235         }
 236
 237         memset(rqi, 0, sizeof(*rqi));
 238         rqi->rq_pool_type = typ;
 239         rqi->start_time = jiffies;
 240         rqi->req = (struct snic_host_req *) (rqi + 1);
 241         rqi->req_len = sizeof(struct snic_host_req);
 242         rqi->snic = snic;
 243
 244         rqi->req = (struct snic_host_req *)(rqi + 1);
 245
 246         if (sg_cnt == 0)
 247                 goto end;
 248
 249         rqi->req_len += (sg_cnt * sizeof(struct snic_sg_desc));
 250
 251         if (sg_cnt > atomic64_read(&snic->s_stats.io.max_sgl))
 252                 atomic64_set(&snic->s_stats.io.max_sgl, sg_cnt);
 253
 254         SNIC_BUG_ON(sg_cnt > SNIC_MAX_SG_DESC_CNT);
 255         atomic64_inc(&snic->s_stats.io.sgl_cnt[sg_cnt - 1]);
 256
 257 end:
 258         memset(rqi->req, 0, rqi->req_len);
 259
 260         /* pre initialization of init_ctx to support req_to_rqi */
 261         rqi->req->hdr.init_ctx = (ulong) rqi;
 262
 263         SNIC_SCSI_DBG(snic->shost, "Req_alloc:rqi = %p allocatd.\n", rqi);
 264
 265         return rqi;
 266 } /* end of snic_req_init */
 267
 268 /*
 269  * snic_abort_req_init : Inits abort request.
 270  */
 271 struct snic_host_req *
 272 snic_abort_req_init(struct snic *snic, struct snic_req_info *rqi)
 273 {
 274         struct snic_host_req *req = NULL;
 275
 276         SNIC_BUG_ON(!rqi);
 277
 278         /* If abort to be issued second time, then reuse */
 279         if (rqi->abort_req)
 280                 return rqi->abort_req;
 281
 282
 283         req = mempool_alloc(snic->req_pool[SNIC_REQ_TM_CACHE], GFP_ATOMIC);
 284         if (!req) {
 285                 SNIC_HOST_ERR(snic->shost, "abts:Failed to alloc tm req.\n");
 286                 WARN_ON_ONCE(1);
 287
 288                 return NULL;
 289         }
 290
 291         rqi->abort_req = req;
 292         memset(req, 0, sizeof(struct snic_host_req));
 293         /* pre initialization of init_ctx to support req_to_rqi */
 294         req->hdr.init_ctx = (ulong) rqi;
 295
 296         return req;
 297 } /* end of snic_abort_req_init */
 298
 299 /*
 300  * snic_dr_req_init : Inits device reset req
 301  */
 302 struct snic_host_req *
 303 snic_dr_req_init(struct snic *snic, struct snic_req_info *rqi)
 304 {
 305         struct snic_host_req *req = NULL;
 306
 307         SNIC_BUG_ON(!rqi);
 308
 309         req = mempool_alloc(snic->req_pool[SNIC_REQ_TM_CACHE], GFP_ATOMIC);
 310         if (!req) {
 311                 SNIC_HOST_ERR(snic->shost, "dr:Failed to alloc tm req.\n");
 312                 WARN_ON_ONCE(1);
 313
 314                 return NULL;
 315         }
 316
 317         SNIC_BUG_ON(rqi->dr_req != NULL);
 318         rqi->dr_req = req;
 319         memset(req, 0, sizeof(struct snic_host_req));
 320         /* pre initialization of init_ctx to support req_to_rqi */
 321         req->hdr.init_ctx = (ulong) rqi;
 322
 323         return req;
 324 } /* end of snic_dr_req_init */
 325
 326 /* frees snic_req_info and snic_host_req */
 327 void
 328 snic_req_free(struct snic *snic, struct snic_req_info *rqi)
 329 {
 330         SNIC_BUG_ON(rqi->req == rqi->abort_req);
 331         SNIC_BUG_ON(rqi->req == rqi->dr_req);
 332         SNIC_BUG_ON(rqi->sge_va != 0);
 333
 334         SNIC_SCSI_DBG(snic->shost,
 335                       "Req_free:rqi %p:ioreq %p:abt %p:dr %p\n",
 336                       rqi, rqi->req, rqi->abort_req, rqi->dr_req);
 337
 338         if (rqi->abort_req) {
 339                 if (rqi->abort_req->req_pa)
 340                         dma_unmap_single(&snic->pdev->dev,
 341                                          rqi->abort_req->req_pa,
 342                                          sizeof(struct snic_host_req),
 343                                          DMA_TO_DEVICE);
 344
 345                 mempool_free(rqi->abort_req, snic->req_pool[SNIC_REQ_TM_CACHE]);
 346         }
 347
 348         if (rqi->dr_req) {
 349                 if (rqi->dr_req->req_pa)
 350                         dma_unmap_single(&snic->pdev->dev,
 351                                          rqi->dr_req->req_pa,
 352                                          sizeof(struct snic_host_req),
 353                                          DMA_TO_DEVICE);
 354
 355                 mempool_free(rqi->dr_req, snic->req_pool[SNIC_REQ_TM_CACHE]);
 356         }
 357
 358         if (rqi->req->req_pa)
 359                 dma_unmap_single(&snic->pdev->dev,
 360                                  rqi->req->req_pa,
 361                                  rqi->req_len,
 362                                  DMA_TO_DEVICE);
 363
 364         mempool_free(rqi, snic->req_pool[rqi->rq_pool_type]);
 365 }
 366
 367 void
 368 snic_pci_unmap_rsp_buf(struct snic *snic, struct snic_req_info *rqi)
 369 {
 370         struct snic_sg_desc *sgd;
 371
 372         sgd = req_to_sgl(rqi_to_req(rqi));
 373         SNIC_BUG_ON(sgd[0].addr == 0);
 374         dma_unmap_single(&snic->pdev->dev,
 375                          le64_to_cpu(sgd[0].addr),
 376                          le32_to_cpu(sgd[0].len),
 377                          DMA_FROM_DEVICE);
 378 }
 379
 380 /*
 381  * snic_free_all_untagged_reqs: Walks through untagged reqs and frees them.
 382  */
 383 void
 384 snic_free_all_untagged_reqs(struct snic *snic)
 385 {
 386         struct snic_req_info *rqi;
 387         struct list_head *cur, *nxt;
 388         unsigned long flags;
 389
 390         spin_lock_irqsave(&snic->spl_cmd_lock, flags);
 391         list_for_each_safe(cur, nxt, &snic->spl_cmd_list) {
 392                 rqi = list_entry(cur, struct snic_req_info, list);
 393                 list_del_init(&rqi->list);
 394                 if (rqi->sge_va) {
 395                         snic_pci_unmap_rsp_buf(snic, rqi);
 396                         kfree((void *)rqi->sge_va);
 397                         rqi->sge_va = 0;
 398                 }
 399
 400                 snic_req_free(snic, rqi);
 401         }
 402         spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
 403 }
 404
 405 /*
 406  * snic_release_untagged_req : Unlinks the untagged req and frees it.
 407  */
 408 void
 409 snic_release_untagged_req(struct snic *snic, struct snic_req_info *rqi)
 410 {
 411         unsigned long flags;
 412
 413         spin_lock_irqsave(&snic->snic_lock, flags);
 414         if (snic->in_remove) {
 415                 spin_unlock_irqrestore(&snic->snic_lock, flags);
 416                 goto end;
 417         }
 418         spin_unlock_irqrestore(&snic->snic_lock, flags);
 419
 420         spin_lock_irqsave(&snic->spl_cmd_lock, flags);
 421         if (list_empty(&rqi->list)) {
 422                 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
 423                 goto end;
 424         }
 425         list_del_init(&rqi->list);
 426         spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
 427         snic_req_free(snic, rqi);
 428
 429 end:
 430         return;
 431 }
 432
 433 /* dump buf in hex fmt */
 434 void
 435 snic_hex_dump(char *pfx, char *data, int len)
 436 {
 437         SNIC_INFO("%s Dumping Data of Len = %d\n", pfx, len);
 438         print_hex_dump_bytes(pfx, DUMP_PREFIX_NONE, data, len);
 439 }
 440
 441 #define LINE_BUFSZ      128     /* for snic_print_desc fn */
 442 static void
 443 snic_dump_desc(const char *fn, char *os_buf, int len)
 444 {
 445         struct snic_host_req *req = (struct snic_host_req *) os_buf;
 446         struct snic_fw_req *fwreq = (struct snic_fw_req *) os_buf;
 447         struct snic_req_info *rqi = NULL;
 448         char line[LINE_BUFSZ] = { '\0' };
 449         char *cmd_str = NULL;
 450
 451         if (req->hdr.type >= SNIC_RSP_REPORT_TGTS_CMPL)
 452                 rqi = (struct snic_req_info *) fwreq->hdr.init_ctx;
 453         else
 454                 rqi = (struct snic_req_info *) req->hdr.init_ctx;
 455
 456         SNIC_BUG_ON(rqi == NULL || rqi->req == NULL);
 457         switch (req->hdr.type) {
 458         case SNIC_REQ_REPORT_TGTS:
 459                 cmd_str = "report-tgt : ";
 460                 snprintf(line, LINE_BUFSZ, "SNIC_REQ_REPORT_TGTS :");
 461                 break;
 462
 463         case SNIC_REQ_ICMND:
 464                 cmd_str = "icmnd : ";
 465                 snprintf(line, LINE_BUFSZ, "SNIC_REQ_ICMND : 0x%x :",
 466                          req->u.icmnd.cdb[0]);
 467                 break;
 468
 469         case SNIC_REQ_ITMF:
 470                 cmd_str = "itmf : ";
 471                 snprintf(line, LINE_BUFSZ, "SNIC_REQ_ITMF :");
 472                 break;
 473
 474         case SNIC_REQ_HBA_RESET:
 475                 cmd_str = "hba reset :";
 476                 snprintf(line, LINE_BUFSZ, "SNIC_REQ_HBA_RESET :");
 477                 break;
 478
 479         case SNIC_REQ_EXCH_VER:
 480                 cmd_str = "exch ver : ";
 481                 snprintf(line, LINE_BUFSZ, "SNIC_REQ_EXCH_VER :");
 482                 break;
 483
 484         case SNIC_REQ_TGT_INFO:
 485                 cmd_str = "tgt info : ";
 486                 break;
 487
 488         case SNIC_RSP_REPORT_TGTS_CMPL:
 489                 cmd_str = "report tgt cmpl : ";
 490                 snprintf(line, LINE_BUFSZ, "SNIC_RSP_REPORT_TGTS_CMPL :");
 491                 break;
 492
 493         case SNIC_RSP_ICMND_CMPL:
 494                 cmd_str = "icmnd_cmpl : ";
 495                 snprintf(line, LINE_BUFSZ, "SNIC_RSP_ICMND_CMPL : 0x%x :",
 496                          rqi->req->u.icmnd.cdb[0]);
 497                 break;
 498
 499         case SNIC_RSP_ITMF_CMPL:
 500                 cmd_str = "itmf_cmpl : ";
 501                 snprintf(line, LINE_BUFSZ, "SNIC_RSP_ITMF_CMPL :");
 502                 break;
 503
 504         case SNIC_RSP_HBA_RESET_CMPL:
 505                 cmd_str = "hba_reset_cmpl : ";
 506                 snprintf(line, LINE_BUFSZ, "SNIC_RSP_HBA_RESET_CMPL :");
 507                 break;
 508
 509         case SNIC_RSP_EXCH_VER_CMPL:
 510                 cmd_str = "exch_ver_cmpl : ";
 511                 snprintf(line, LINE_BUFSZ, "SNIC_RSP_EXCH_VER_CMPL :");
 512                 break;
 513
 514         case SNIC_MSG_ACK:
 515                 cmd_str = "msg ack : ";
 516                 snprintf(line, LINE_BUFSZ, "SNIC_MSG_ACK :");
 517                 break;
 518
 519         case SNIC_MSG_ASYNC_EVNOTIFY:
 520                 cmd_str = "async notify : ";
 521                 snprintf(line, LINE_BUFSZ, "SNIC_MSG_ASYNC_EVNOTIFY :");
 522                 break;
 523
 524         default:
 525                 cmd_str = "unknown : ";
 526                 SNIC_BUG_ON(1);
 527                 break;
 528         }
 529
 530         SNIC_INFO("%s:%s >>cmndid=%x:sg_cnt = %x:status = %x:ctx = %lx.\n",
 531                   fn, line, req->hdr.cmnd_id, req->hdr.sg_cnt, req->hdr.status,
 532                   req->hdr.init_ctx);
 533
 534         /* Enable it, to dump byte stream */
 535         if (snic_log_level & 0x20)
 536                 snic_hex_dump(cmd_str, os_buf, len);
 537 } /* end of __snic_print_desc */
 538
 539 void
 540 snic_print_desc(const char *fn, char *os_buf, int len)
 541 {
 542         if (snic_log_level & SNIC_DESC_LOGGING)
 543                 snic_dump_desc(fn, os_buf, len);
 544 }
 545
 546 void
 547 snic_calc_io_process_time(struct snic *snic, struct snic_req_info *rqi)
 548 {
 549         u64 duration;
 550
 551         duration = jiffies - rqi->start_time;
 552
 553         if (duration > atomic64_read(&snic->s_stats.io.max_time))
 554                 atomic64_set(&snic->s_stats.io.max_time, duration);
 555 }