drivers/net/ethernet/fungible/funcore/fun_dev.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2
   3 #include <linux/bitmap.h>
   4 #include <linux/delay.h>
   5 #include <linux/interrupt.h>
   6 #include <linux/io.h>
   7 #include <linux/io-64-nonatomic-lo-hi.h>
   8 #include <linux/mm.h>
   9 #include <linux/module.h>
  10 #include <linux/nvme.h>
  11 #include <linux/pci.h>
  12 #include <linux/wait.h>
  13 #include <linux/sched/signal.h>
  14
  15 #include "fun_queue.h"
  16 #include "fun_dev.h"
  17
  18 #define FUN_ADMIN_CMD_TO_MS 3000
  19
  20 enum {
  21         AQA_ASQS_SHIFT = 0,
  22         AQA_ACQS_SHIFT = 16,
  23         AQA_MIN_QUEUE_SIZE = 2,
  24         AQA_MAX_QUEUE_SIZE = 4096
  25 };
  26
  27 /* context for admin commands */
  28 struct fun_cmd_ctx {
  29         fun_admin_callback_t cb;  /* callback to invoke on completion */
  30         void *cb_data;            /* user data provided to callback */
  31         int cpu;                  /* CPU where the cmd's tag was allocated */
  32 };
  33
  34 /* Context for synchronous admin commands. */
  35 struct fun_sync_cmd_ctx {
  36         struct completion compl;
  37         u8 *rsp_buf;              /* caller provided response buffer */
  38         unsigned int rsp_len;     /* response buffer size */
  39         u8 rsp_status;            /* command response status */
  40 };
  41
  42 /* Wait for the CSTS.RDY bit to match @enabled. */
  43 static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
  44 {
  45         unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
  46         u32 bit = enabled ? NVME_CSTS_RDY : 0;
  47         unsigned long deadline;
  48
  49         deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
  50
  51         for (;;) {
  52                 u32 csts = readl(fdev->bar + NVME_REG_CSTS);
  53
  54                 if (csts == ~0) {
  55                         dev_err(fdev->dev, "CSTS register read %#x\n", csts);
  56                         return -EIO;
  57                 }
  58
  59                 if ((csts & NVME_CSTS_RDY) == bit)
  60                         return 0;
  61
  62                 if (time_is_before_jiffies(deadline))
  63                         break;
  64
  65                 msleep(100);
  66         }
  67
  68         dev_err(fdev->dev,
  69                 "Timed out waiting for device to indicate RDY %u; aborting %s\n",
  70                 enabled, enabled ? "initialization" : "reset");
  71         return -ETIMEDOUT;
  72 }
  73
  74 /* Check CSTS and return an error if it is unreadable or has unexpected
  75  * RDY value.
  76  */
  77 static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
  78 {
  79         u32 csts = readl(fdev->bar + NVME_REG_CSTS);
  80         u32 actual_rdy = csts & NVME_CSTS_RDY;
  81
  82         if (csts == ~0) {
  83                 dev_err(fdev->dev, "CSTS register read %#x\n", csts);
  84                 return -EIO;
  85         }
  86         if (actual_rdy != expected_rdy) {
  87                 dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
  88                 return -EINVAL;
  89         }
  90         return 0;
  91 }
  92
  93 /* Check that CSTS RDY has the expected value. Then write a new value to the CC
  94  * register and wait for CSTS RDY to match the new CC ENABLE state.
  95  */
  96 static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
  97 {
  98         int rc = fun_check_csts_rdy(fdev, initial_rdy);
  99
 100         if (rc)
 101                 return rc;
 102         writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
 103         return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
 104 }
 105
 106 static int fun_disable_ctrl(struct fun_dev *fdev)
 107 {
 108         fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);
 109         return fun_update_cc_enable(fdev, 1);
 110 }
 111
 112 static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
 113                            u32 admin_sqesz_log2)
 114 {
 115         fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |
 116                        (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |
 117                        ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |
 118                        NVME_CC_ENABLE;
 119
 120         return fun_update_cc_enable(fdev, 0);
 121 }
 122
 123 static int fun_map_bars(struct fun_dev *fdev, const char *name)
 124 {
 125         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 126         int err;
 127
 128         err = pci_request_mem_regions(pdev, name);
 129         if (err) {
 130                 dev_err(&pdev->dev,
 131                         "Couldn't get PCI memory resources, err %d\n", err);
 132                 return err;
 133         }
 134
 135         fdev->bar = pci_ioremap_bar(pdev, 0);
 136         if (!fdev->bar) {
 137                 dev_err(&pdev->dev, "Couldn't map BAR 0\n");
 138                 pci_release_mem_regions(pdev);
 139                 return -ENOMEM;
 140         }
 141
 142         return 0;
 143 }
 144
 145 static void fun_unmap_bars(struct fun_dev *fdev)
 146 {
 147         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 148
 149         if (fdev->bar) {
 150                 iounmap(fdev->bar);
 151                 fdev->bar = NULL;
 152                 pci_release_mem_regions(pdev);
 153         }
 154 }
 155
 156 static int fun_set_dma_masks(struct device *dev)
 157 {
 158         int err;
 159
 160         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
 161         if (err)
 162                 dev_err(dev, "DMA mask configuration failed, err %d\n", err);
 163         return err;
 164 }
 165
 166 static irqreturn_t fun_admin_irq(int irq, void *data)
 167 {
 168         struct fun_queue *funq = data;
 169
 170         return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
 171 }
 172
 173 static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,
 174                                    void *entry, const struct fun_cqe_info *info)
 175 {
 176         const struct fun_admin_rsp_common *rsp_common = entry;
 177         struct fun_dev *fdev = funq->fdev;
 178         struct fun_cmd_ctx *cmd_ctx;
 179         int cpu;
 180         u16 cid;
 181
 182         if (info->sqhd == cpu_to_be16(0xffff)) {
 183                 dev_dbg(fdev->dev, "adminq event");
 184                 if (fdev->adminq_cb)
 185                         fdev->adminq_cb(fdev, entry);
 186                 return;
 187         }
 188
 189         cid = be16_to_cpu(rsp_common->cid);
 190         dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
 191                 rsp_common->op, rsp_common->ret);
 192
 193         cmd_ctx = &fdev->cmd_ctx[cid];
 194         if (cmd_ctx->cpu < 0) {
 195                 dev_err(fdev->dev,
 196                         "admin CQE with CID=%u, op=%u does not match a pending command\n",
 197                         cid, rsp_common->op);
 198                 return;
 199         }
 200
 201         if (cmd_ctx->cb)
 202                 cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
 203
 204         cpu = cmd_ctx->cpu;
 205         cmd_ctx->cpu = -1;
 206         sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
 207 }
 208
 209 static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
 210 {
 211         unsigned int i;
 212
 213         fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
 214         if (!fdev->cmd_ctx)
 215                 return -ENOMEM;
 216
 217         for (i = 0; i < ntags; i++)
 218                 fdev->cmd_ctx[i].cpu = -1;
 219
 220         return 0;
 221 }
 222
 223 /* Allocate and enable an admin queue and assign it the first IRQ vector. */
 224 static int fun_enable_admin_queue(struct fun_dev *fdev,
 225                                   const struct fun_dev_params *areq)
 226 {
 227         struct fun_queue_alloc_req qreq = {
 228                 .cqe_size_log2 = areq->cqe_size_log2,
 229                 .sqe_size_log2 = areq->sqe_size_log2,
 230                 .cq_depth = areq->cq_depth,
 231                 .sq_depth = areq->sq_depth,
 232                 .rq_depth = areq->rq_depth,
 233         };
 234         unsigned int ntags = areq->sq_depth - 1;
 235         struct fun_queue *funq;
 236         int rc;
 237
 238         if (fdev->admin_q)
 239                 return -EEXIST;
 240
 241         if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||
 242             areq->sq_depth > AQA_MAX_QUEUE_SIZE ||
 243             areq->cq_depth < AQA_MIN_QUEUE_SIZE ||
 244             areq->cq_depth > AQA_MAX_QUEUE_SIZE)
 245                 return -EINVAL;
 246
 247         fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
 248         if (!fdev->admin_q)
 249                 return -ENOMEM;
 250
 251         rc = fun_init_cmd_ctx(fdev, ntags);
 252         if (rc)
 253                 goto free_q;
 254
 255         rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
 256                                      GFP_KERNEL, dev_to_node(fdev->dev));
 257         if (rc)
 258                 goto free_cmd_ctx;
 259
 260         funq = fdev->admin_q;
 261         funq->cq_vector = 0;
 262         rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
 263         if (rc)
 264                 goto free_sbq;
 265
 266         fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
 267         fdev->adminq_cb = areq->event_cb;
 268
 269         writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |
 270                (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
 271                fdev->bar + NVME_REG_AQA);
 272
 273         writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
 274         writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);
 275
 276         rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
 277         if (rc)
 278                 goto free_irq;
 279
 280         if (areq->rq_depth) {
 281                 rc = fun_create_rq(funq);
 282                 if (rc)
 283                         goto disable_ctrl;
 284
 285                 funq_rq_post(funq);
 286         }
 287
 288         return 0;
 289
 290 disable_ctrl:
 291         fun_disable_ctrl(fdev);
 292 free_irq:
 293         fun_free_irq(funq);
 294 free_sbq:
 295         sbitmap_queue_free(&fdev->admin_sbq);
 296 free_cmd_ctx:
 297         kvfree(fdev->cmd_ctx);
 298         fdev->cmd_ctx = NULL;
 299 free_q:
 300         fun_free_queue(fdev->admin_q);
 301         fdev->admin_q = NULL;
 302         return rc;
 303 }
 304
 305 static void fun_disable_admin_queue(struct fun_dev *fdev)
 306 {
 307         struct fun_queue *admq = fdev->admin_q;
 308
 309         if (!admq)
 310                 return;
 311
 312         fun_disable_ctrl(fdev);
 313
 314         fun_free_irq(admq);
 315         __fun_process_cq(admq, 0);
 316
 317         sbitmap_queue_free(&fdev->admin_sbq);
 318
 319         kvfree(fdev->cmd_ctx);
 320         fdev->cmd_ctx = NULL;
 321
 322         fun_free_queue(admq);
 323         fdev->admin_q = NULL;
 324 }
 325
 326 /* Return %true if the admin queue has stopped servicing commands as can be
 327  * detected through registers. This isn't exhaustive and may provide false
 328  * negatives.
 329  */
 330 static bool fun_adminq_stopped(struct fun_dev *fdev)
 331 {
 332         u32 csts = readl(fdev->bar + NVME_REG_CSTS);
 333
 334         return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;
 335 }
 336
 337 static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)
 338 {
 339         struct sbitmap_queue *sbq = &fdev->admin_sbq;
 340         struct sbq_wait_state *ws = &sbq->ws[0];
 341         DEFINE_SBQ_WAIT(wait);
 342         int tag;
 343
 344         for (;;) {
 345                 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
 346                 if (fdev->suppress_cmds) {
 347                         tag = -ESHUTDOWN;
 348                         break;
 349                 }
 350                 tag = sbitmap_queue_get(sbq, cpup);
 351                 if (tag >= 0)
 352                         break;
 353                 schedule();
 354         }
 355
 356         sbitmap_finish_wait(sbq, ws, &wait);
 357         return tag;
 358 }
 359
 360 /* Submit an asynchronous admin command. Caller is responsible for implementing
 361  * any waiting or timeout. Upon command completion the callback @cb is called.
 362  */
 363 int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,
 364                          fun_admin_callback_t cb, void *cb_data, bool wait_ok)
 365 {
 366         struct fun_queue *funq = fdev->admin_q;
 367         unsigned int cmdsize = cmd->len8 * 8;
 368         struct fun_cmd_ctx *cmd_ctx;
 369         int tag, cpu, rc = 0;
 370
 371         if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
 372                 return -EMSGSIZE;
 373
 374         tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
 375         if (tag < 0) {
 376                 if (!wait_ok)
 377                         return -EAGAIN;
 378                 tag = fun_wait_for_tag(fdev, &cpu);
 379                 if (tag < 0)
 380                         return tag;
 381         }
 382
 383         cmd->cid = cpu_to_be16(tag);
 384
 385         cmd_ctx = &fdev->cmd_ctx[tag];
 386         cmd_ctx->cb = cb;
 387         cmd_ctx->cb_data = cb_data;
 388
 389         spin_lock(&funq->sq_lock);
 390
 391         if (unlikely(fdev->suppress_cmds)) {
 392                 rc = -ESHUTDOWN;
 393                 sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
 394         } else {
 395                 cmd_ctx->cpu = cpu;
 396                 memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
 397
 398                 dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
 399                         cmd);
 400
 401                 if (++funq->sq_tail == funq->sq_depth)
 402                         funq->sq_tail = 0;
 403                 writel(funq->sq_tail, funq->sq_db);
 404         }
 405         spin_unlock(&funq->sq_lock);
 406         return rc;
 407 }
 408
 409 /* Abandon a pending admin command by clearing the issuer's callback data.
 410  * Failure indicates that the command either has already completed or its
 411  * completion is racing with this call.
 412  */
 413 static bool fun_abandon_admin_cmd(struct fun_dev *fd,
 414                                   const struct fun_admin_req_common *cmd,
 415                                   void *cb_data)
 416 {
 417         u16 cid = be16_to_cpu(cmd->cid);
 418         struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
 419
 420         return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
 421 }
 422
 423 /* Stop submission of new admin commands and wake up any processes waiting for
 424  * tags. Already submitted commands are left to complete or time out.
 425  */
 426 static void fun_admin_stop(struct fun_dev *fdev)
 427 {
 428         spin_lock(&fdev->admin_q->sq_lock);
 429         fdev->suppress_cmds = true;
 430         spin_unlock(&fdev->admin_q->sq_lock);
 431         sbitmap_queue_wake_all(&fdev->admin_sbq);
 432 }
 433
 434 /* The callback for synchronous execution of admin commands. It copies the
 435  * command response to the caller's buffer and signals completion.
 436  */
 437 static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)
 438 {
 439         const struct fun_admin_rsp_common *rsp_common = rsp;
 440         struct fun_sync_cmd_ctx *ctx = cb_data;
 441
 442         if (!ctx)
 443                 return;         /* command issuer timed out and left */
 444         if (ctx->rsp_buf) {
 445                 unsigned int rsp_len = rsp_common->len8 * 8;
 446
 447                 if (unlikely(rsp_len > ctx->rsp_len)) {
 448                         dev_err(fd->dev,
 449                                 "response for op %u is %uB > response buffer %uB\n",
 450                                 rsp_common->op, rsp_len, ctx->rsp_len);
 451                         rsp_len = ctx->rsp_len;
 452                 }
 453                 memcpy(ctx->rsp_buf, rsp, rsp_len);
 454         }
 455         ctx->rsp_status = rsp_common->ret;
 456         complete(&ctx->compl);
 457 }
 458
 459 /* Submit a synchronous admin command. */
 460 int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
 461                               struct fun_admin_req_common *cmd, void *rsp,
 462                               size_t rspsize, unsigned int timeout)
 463 {
 464         struct fun_sync_cmd_ctx ctx = {
 465                 .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
 466                 .rsp_buf = rsp,
 467                 .rsp_len = rspsize,
 468         };
 469         unsigned int cmdlen = cmd->len8 * 8;
 470         unsigned long jiffies_left;
 471         int ret;
 472
 473         ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
 474                                    true);
 475         if (ret)
 476                 return ret;
 477
 478         if (!timeout)
 479                 timeout = FUN_ADMIN_CMD_TO_MS;
 480
 481         jiffies_left = wait_for_completion_timeout(&ctx.compl,
 482                                                    msecs_to_jiffies(timeout));
 483         if (!jiffies_left) {
 484                 /* The command timed out. Attempt to cancel it so we can return.
 485                  * But if the command is in the process of completing we'll
 486                  * wait for it.
 487                  */
 488                 if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
 489                         dev_err(fdev->dev, "admin command timed out: %*ph\n",
 490                                 cmdlen, cmd);
 491                         fun_admin_stop(fdev);
 492                         /* see if the timeout was due to a queue failure */
 493                         if (fun_adminq_stopped(fdev))
 494                                 dev_err(fdev->dev,
 495                                         "device does not accept admin commands\n");
 496
 497                         return -ETIMEDOUT;
 498                 }
 499                 wait_for_completion(&ctx.compl);
 500         }
 501
 502         if (ctx.rsp_status) {
 503                 dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
 504                         ctx.rsp_status, cmdlen, cmd);
 505         }
 506
 507         return -ctx.rsp_status;
 508 }
 509 EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
 510
 511 /* Return the number of device resources of the requested type. */
 512 int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
 513 {
 514         union {
 515                 struct fun_admin_res_count_req req;
 516                 struct fun_admin_res_count_rsp rsp;
 517         } cmd;
 518         int rc;
 519
 520         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
 521         cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
 522                                                     0, 0);
 523
 524         rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
 525                                        sizeof(cmd), 0);
 526         return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
 527 }
 528 EXPORT_SYMBOL_GPL(fun_get_res_count);
 529
 530 /* Request that the instance of resource @res with the given id be deleted. */
 531 int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
 532                     unsigned int flags, u32 id)
 533 {
 534         struct fun_admin_generic_destroy_req req = {
 535                 .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
 536                 .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
 537                                                        flags, id)
 538         };
 539
 540         return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
 541 }
 542 EXPORT_SYMBOL_GPL(fun_res_destroy);
 543
 544 /* Bind two entities of the given types and IDs. */
 545 int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
 546              unsigned int id0, enum fun_admin_bind_type type1,
 547              unsigned int id1)
 548 {
 549         struct {
 550                 struct fun_admin_bind_req req;
 551                 struct fun_admin_bind_entry entry[2];
 552         } cmd = {
 553                 .req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
 554                                                          sizeof(cmd)),
 555                 .entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),
 556                 .entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),
 557         };
 558
 559         return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);
 560 }
 561 EXPORT_SYMBOL_GPL(fun_bind);
 562
 563 static int fun_get_dev_limits(struct fun_dev *fdev)
 564 {
 565         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 566         unsigned int cq_count, sq_count, num_dbs;
 567         int rc;
 568
 569         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
 570         if (rc < 0)
 571                 return rc;
 572         cq_count = rc;
 573
 574         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
 575         if (rc < 0)
 576                 return rc;
 577         sq_count = rc;
 578
 579         /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
 580          * device must provide additional queues.
 581          */
 582         if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)
 583                 return -EINVAL;
 584
 585         /* Calculate the max QID based on SQ/CQ/doorbell counts.
 586          * SQ/CQ doorbells alternate.
 587          */
 588         num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >>
 589                   (2 + NVME_CAP_STRIDE(fdev->cap_reg));
 590         fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
 591         fdev->kern_end_qid = fdev->max_qid + 1;
 592         return 0;
 593 }
 594
 595 /* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
 596 static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
 597 {
 598         int vecs, num_msix = pci_msix_vec_count(pdev);
 599
 600         if (num_msix < 0)
 601                 return num_msix;
 602         if (min_vecs > num_msix)
 603                 return -ERANGE;
 604
 605         vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
 606         if (vecs > 0) {
 607                 dev_info(&pdev->dev,
 608                          "Allocated %d IRQ vectors of %d requested\n",
 609                          vecs, num_msix);
 610         } else {
 611                 dev_err(&pdev->dev,
 612                         "Unable to allocate at least %u IRQ vectors\n",
 613                         min_vecs);
 614         }
 615         return vecs;
 616 }
 617
 618 /* Allocate and initialize the IRQ manager state. */
 619 static int fun_alloc_irq_mgr(struct fun_dev *fdev)
 620 {
 621         fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
 622         if (!fdev->irq_map)
 623                 return -ENOMEM;
 624
 625         spin_lock_init(&fdev->irqmgr_lock);
 626         /* mark IRQ 0 allocated, it is used by the admin queue */
 627         __set_bit(0, fdev->irq_map);
 628         fdev->irqs_avail = fdev->num_irqs - 1;
 629         return 0;
 630 }
 631
 632 /* Reserve @nirqs of the currently available IRQs and return their indices. */
 633 int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)
 634 {
 635         unsigned int b, n = 0;
 636         int err = -ENOSPC;
 637
 638         if (!nirqs)
 639                 return 0;
 640
 641         spin_lock(&fdev->irqmgr_lock);
 642         if (nirqs > fdev->irqs_avail)
 643                 goto unlock;
 644
 645         for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
 646                 __set_bit(b, fdev->irq_map);
 647                 irq_indices[n++] = b;
 648                 if (n >= nirqs)
 649                         break;
 650         }
 651
 652         WARN_ON(n < nirqs);
 653         fdev->irqs_avail -= n;
 654         err = n;
 655 unlock:
 656         spin_unlock(&fdev->irqmgr_lock);
 657         return err;
 658 }
 659 EXPORT_SYMBOL(fun_reserve_irqs);
 660
 661 /* Release @nirqs previously allocated IRQS with the supplied indices. */
 662 void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
 663                       u16 *irq_indices)
 664 {
 665         unsigned int i;
 666
 667         spin_lock(&fdev->irqmgr_lock);
 668         for (i = 0; i < nirqs; i++)
 669                 __clear_bit(irq_indices[i], fdev->irq_map);
 670         fdev->irqs_avail += nirqs;
 671         spin_unlock(&fdev->irqmgr_lock);
 672 }
 673 EXPORT_SYMBOL(fun_release_irqs);
 674
 675 static void fun_serv_handler(struct work_struct *work)
 676 {
 677         struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
 678
 679         if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
 680                 return;
 681         if (fd->serv_cb)
 682                 fd->serv_cb(fd);
 683 }
 684
 685 void fun_serv_stop(struct fun_dev *fd)
 686 {
 687         set_bit(FUN_SERV_DISABLED, &fd->service_flags);
 688         cancel_work_sync(&fd->service_task);
 689 }
 690 EXPORT_SYMBOL_GPL(fun_serv_stop);
 691
 692 void fun_serv_restart(struct fun_dev *fd)
 693 {
 694         clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
 695         if (fd->service_flags)
 696                 schedule_work(&fd->service_task);
 697 }
 698 EXPORT_SYMBOL_GPL(fun_serv_restart);
 699
 700 void fun_serv_sched(struct fun_dev *fd)
 701 {
 702         if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
 703                 schedule_work(&fd->service_task);
 704 }
 705 EXPORT_SYMBOL_GPL(fun_serv_sched);
 706
 707 /* Check and try to get the device into a proper state for initialization,
 708  * i.e., CSTS.RDY = CC.EN = 0.
 709  */
 710 static int sanitize_dev(struct fun_dev *fdev)
 711 {
 712         int rc;
 713
 714         fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
 715         fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);
 716
 717         /* First get RDY to agree with the current EN. Give RDY the opportunity
 718          * to complete a potential recent EN change.
 719          */
 720         rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
 721         if (rc)
 722                 return rc;
 723
 724         /* Next, reset the device if EN is currently 1. */
 725         if (fdev->cc_reg & NVME_CC_ENABLE)
 726                 rc = fun_disable_ctrl(fdev);
 727
 728         return rc;
 729 }
 730
 731 /* Undo the device initialization of fun_dev_enable(). */
 732 void fun_dev_disable(struct fun_dev *fdev)
 733 {
 734         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 735
 736         pci_set_drvdata(pdev, NULL);
 737
 738         if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
 739                 fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
 740                                 fdev->fw_handle);
 741                 fdev->fw_handle = FUN_HCI_ID_INVALID;
 742         }
 743
 744         fun_disable_admin_queue(fdev);
 745
 746         bitmap_free(fdev->irq_map);
 747         pci_free_irq_vectors(pdev);
 748
 749         pci_disable_device(pdev);
 750
 751         fun_unmap_bars(fdev);
 752 }
 753 EXPORT_SYMBOL(fun_dev_disable);
 754
 755 /* Perform basic initialization of a device, including
 756  * - PCI config space setup and BAR0 mapping
 757  * - interrupt management initialization
 758  * - 1 admin queue setup
 759  * - determination of some device limits, such as number of queues.
 760  */
 761 int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
 762                    const struct fun_dev_params *areq, const char *name)
 763 {
 764         int rc;
 765
 766         fdev->dev = &pdev->dev;
 767         rc = fun_map_bars(fdev, name);
 768         if (rc)
 769                 return rc;
 770
 771         rc = fun_set_dma_masks(fdev->dev);
 772         if (rc)
 773                 goto unmap;
 774
 775         rc = pci_enable_device_mem(pdev);
 776         if (rc) {
 777                 dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
 778                 goto unmap;
 779         }
 780
 781         rc = sanitize_dev(fdev);
 782         if (rc)
 783                 goto disable_dev;
 784
 785         fdev->fw_handle = FUN_HCI_ID_INVALID;
 786         fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
 787         fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
 788         fdev->dbs = fdev->bar + NVME_REG_DBS;
 789
 790         INIT_WORK(&fdev->service_task, fun_serv_handler);
 791         fdev->service_flags = FUN_SERV_DISABLED;
 792         fdev->serv_cb = areq->serv_cb;
 793
 794         rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
 795         if (rc < 0)
 796                 goto disable_dev;
 797         fdev->num_irqs = rc;
 798
 799         rc = fun_alloc_irq_mgr(fdev);
 800         if (rc)
 801                 goto free_irqs;
 802
 803         pci_set_master(pdev);
 804         rc = fun_enable_admin_queue(fdev, areq);
 805         if (rc)
 806                 goto free_irq_mgr;
 807
 808         rc = fun_get_dev_limits(fdev);
 809         if (rc < 0)
 810                 goto disable_admin;
 811
 812         pci_save_state(pdev);
 813         pci_set_drvdata(pdev, fdev);
 814         pcie_print_link_status(pdev);
 815         dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
 816                 fdev->q_depth, fdev->db_stride, fdev->max_qid,
 817                 fdev->kern_end_qid);
 818         return 0;
 819
 820 disable_admin:
 821         fun_disable_admin_queue(fdev);
 822 free_irq_mgr:
 823         bitmap_free(fdev->irq_map);
 824 free_irqs:
 825         pci_free_irq_vectors(pdev);
 826 disable_dev:
 827         pci_disable_device(pdev);
 828 unmap:
 829         fun_unmap_bars(fdev);
 830         return rc;
 831 }
 832 EXPORT_SYMBOL(fun_dev_enable);
 833
 834 MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
 835 MODULE_DESCRIPTION("Core services driver for Fungible devices");
 836 MODULE_LICENSE("Dual BSD/GPL");