drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/compat.h>
  10 #include <linux/delay.h>
  11 #include <linux/errno.h>
  12 #include <linux/hdreg.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/backing-dev.h>
  16 #include <linux/slab.h>
  17 #include <linux/types.h>
  18 #include <linux/pr.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/nvme_ioctl.h>
  21 #include <linux/pm_qos.h>
  22 #include <asm/unaligned.h>
  23
  24 #include "nvme.h"
  25 #include "fabrics.h"
  26
  27 #define CREATE_TRACE_POINTS
  28 #include "trace.h"
  29
  30 #define NVME_MINORS             (1U << MINORBITS)
  31
  32 unsigned int admin_timeout = 60;
  33 module_param(admin_timeout, uint, 0644);
  34 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  35 EXPORT_SYMBOL_GPL(admin_timeout);
  36
  37 unsigned int nvme_io_timeout = 30;
  38 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  39 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  40 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  41
  42 static unsigned char shutdown_timeout = 5;
  43 module_param(shutdown_timeout, byte, 0644);
  44 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  45
  46 static u8 nvme_max_retries = 5;
  47 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  48 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  49
  50 static unsigned long default_ps_max_latency_us = 100000;
  51 module_param(default_ps_max_latency_us, ulong, 0644);
  52 MODULE_PARM_DESC(default_ps_max_latency_us,
  53                  "max power saving latency for new devices; use PM QOS to change per device");
  54
  55 static bool force_apst;
  56 module_param(force_apst, bool, 0644);
  57 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  58
  59 static bool streams;
  60 module_param(streams, bool, 0644);
  61 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  62
  63 /*
  64  * nvme_wq - hosts nvme related works that are not reset or delete
  65  * nvme_reset_wq - hosts nvme reset works
  66  * nvme_delete_wq - hosts nvme delete works
  67  *
  68  * nvme_wq will host works such as scan, aen handling, fw activation,
  69  * keep-alive, periodic reconnects etc. nvme_reset_wq
  70  * runs reset works which also flush works hosted on nvme_wq for
  71  * serialization purposes. nvme_delete_wq host controller deletion
  72  * works which flush reset works for serialization.
  73  */
  74 struct workqueue_struct *nvme_wq;
  75 EXPORT_SYMBOL_GPL(nvme_wq);
  76
  77 struct workqueue_struct *nvme_reset_wq;
  78 EXPORT_SYMBOL_GPL(nvme_reset_wq);
  79
  80 struct workqueue_struct *nvme_delete_wq;
  81 EXPORT_SYMBOL_GPL(nvme_delete_wq);
  82
  83 static LIST_HEAD(nvme_subsystems);
  84 static DEFINE_MUTEX(nvme_subsystems_lock);
  85
  86 static DEFINE_IDA(nvme_instance_ida);
  87 static dev_t nvme_chr_devt;
  88 static struct class *nvme_class;
  89 static struct class *nvme_subsys_class;
  90
  91 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
  92 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  93                                            unsigned nsid);
  94
  95 static void nvme_update_bdev_size(struct gendisk *disk)
  96 {
  97         struct block_device *bdev = bdget_disk(disk, 0);
  98
  99         if (bdev) {
 100                 bd_set_nr_sectors(bdev, get_capacity(disk));
 101                 bdput(bdev);
 102         }
 103 }
 104
 105 /*
 106  * Prepare a queue for teardown.
 107  *
 108  * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
 109  * the capacity to 0 after that to avoid blocking dispatchers that may be
 110  * holding bd_butex.  This will end buffered writers dirtying pages that can't
 111  * be synced.
 112  */
 113 static void nvme_set_queue_dying(struct nvme_ns *ns)
 114 {
 115         if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
 116                 return;
 117
 118         blk_set_queue_dying(ns->queue);
 119         blk_mq_unquiesce_queue(ns->queue);
 120
 121         set_capacity(ns->disk, 0);
 122         nvme_update_bdev_size(ns->disk);
 123 }
 124
 125 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
 126 {
 127         /*
 128          * Only new queue scan work when admin and IO queues are both alive
 129          */
 130         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 131                 queue_work(nvme_wq, &ctrl->scan_work);
 132 }
 133
 134 /*
 135  * Use this function to proceed with scheduling reset_work for a controller
 136  * that had previously been set to the resetting state. This is intended for
 137  * code paths that can't be interrupted by other reset attempts. A hot removal
 138  * may prevent this from succeeding.
 139  */
 140 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 141 {
 142         if (ctrl->state != NVME_CTRL_RESETTING)
 143                 return -EBUSY;
 144         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 145                 return -EBUSY;
 146         return 0;
 147 }
 148 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 149
 150 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 151 {
 152         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 153                 return -EBUSY;
 154         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 155                 return -EBUSY;
 156         return 0;
 157 }
 158 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 159
 160 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 161 {
 162         int ret;
 163
 164         ret = nvme_reset_ctrl(ctrl);
 165         if (!ret) {
 166                 flush_work(&ctrl->reset_work);
 167                 if (ctrl->state != NVME_CTRL_LIVE)
 168                         ret = -ENETRESET;
 169         }
 170
 171         return ret;
 172 }
 173 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
 174
 175 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 176 {
 177         dev_info(ctrl->device,
 178                  "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
 179
 180         flush_work(&ctrl->reset_work);
 181         nvme_stop_ctrl(ctrl);
 182         nvme_remove_namespaces(ctrl);
 183         ctrl->ops->delete_ctrl(ctrl);
 184         nvme_uninit_ctrl(ctrl);
 185 }
 186
 187 static void nvme_delete_ctrl_work(struct work_struct *work)
 188 {
 189         struct nvme_ctrl *ctrl =
 190                 container_of(work, struct nvme_ctrl, delete_work);
 191
 192         nvme_do_delete_ctrl(ctrl);
 193 }
 194
 195 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 196 {
 197         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 198                 return -EBUSY;
 199         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 200                 return -EBUSY;
 201         return 0;
 202 }
 203 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 204
 205 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 206 {
 207         /*
 208          * Keep a reference until nvme_do_delete_ctrl() complete,
 209          * since ->delete_ctrl can free the controller.
 210          */
 211         nvme_get_ctrl(ctrl);
 212         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 213                 nvme_do_delete_ctrl(ctrl);
 214         nvme_put_ctrl(ctrl);
 215 }
 216
 217 static blk_status_t nvme_error_status(u16 status)
 218 {
 219         switch (status & 0x7ff) {
 220         case NVME_SC_SUCCESS:
 221                 return BLK_STS_OK;
 222         case NVME_SC_CAP_EXCEEDED:
 223                 return BLK_STS_NOSPC;
 224         case NVME_SC_LBA_RANGE:
 225         case NVME_SC_CMD_INTERRUPTED:
 226         case NVME_SC_NS_NOT_READY:
 227                 return BLK_STS_TARGET;
 228         case NVME_SC_BAD_ATTRIBUTES:
 229         case NVME_SC_ONCS_NOT_SUPPORTED:
 230         case NVME_SC_INVALID_OPCODE:
 231         case NVME_SC_INVALID_FIELD:
 232         case NVME_SC_INVALID_NS:
 233                 return BLK_STS_NOTSUPP;
 234         case NVME_SC_WRITE_FAULT:
 235         case NVME_SC_READ_ERROR:
 236         case NVME_SC_UNWRITTEN_BLOCK:
 237         case NVME_SC_ACCESS_DENIED:
 238         case NVME_SC_READ_ONLY:
 239         case NVME_SC_COMPARE_FAILED:
 240                 return BLK_STS_MEDIUM;
 241         case NVME_SC_GUARD_CHECK:
 242         case NVME_SC_APPTAG_CHECK:
 243         case NVME_SC_REFTAG_CHECK:
 244         case NVME_SC_INVALID_PI:
 245                 return BLK_STS_PROTECTION;
 246         case NVME_SC_RESERVATION_CONFLICT:
 247                 return BLK_STS_NEXUS;
 248         case NVME_SC_HOST_PATH_ERROR:
 249                 return BLK_STS_TRANSPORT;
 250         case NVME_SC_ZONE_TOO_MANY_ACTIVE:
 251                 return BLK_STS_ZONE_ACTIVE_RESOURCE;
 252         case NVME_SC_ZONE_TOO_MANY_OPEN:
 253                 return BLK_STS_ZONE_OPEN_RESOURCE;
 254         default:
 255                 return BLK_STS_IOERR;
 256         }
 257 }
 258
 259 static void nvme_retry_req(struct request *req)
 260 {
 261         struct nvme_ns *ns = req->q->queuedata;
 262         unsigned long delay = 0;
 263         u16 crd;
 264
 265         /* The mask and shift result must be <= 3 */
 266         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 267         if (ns && crd)
 268                 delay = ns->ctrl->crdt[crd - 1] * 100;
 269
 270         nvme_req(req)->retries++;
 271         blk_mq_requeue_request(req, false);
 272         blk_mq_delay_kick_requeue_list(req->q, delay);
 273 }
 274
 275 enum nvme_disposition {
 276         COMPLETE,
 277         RETRY,
 278         FAILOVER,
 279 };
 280
 281 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 282 {
 283         if (likely(nvme_req(req)->status == 0))
 284                 return COMPLETE;
 285
 286         if (blk_noretry_request(req) ||
 287             (nvme_req(req)->status & NVME_SC_DNR) ||
 288             nvme_req(req)->retries >= nvme_max_retries)
 289                 return COMPLETE;
 290
 291         if (req->cmd_flags & REQ_NVME_MPATH) {
 292                 if (nvme_is_path_error(nvme_req(req)->status) ||
 293                     blk_queue_dying(req->q))
 294                         return FAILOVER;
 295         } else {
 296                 if (blk_queue_dying(req->q))
 297                         return COMPLETE;
 298         }
 299
 300         return RETRY;
 301 }
 302
 303 static inline void nvme_end_req(struct request *req)
 304 {
 305         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 306
 307         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 308             req_op(req) == REQ_OP_ZONE_APPEND)
 309                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 310                         le64_to_cpu(nvme_req(req)->result.u64));
 311
 312         nvme_trace_bio_complete(req, status);
 313         blk_mq_end_request(req, status);
 314 }
 315
 316 void nvme_complete_rq(struct request *req)
 317 {
 318         trace_nvme_complete_rq(req);
 319         nvme_cleanup_cmd(req);
 320
 321         if (nvme_req(req)->ctrl->kas)
 322                 nvme_req(req)->ctrl->comp_seen = true;
 323
 324         switch (nvme_decide_disposition(req)) {
 325         case COMPLETE:
 326                 nvme_end_req(req);
 327                 return;
 328         case RETRY:
 329                 nvme_retry_req(req);
 330                 return;
 331         case FAILOVER:
 332                 nvme_failover_req(req);
 333                 return;
 334         }
 335 }
 336 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 337
 338 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
 339 {
 340         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 341                                 "Cancelling I/O %d", req->tag);
 342
 343         /* don't abort one completed request */
 344         if (blk_mq_request_completed(req))
 345                 return true;
 346
 347         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 348         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 349         blk_mq_complete_request(req);
 350         return true;
 351 }
 352 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 353
 354 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
 355 {
 356         if (ctrl->tagset) {
 357                 blk_mq_tagset_busy_iter(ctrl->tagset,
 358                                 nvme_cancel_request, ctrl);
 359                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
 363
 364 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 365 {
 366         if (ctrl->admin_tagset) {
 367                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
 368                                 nvme_cancel_request, ctrl);
 369                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
 370         }
 371 }
 372 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 373
 374 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 375                 enum nvme_ctrl_state new_state)
 376 {
 377         enum nvme_ctrl_state old_state;
 378         unsigned long flags;
 379         bool changed = false;
 380
 381         spin_lock_irqsave(&ctrl->lock, flags);
 382
 383         old_state = ctrl->state;
 384         switch (new_state) {
 385         case NVME_CTRL_LIVE:
 386                 switch (old_state) {
 387                 case NVME_CTRL_NEW:
 388                 case NVME_CTRL_RESETTING:
 389                 case NVME_CTRL_CONNECTING:
 390                         changed = true;
 391                         fallthrough;
 392                 default:
 393                         break;
 394                 }
 395                 break;
 396         case NVME_CTRL_RESETTING:
 397                 switch (old_state) {
 398                 case NVME_CTRL_NEW:
 399                 case NVME_CTRL_LIVE:
 400                         changed = true;
 401                         fallthrough;
 402                 default:
 403                         break;
 404                 }
 405                 break;
 406         case NVME_CTRL_CONNECTING:
 407                 switch (old_state) {
 408                 case NVME_CTRL_NEW:
 409                 case NVME_CTRL_RESETTING:
 410                         changed = true;
 411                         fallthrough;
 412                 default:
 413                         break;
 414                 }
 415                 break;
 416         case NVME_CTRL_DELETING:
 417                 switch (old_state) {
 418                 case NVME_CTRL_LIVE:
 419                 case NVME_CTRL_RESETTING:
 420                 case NVME_CTRL_CONNECTING:
 421                         changed = true;
 422                         fallthrough;
 423                 default:
 424                         break;
 425                 }
 426                 break;
 427         case NVME_CTRL_DELETING_NOIO:
 428                 switch (old_state) {
 429                 case NVME_CTRL_DELETING:
 430                 case NVME_CTRL_DEAD:
 431                         changed = true;
 432                         fallthrough;
 433                 default:
 434                         break;
 435                 }
 436                 break;
 437         case NVME_CTRL_DEAD:
 438                 switch (old_state) {
 439                 case NVME_CTRL_DELETING:
 440                         changed = true;
 441                         fallthrough;
 442                 default:
 443                         break;
 444                 }
 445                 break;
 446         default:
 447                 break;
 448         }
 449
 450         if (changed) {
 451                 ctrl->state = new_state;
 452                 wake_up_all(&ctrl->state_wq);
 453         }
 454
 455         spin_unlock_irqrestore(&ctrl->lock, flags);
 456         if (changed && ctrl->state == NVME_CTRL_LIVE)
 457                 nvme_kick_requeue_lists(ctrl);
 458         return changed;
 459 }
 460 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 461
 462 /*
 463  * Returns true for sink states that can't ever transition back to live.
 464  */
 465 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 466 {
 467         switch (ctrl->state) {
 468         case NVME_CTRL_NEW:
 469         case NVME_CTRL_LIVE:
 470         case NVME_CTRL_RESETTING:
 471         case NVME_CTRL_CONNECTING:
 472                 return false;
 473         case NVME_CTRL_DELETING:
 474         case NVME_CTRL_DELETING_NOIO:
 475         case NVME_CTRL_DEAD:
 476                 return true;
 477         default:
 478                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 479                 return true;
 480         }
 481 }
 482
 483 /*
 484  * Waits for the controller state to be resetting, or returns false if it is
 485  * not possible to ever transition to that state.
 486  */
 487 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 488 {
 489         wait_event(ctrl->state_wq,
 490                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 491                    nvme_state_terminal(ctrl));
 492         return ctrl->state == NVME_CTRL_RESETTING;
 493 }
 494 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 495
 496 static void nvme_free_ns_head(struct kref *ref)
 497 {
 498         struct nvme_ns_head *head =
 499                 container_of(ref, struct nvme_ns_head, ref);
 500
 501         nvme_mpath_remove_disk(head);
 502         ida_simple_remove(&head->subsys->ns_ida, head->instance);
 503         cleanup_srcu_struct(&head->srcu);
 504         nvme_put_subsystem(head->subsys);
 505         kfree(head);
 506 }
 507
 508 static void nvme_put_ns_head(struct nvme_ns_head *head)
 509 {
 510         kref_put(&head->ref, nvme_free_ns_head);
 511 }
 512
 513 static void nvme_free_ns(struct kref *kref)
 514 {
 515         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 516
 517         if (ns->ndev)
 518                 nvme_nvm_unregister(ns);
 519
 520         put_disk(ns->disk);
 521         nvme_put_ns_head(ns->head);
 522         nvme_put_ctrl(ns->ctrl);
 523         kfree(ns);
 524 }
 525
 526 void nvme_put_ns(struct nvme_ns *ns)
 527 {
 528         kref_put(&ns->kref, nvme_free_ns);
 529 }
 530 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 531
 532 static inline void nvme_clear_nvme_request(struct request *req)
 533 {
 534         nvme_req(req)->retries = 0;
 535         nvme_req(req)->flags = 0;
 536         req->rq_flags |= RQF_DONTPREP;
 537 }
 538
 539 static inline unsigned int nvme_req_op(struct nvme_command *cmd)
 540 {
 541         return nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 542 }
 543
 544 static inline void nvme_init_request(struct request *req,
 545                 struct nvme_command *cmd)
 546 {
 547         if (req->q->queuedata)
 548                 req->timeout = NVME_IO_TIMEOUT;
 549         else /* no queuedata implies admin queue */
 550                 req->timeout = ADMIN_TIMEOUT;
 551
 552         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 553         nvme_clear_nvme_request(req);
 554         nvme_req(req)->cmd = cmd;
 555 }
 556
 557 struct request *nvme_alloc_request(struct request_queue *q,
 558                 struct nvme_command *cmd, blk_mq_req_flags_t flags)
 559 {
 560         struct request *req;
 561
 562         req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
 563         if (!IS_ERR(req))
 564                 nvme_init_request(req, cmd);
 565         return req;
 566 }
 567 EXPORT_SYMBOL_GPL(nvme_alloc_request);
 568
 569 struct request *nvme_alloc_request_qid(struct request_queue *q,
 570                 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
 571 {
 572         struct request *req;
 573
 574         req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
 575                         qid ? qid - 1 : 0);
 576         if (!IS_ERR(req))
 577                 nvme_init_request(req, cmd);
 578         return req;
 579 }
 580 EXPORT_SYMBOL_GPL(nvme_alloc_request_qid);
 581
 582 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 583 {
 584         struct nvme_command c;
 585
 586         memset(&c, 0, sizeof(c));
 587
 588         c.directive.opcode = nvme_admin_directive_send;
 589         c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 590         c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 591         c.directive.dtype = NVME_DIR_IDENTIFY;
 592         c.directive.tdtype = NVME_DIR_STREAMS;
 593         c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
 594
 595         return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
 596 }
 597
 598 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
 599 {
 600         return nvme_toggle_streams(ctrl, false);
 601 }
 602
 603 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
 604 {
 605         return nvme_toggle_streams(ctrl, true);
 606 }
 607
 608 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
 609                                   struct streams_directive_params *s, u32 nsid)
 610 {
 611         struct nvme_command c;
 612
 613         memset(&c, 0, sizeof(c));
 614         memset(s, 0, sizeof(*s));
 615
 616         c.directive.opcode = nvme_admin_directive_recv;
 617         c.directive.nsid = cpu_to_le32(nsid);
 618         c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
 619         c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
 620         c.directive.dtype = NVME_DIR_STREAMS;
 621
 622         return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
 623 }
 624
 625 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 626 {
 627         struct streams_directive_params s;
 628         int ret;
 629
 630         if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
 631                 return 0;
 632         if (!streams)
 633                 return 0;
 634
 635         ret = nvme_enable_streams(ctrl);
 636         if (ret)
 637                 return ret;
 638
 639         ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 640         if (ret)
 641                 goto out_disable_stream;
 642
 643         ctrl->nssa = le16_to_cpu(s.nssa);
 644         if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
 645                 dev_info(ctrl->device, "too few streams (%u) available\n",
 646                                         ctrl->nssa);
 647                 goto out_disable_stream;
 648         }
 649
 650         ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
 651         dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
 652         return 0;
 653
 654 out_disable_stream:
 655         nvme_disable_streams(ctrl);
 656         return ret;
 657 }
 658
 659 /*
 660  * Check if 'req' has a write hint associated with it. If it does, assign
 661  * a valid namespace stream to the write.
 662  */
 663 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
 664                                      struct request *req, u16 *control,
 665                                      u32 *dsmgmt)
 666 {
 667         enum rw_hint streamid = req->write_hint;
 668
 669         if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
 670                 streamid = 0;
 671         else {
 672                 streamid--;
 673                 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
 674                         return;
 675
 676                 *control |= NVME_RW_DTYPE_STREAMS;
 677                 *dsmgmt |= streamid << 16;
 678         }
 679
 680         if (streamid < ARRAY_SIZE(req->q->write_hints))
 681                 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
 682 }
 683
 684 static inline void nvme_setup_passthrough(struct request *req,
 685                 struct nvme_command *cmd)
 686 {
 687         memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 688         /* passthru commands should let the driver set the SGL flags */
 689         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 690 }
 691
 692 static inline void nvme_setup_flush(struct nvme_ns *ns,
 693                 struct nvme_command *cmnd)
 694 {
 695         cmnd->common.opcode = nvme_cmd_flush;
 696         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 697 }
 698
 699 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 700                 struct nvme_command *cmnd)
 701 {
 702         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 703         struct nvme_dsm_range *range;
 704         struct bio *bio;
 705
 706         /*
 707          * Some devices do not consider the DSM 'Number of Ranges' field when
 708          * determining how much data to DMA. Always allocate memory for maximum
 709          * number of segments to prevent device reading beyond end of buffer.
 710          */
 711         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 712
 713         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 714         if (!range) {
 715                 /*
 716                  * If we fail allocation our range, fallback to the controller
 717                  * discard page. If that's also busy, it's safe to return
 718                  * busy, as we know we can make progress once that's freed.
 719                  */
 720                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 721                         return BLK_STS_RESOURCE;
 722
 723                 range = page_address(ns->ctrl->discard_page);
 724         }
 725
 726         if (queue_max_discard_segments(req->q) == 1) {
 727                 u64 slba = nvme_sect_to_lba(ns, blk_rq_pos(req));
 728                 u32 nlb = blk_rq_sectors(req) >> (ns->lba_shift - 9);
 729
 730                 range[0].cattr = cpu_to_le32(0);
 731                 range[0].nlb = cpu_to_le32(nlb);
 732                 range[0].slba = cpu_to_le64(slba);
 733                 n = 1;
 734         } else {
 735                 __rq_for_each_bio(bio, req) {
 736                         u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 737                         u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 738
 739                         if (n < segments) {
 740                                 range[n].cattr = cpu_to_le32(0);
 741                                 range[n].nlb = cpu_to_le32(nlb);
 742                                 range[n].slba = cpu_to_le64(slba);
 743                         }
 744                         n++;
 745                 }
 746         }
 747
 748         if (WARN_ON_ONCE(n != segments)) {
 749                 if (virt_to_page(range) == ns->ctrl->discard_page)
 750                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 751                 else
 752                         kfree(range);
 753                 return BLK_STS_IOERR;
 754         }
 755
 756         cmnd->dsm.opcode = nvme_cmd_dsm;
 757         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 758         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 759         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 760
 761         req->special_vec.bv_page = virt_to_page(range);
 762         req->special_vec.bv_offset = offset_in_page(range);
 763         req->special_vec.bv_len = alloc_size;
 764         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 765
 766         return BLK_STS_OK;
 767 }
 768
 769 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 770                 struct request *req, struct nvme_command *cmnd)
 771 {
 772         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 773                 return nvme_setup_discard(ns, req, cmnd);
 774
 775         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 776         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 777         cmnd->write_zeroes.slba =
 778                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 779         cmnd->write_zeroes.length =
 780                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 781         if (nvme_ns_has_pi(ns))
 782                 cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);
 783         else
 784                 cmnd->write_zeroes.control = 0;
 785         return BLK_STS_OK;
 786 }
 787
 788 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 789                 struct request *req, struct nvme_command *cmnd,
 790                 enum nvme_opcode op)
 791 {
 792         struct nvme_ctrl *ctrl = ns->ctrl;
 793         u16 control = 0;
 794         u32 dsmgmt = 0;
 795
 796         if (req->cmd_flags & REQ_FUA)
 797                 control |= NVME_RW_FUA;
 798         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 799                 control |= NVME_RW_LR;
 800
 801         if (req->cmd_flags & REQ_RAHEAD)
 802                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 803
 804         cmnd->rw.opcode = op;
 805         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 806         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 807         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 808
 809         if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
 810                 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
 811
 812         if (ns->ms) {
 813                 /*
 814                  * If formated with metadata, the block layer always provides a
 815                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 816                  * we enable the PRACT bit for protection information or set the
 817                  * namespace capacity to zero to prevent any I/O.
 818                  */
 819                 if (!blk_integrity_rq(req)) {
 820                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 821                                 return BLK_STS_NOTSUPP;
 822                         control |= NVME_RW_PRINFO_PRACT;
 823                 }
 824
 825                 switch (ns->pi_type) {
 826                 case NVME_NS_DPS_PI_TYPE3:
 827                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 828                         break;
 829                 case NVME_NS_DPS_PI_TYPE1:
 830                 case NVME_NS_DPS_PI_TYPE2:
 831                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 832                                         NVME_RW_PRINFO_PRCHK_REF;
 833                         if (op == nvme_cmd_zone_append)
 834                                 control |= NVME_RW_APPEND_PIREMAP;
 835                         cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 836                         break;
 837                 }
 838         }
 839
 840         cmnd->rw.control = cpu_to_le16(control);
 841         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 842         return 0;
 843 }
 844
 845 void nvme_cleanup_cmd(struct request *req)
 846 {
 847         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 848                 struct nvme_ns *ns = req->rq_disk->private_data;
 849                 struct page *page = req->special_vec.bv_page;
 850
 851                 if (page == ns->ctrl->discard_page)
 852                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 853                 else
 854                         kfree(page_address(page) + req->special_vec.bv_offset);
 855         }
 856 }
 857 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 858
 859 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 860                 struct nvme_command *cmd)
 861 {
 862         struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 863         blk_status_t ret = BLK_STS_OK;
 864
 865         if (!(req->rq_flags & RQF_DONTPREP))
 866                 nvme_clear_nvme_request(req);
 867
 868         memset(cmd, 0, sizeof(*cmd));
 869         switch (req_op(req)) {
 870         case REQ_OP_DRV_IN:
 871         case REQ_OP_DRV_OUT:
 872                 nvme_setup_passthrough(req, cmd);
 873                 break;
 874         case REQ_OP_FLUSH:
 875                 nvme_setup_flush(ns, cmd);
 876                 break;
 877         case REQ_OP_ZONE_RESET_ALL:
 878         case REQ_OP_ZONE_RESET:
 879                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 880                 break;
 881         case REQ_OP_ZONE_OPEN:
 882                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 883                 break;
 884         case REQ_OP_ZONE_CLOSE:
 885                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 886                 break;
 887         case REQ_OP_ZONE_FINISH:
 888                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 889                 break;
 890         case REQ_OP_WRITE_ZEROES:
 891                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 892                 break;
 893         case REQ_OP_DISCARD:
 894                 ret = nvme_setup_discard(ns, req, cmd);
 895                 break;
 896         case REQ_OP_READ:
 897                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 898                 break;
 899         case REQ_OP_WRITE:
 900                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 901                 break;
 902         case REQ_OP_ZONE_APPEND:
 903                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 904                 break;
 905         default:
 906                 WARN_ON_ONCE(1);
 907                 return BLK_STS_IOERR;
 908         }
 909
 910         if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
 911                 nvme_req(req)->genctr++;
 912         cmd->common.command_id = nvme_cid(req);
 913         trace_nvme_setup_cmd(req, cmd);
 914         return ret;
 915 }
 916 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 917
 918 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
 919 {
 920         struct completion *waiting = rq->end_io_data;
 921
 922         rq->end_io_data = NULL;
 923         complete(waiting);
 924 }
 925
 926 static void nvme_execute_rq_polled(struct request_queue *q,
 927                 struct gendisk *bd_disk, struct request *rq, int at_head)
 928 {
 929         DECLARE_COMPLETION_ONSTACK(wait);
 930
 931         WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
 932
 933         rq->cmd_flags |= REQ_HIPRI;
 934         rq->end_io_data = &wait;
 935         blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
 936
 937         while (!completion_done(&wait)) {
 938                 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
 939                 cond_resched();
 940         }
 941 }
 942
 943 /*
 944  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 945  * if the result is positive, it's an NVM Express status code
 946  */
 947 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 948                 union nvme_result *result, void *buffer, unsigned bufflen,
 949                 unsigned timeout, int qid, int at_head,
 950                 blk_mq_req_flags_t flags, bool poll)
 951 {
 952         struct request *req;
 953         int ret;
 954
 955         if (qid == NVME_QID_ANY)
 956                 req = nvme_alloc_request(q, cmd, flags);
 957         else
 958                 req = nvme_alloc_request_qid(q, cmd, flags, qid);
 959         if (IS_ERR(req))
 960                 return PTR_ERR(req);
 961
 962         if (timeout)
 963                 req->timeout = timeout;
 964
 965         if (buffer && bufflen) {
 966                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
 967                 if (ret)
 968                         goto out;
 969         }
 970
 971         if (poll)
 972                 nvme_execute_rq_polled(req->q, NULL, req, at_head);
 973         else
 974                 blk_execute_rq(req->q, NULL, req, at_head);
 975         if (result)
 976                 *result = nvme_req(req)->result;
 977         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 978                 ret = -EINTR;
 979         else
 980                 ret = nvme_req(req)->status;
 981  out:
 982         blk_mq_free_request(req);
 983         return ret;
 984 }
 985 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
 986
 987 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 988                 void *buffer, unsigned bufflen)
 989 {
 990         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
 991                         NVME_QID_ANY, 0, 0, false);
 992 }
 993 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 994
 995 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
 996                 unsigned len, u32 seed, bool write)
 997 {
 998         struct bio_integrity_payload *bip;
 999         int ret = -ENOMEM;
1000         void *buf;
1001
1002         buf = kmalloc(len, GFP_KERNEL);
1003         if (!buf)
1004                 goto out;
1005
1006         ret = -EFAULT;
1007         if (write && copy_from_user(buf, ubuf, len))
1008                 goto out_free_meta;
1009
1010         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
1011         if (IS_ERR(bip)) {
1012                 ret = PTR_ERR(bip);
1013                 goto out_free_meta;
1014         }
1015
1016         bip->bip_iter.bi_size = len;
1017         bip->bip_iter.bi_sector = seed;
1018         ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
1019                         offset_in_page(buf));
1020         if (ret == len)
1021                 return buf;
1022         ret = -ENOMEM;
1023 out_free_meta:
1024         kfree(buf);
1025 out:
1026         return ERR_PTR(ret);
1027 }
1028
1029 static u32 nvme_known_admin_effects(u8 opcode)
1030 {
1031         switch (opcode) {
1032         case nvme_admin_format_nvm:
1033                 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
1034                         NVME_CMD_EFFECTS_CSE_MASK;
1035         case nvme_admin_sanitize_nvm:
1036                 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
1037         default:
1038                 break;
1039         }
1040         return 0;
1041 }
1042
1043 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1044 {
1045         u32 effects = 0;
1046
1047         if (ns) {
1048                 if (ns->head->effects)
1049                         effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1050                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1051                         dev_warn(ctrl->device,
1052                                  "IO command:%02x has unhandled effects:%08x\n",
1053                                  opcode, effects);
1054                 return 0;
1055         }
1056
1057         if (ctrl->effects)
1058                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1059         effects |= nvme_known_admin_effects(opcode);
1060
1061         return effects;
1062 }
1063 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1064
1065 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1066                                u8 opcode)
1067 {
1068         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1069
1070         /*
1071          * For simplicity, IO to all namespaces is quiesced even if the command
1072          * effects say only one namespace is affected.
1073          */
1074         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1075                 mutex_lock(&ctrl->scan_lock);
1076                 mutex_lock(&ctrl->subsys->lock);
1077                 nvme_mpath_start_freeze(ctrl->subsys);
1078                 nvme_mpath_wait_freeze(ctrl->subsys);
1079                 nvme_start_freeze(ctrl);
1080                 nvme_wait_freeze(ctrl);
1081         }
1082         return effects;
1083 }
1084
1085 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1086 {
1087         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1088                 nvme_unfreeze(ctrl);
1089                 nvme_mpath_unfreeze(ctrl->subsys);
1090                 mutex_unlock(&ctrl->subsys->lock);
1091                 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1092                 mutex_unlock(&ctrl->scan_lock);
1093         }
1094         if (effects & NVME_CMD_EFFECTS_CCC)
1095                 nvme_init_identify(ctrl);
1096         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1097                 nvme_queue_scan(ctrl);
1098                 flush_work(&ctrl->scan_work);
1099         }
1100 }
1101
1102 void nvme_execute_passthru_rq(struct request *rq)
1103 {
1104         struct nvme_command *cmd = nvme_req(rq)->cmd;
1105         struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1106         struct nvme_ns *ns = rq->q->queuedata;
1107         struct gendisk *disk = ns ? ns->disk : NULL;
1108         u32 effects;
1109
1110         effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1111         blk_execute_rq(rq->q, disk, rq, 0);
1112         nvme_passthru_end(ctrl, effects);
1113 }
1114 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1115
1116 static int nvme_submit_user_cmd(struct request_queue *q,
1117                 struct nvme_command *cmd, void __user *ubuffer,
1118                 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
1119                 u32 meta_seed, u64 *result, unsigned timeout)
1120 {
1121         bool write = nvme_is_write(cmd);
1122         struct nvme_ns *ns = q->queuedata;
1123         struct gendisk *disk = ns ? ns->disk : NULL;
1124         struct request *req;
1125         struct bio *bio = NULL;
1126         void *meta = NULL;
1127         int ret;
1128
1129         req = nvme_alloc_request(q, cmd, 0);
1130         if (IS_ERR(req))
1131                 return PTR_ERR(req);
1132
1133         if (timeout)
1134                 req->timeout = timeout;
1135         nvme_req(req)->flags |= NVME_REQ_USERCMD;
1136
1137         if (ubuffer && bufflen) {
1138                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1139                                 GFP_KERNEL);
1140                 if (ret)
1141                         goto out;
1142                 bio = req->bio;
1143                 bio->bi_disk = disk;
1144                 if (disk && meta_buffer && meta_len) {
1145                         meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1146                                         meta_seed, write);
1147                         if (IS_ERR(meta)) {
1148                                 ret = PTR_ERR(meta);
1149                                 goto out_unmap;
1150                         }
1151                         req->cmd_flags |= REQ_INTEGRITY;
1152                 }
1153         }
1154
1155         nvme_execute_passthru_rq(req);
1156         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1157                 ret = -EINTR;
1158         else
1159                 ret = nvme_req(req)->status;
1160         if (result)
1161                 *result = le64_to_cpu(nvme_req(req)->result.u64);
1162         if (meta && !ret && !write) {
1163                 if (copy_to_user(meta_buffer, meta, meta_len))
1164                         ret = -EFAULT;
1165         }
1166         kfree(meta);
1167  out_unmap:
1168         if (bio)
1169                 blk_rq_unmap_user(bio);
1170  out:
1171         blk_mq_free_request(req);
1172         return ret;
1173 }
1174
1175 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1176 {
1177         struct nvme_ctrl *ctrl = rq->end_io_data;
1178         unsigned long flags;
1179         bool startka = false;
1180
1181         blk_mq_free_request(rq);
1182
1183         if (status) {
1184                 dev_err(ctrl->device,
1185                         "failed nvme_keep_alive_end_io error=%d\n",
1186                                 status);
1187                 return;
1188         }
1189
1190         ctrl->comp_seen = false;
1191         spin_lock_irqsave(&ctrl->lock, flags);
1192         if (ctrl->state == NVME_CTRL_LIVE ||
1193             ctrl->state == NVME_CTRL_CONNECTING)
1194                 startka = true;
1195         spin_unlock_irqrestore(&ctrl->lock, flags);
1196         if (startka)
1197                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1198 }
1199
1200 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1201 {
1202         struct request *rq;
1203
1204         rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd,
1205                         BLK_MQ_REQ_RESERVED);
1206         if (IS_ERR(rq))
1207                 return PTR_ERR(rq);
1208
1209         rq->timeout = ctrl->kato * HZ;
1210         rq->end_io_data = ctrl;
1211
1212         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1213
1214         return 0;
1215 }
1216
1217 static void nvme_keep_alive_work(struct work_struct *work)
1218 {
1219         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1220                         struct nvme_ctrl, ka_work);
1221         bool comp_seen = ctrl->comp_seen;
1222
1223         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1224                 dev_dbg(ctrl->device,
1225                         "reschedule traffic based keep-alive timer\n");
1226                 ctrl->comp_seen = false;
1227                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1228                 return;
1229         }
1230
1231         if (nvme_keep_alive(ctrl)) {
1232                 /* allocation failure, reset the controller */
1233                 dev_err(ctrl->device, "keep-alive failed\n");
1234                 nvme_reset_ctrl(ctrl);
1235                 return;
1236         }
1237 }
1238
1239 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1240 {
1241         if (unlikely(ctrl->kato == 0))
1242                 return;
1243
1244         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1245 }
1246
1247 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1248 {
1249         if (unlikely(ctrl->kato == 0))
1250                 return;
1251
1252         cancel_delayed_work_sync(&ctrl->ka_work);
1253 }
1254 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1255
1256 /*
1257  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1258  * flag, thus sending any new CNS opcodes has a big chance of not working.
1259  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1260  * (but not for any later version).
1261  */
1262 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1263 {
1264         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1265                 return ctrl->vs < NVME_VS(1, 2, 0);
1266         return ctrl->vs < NVME_VS(1, 1, 0);
1267 }
1268
1269 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1270 {
1271         struct nvme_command c = { };
1272         int error;
1273
1274         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1275         c.identify.opcode = nvme_admin_identify;
1276         c.identify.cns = NVME_ID_CNS_CTRL;
1277
1278         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1279         if (!*id)
1280                 return -ENOMEM;
1281
1282         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1283                         sizeof(struct nvme_id_ctrl));
1284         if (error)
1285                 kfree(*id);
1286         return error;
1287 }
1288
1289 static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1290 {
1291         return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1292 }
1293
1294 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1295                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1296 {
1297         const char *warn_str = "ctrl returned bogus length:";
1298         void *data = cur;
1299
1300         switch (cur->nidt) {
1301         case NVME_NIDT_EUI64:
1302                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1303                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1304                                  warn_str, cur->nidl);
1305                         return -1;
1306                 }
1307                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1308                         return NVME_NIDT_EUI64_LEN;
1309                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1310                 return NVME_NIDT_EUI64_LEN;
1311         case NVME_NIDT_NGUID:
1312                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1313                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1314                                  warn_str, cur->nidl);
1315                         return -1;
1316                 }
1317                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1318                         return NVME_NIDT_NGUID_LEN;
1319                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1320                 return NVME_NIDT_NGUID_LEN;
1321         case NVME_NIDT_UUID:
1322                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1323                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1324                                  warn_str, cur->nidl);
1325                         return -1;
1326                 }
1327                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1328                         return NVME_NIDT_UUID_LEN;
1329                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1330                 return NVME_NIDT_UUID_LEN;
1331         case NVME_NIDT_CSI:
1332                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1333                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1334                                  warn_str, cur->nidl);
1335                         return -1;
1336                 }
1337                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1338                 *csi_seen = true;
1339                 return NVME_NIDT_CSI_LEN;
1340         default:
1341                 /* Skip unknown types */
1342                 return cur->nidl;
1343         }
1344 }
1345
1346 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1347                 struct nvme_ns_ids *ids)
1348 {
1349         struct nvme_command c = { };
1350         bool csi_seen = false;
1351         int status, pos, len;
1352         void *data;
1353
1354         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1355                 return 0;
1356         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1357                 return 0;
1358
1359         c.identify.opcode = nvme_admin_identify;
1360         c.identify.nsid = cpu_to_le32(nsid);
1361         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1362
1363         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1364         if (!data)
1365                 return -ENOMEM;
1366
1367         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1368                                       NVME_IDENTIFY_DATA_SIZE);
1369         if (status) {
1370                 dev_warn(ctrl->device,
1371                         "Identify Descriptors failed (%d)\n", status);
1372                 goto free_data;
1373         }
1374
1375         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1376                 struct nvme_ns_id_desc *cur = data + pos;
1377
1378                 if (cur->nidl == 0)
1379                         break;
1380
1381                 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1382                 if (len < 0)
1383                         break;
1384
1385                 len += sizeof(*cur);
1386         }
1387
1388         if (nvme_multi_css(ctrl) && !csi_seen) {
1389                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1390                          nsid);
1391                 status = -EINVAL;
1392         }
1393
1394 free_data:
1395         kfree(data);
1396         return status;
1397 }
1398
1399 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1400                         struct nvme_ns_ids *ids, struct nvme_id_ns **id)
1401 {
1402         struct nvme_command c = { };
1403         int error;
1404
1405         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1406         c.identify.opcode = nvme_admin_identify;
1407         c.identify.nsid = cpu_to_le32(nsid);
1408         c.identify.cns = NVME_ID_CNS_NS;
1409
1410         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1411         if (!*id)
1412                 return -ENOMEM;
1413
1414         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1415         if (error) {
1416                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1417                 goto out_free_id;
1418         }
1419
1420         error = NVME_SC_INVALID_NS | NVME_SC_DNR;
1421         if ((*id)->ncap == 0) /* namespace not allocated or attached */
1422                 goto out_free_id;
1423
1424
1425         if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1426                 dev_info(ctrl->device,
1427                          "Ignoring bogus Namespace Identifiers\n");
1428         } else {
1429                 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1430                     !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1431                         memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
1432                 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1433                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1434                         memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
1435         }
1436
1437         return 0;
1438
1439 out_free_id:
1440         kfree(*id);
1441         return error;
1442 }
1443
1444 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1445                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1446 {
1447         union nvme_result res = { 0 };
1448         struct nvme_command c;
1449         int ret;
1450
1451         memset(&c, 0, sizeof(c));
1452         c.features.opcode = op;
1453         c.features.fid = cpu_to_le32(fid);
1454         c.features.dword11 = cpu_to_le32(dword11);
1455
1456         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1457                         buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1458         if (ret >= 0 && result)
1459                 *result = le32_to_cpu(res.u32);
1460         return ret;
1461 }
1462
1463 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1464                       unsigned int dword11, void *buffer, size_t buflen,
1465                       u32 *result)
1466 {
1467         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1468                              buflen, result);
1469 }
1470 EXPORT_SYMBOL_GPL(nvme_set_features);
1471
1472 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1473                       unsigned int dword11, void *buffer, size_t buflen,
1474                       u32 *result)
1475 {
1476         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1477                              buflen, result);
1478 }
1479 EXPORT_SYMBOL_GPL(nvme_get_features);
1480
1481 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1482 {
1483         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1484         u32 result;
1485         int status, nr_io_queues;
1486
1487         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1488                         &result);
1489         if (status < 0)
1490                 return status;
1491
1492         /*
1493          * Degraded controllers might return an error when setting the queue
1494          * count.  We still want to be able to bring them online and offer
1495          * access to the admin queue, as that might be only way to fix them up.
1496          */
1497         if (status > 0) {
1498                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1499                 *count = 0;
1500         } else {
1501                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1502                 *count = min(*count, nr_io_queues);
1503         }
1504
1505         return 0;
1506 }
1507 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1508
1509 #define NVME_AEN_SUPPORTED \
1510         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1511          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1512
1513 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1514 {
1515         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1516         int status;
1517
1518         if (!supported_aens)
1519                 return;
1520
1521         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1522                         NULL, 0, &result);
1523         if (status)
1524                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1525                          supported_aens);
1526
1527         queue_work(nvme_wq, &ctrl->async_event_work);
1528 }
1529
1530 /*
1531  * Convert integer values from ioctl structures to user pointers, silently
1532  * ignoring the upper bits in the compat case to match behaviour of 32-bit
1533  * kernels.
1534  */
1535 static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1536 {
1537         if (in_compat_syscall())
1538                 ptrval = (compat_uptr_t)ptrval;
1539         return (void __user *)ptrval;
1540 }
1541
1542 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1543 {
1544         struct nvme_user_io io;
1545         struct nvme_command c;
1546         unsigned length, meta_len;
1547         void __user *metadata;
1548
1549         if (copy_from_user(&io, uio, sizeof(io)))
1550                 return -EFAULT;
1551         if (io.flags)
1552                 return -EINVAL;
1553
1554         switch (io.opcode) {
1555         case nvme_cmd_write:
1556         case nvme_cmd_read:
1557         case nvme_cmd_compare:
1558                 break;
1559         default:
1560                 return -EINVAL;
1561         }
1562
1563         length = (io.nblocks + 1) << ns->lba_shift;
1564
1565         if ((io.control & NVME_RW_PRINFO_PRACT) &&
1566             ns->ms == sizeof(struct t10_pi_tuple)) {
1567                 /*
1568                  * Protection information is stripped/inserted by the
1569                  * controller.
1570                  */
1571                 if (nvme_to_user_ptr(io.metadata))
1572                         return -EINVAL;
1573                 meta_len = 0;
1574                 metadata = NULL;
1575         } else {
1576                 meta_len = (io.nblocks + 1) * ns->ms;
1577                 metadata = nvme_to_user_ptr(io.metadata);
1578         }
1579
1580         if (ns->features & NVME_NS_EXT_LBAS) {
1581                 length += meta_len;
1582                 meta_len = 0;
1583         } else if (meta_len) {
1584                 if ((io.metadata & 3) || !io.metadata)
1585                         return -EINVAL;
1586         }
1587
1588         memset(&c, 0, sizeof(c));
1589         c.rw.opcode = io.opcode;
1590         c.rw.flags = io.flags;
1591         c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1592         c.rw.slba = cpu_to_le64(io.slba);
1593         c.rw.length = cpu_to_le16(io.nblocks);
1594         c.rw.control = cpu_to_le16(io.control);
1595         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1596         c.rw.reftag = cpu_to_le32(io.reftag);
1597         c.rw.apptag = cpu_to_le16(io.apptag);
1598         c.rw.appmask = cpu_to_le16(io.appmask);
1599
1600         return nvme_submit_user_cmd(ns->queue, &c,
1601                         nvme_to_user_ptr(io.addr), length,
1602                         metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1603 }
1604
1605 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1606                         struct nvme_passthru_cmd __user *ucmd)
1607 {
1608         struct nvme_passthru_cmd cmd;
1609         struct nvme_command c;
1610         unsigned timeout = 0;
1611         u64 result;
1612         int status;
1613
1614         if (!capable(CAP_SYS_ADMIN))
1615                 return -EACCES;
1616         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1617                 return -EFAULT;
1618         if (cmd.flags)
1619                 return -EINVAL;
1620
1621         memset(&c, 0, sizeof(c));
1622         c.common.opcode = cmd.opcode;
1623         c.common.flags = cmd.flags;
1624         c.common.nsid = cpu_to_le32(cmd.nsid);
1625         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1626         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1627         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1628         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1629         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1630         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1631         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1632         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1633
1634         if (cmd.timeout_ms)
1635                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1636
1637         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1638                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1639                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1640                         0, &result, timeout);
1641
1642         if (status >= 0) {
1643                 if (put_user(result, &ucmd->result))
1644                         return -EFAULT;
1645         }
1646
1647         return status;
1648 }
1649
1650 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1651                         struct nvme_passthru_cmd64 __user *ucmd)
1652 {
1653         struct nvme_passthru_cmd64 cmd;
1654         struct nvme_command c;
1655         unsigned timeout = 0;
1656         int status;
1657
1658         if (!capable(CAP_SYS_ADMIN))
1659                 return -EACCES;
1660         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1661                 return -EFAULT;
1662         if (cmd.flags)
1663                 return -EINVAL;
1664
1665         memset(&c, 0, sizeof(c));
1666         c.common.opcode = cmd.opcode;
1667         c.common.flags = cmd.flags;
1668         c.common.nsid = cpu_to_le32(cmd.nsid);
1669         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1670         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1671         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1672         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1673         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1674         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1675         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1676         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1677
1678         if (cmd.timeout_ms)
1679                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1680
1681         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1682                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1683                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1684                         0, &cmd.result, timeout);
1685
1686         if (status >= 0) {
1687                 if (put_user(cmd.result, &ucmd->result))
1688                         return -EFAULT;
1689         }
1690
1691         return status;
1692 }
1693
1694 /*
1695  * Issue ioctl requests on the first available path.  Note that unlike normal
1696  * block layer requests we will not retry failed request on another controller.
1697  */
1698 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1699                 struct nvme_ns_head **head, int *srcu_idx)
1700 {
1701 #ifdef CONFIG_NVME_MULTIPATH
1702         if (disk->fops == &nvme_ns_head_ops) {
1703                 struct nvme_ns *ns;
1704
1705                 *head = disk->private_data;
1706                 *srcu_idx = srcu_read_lock(&(*head)->srcu);
1707                 ns = nvme_find_path(*head);
1708                 if (!ns)
1709                         srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1710                 return ns;
1711         }
1712 #endif
1713         *head = NULL;
1714         *srcu_idx = -1;
1715         return disk->private_data;
1716 }
1717
1718 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1719 {
1720         if (head)
1721                 srcu_read_unlock(&head->srcu, idx);
1722 }
1723
1724 static bool is_ctrl_ioctl(unsigned int cmd)
1725 {
1726         if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1727                 return true;
1728         if (is_sed_ioctl(cmd))
1729                 return true;
1730         return false;
1731 }
1732
1733 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1734                                   void __user *argp,
1735                                   struct nvme_ns_head *head,
1736                                   int srcu_idx)
1737 {
1738         struct nvme_ctrl *ctrl = ns->ctrl;
1739         int ret;
1740
1741         nvme_get_ctrl(ns->ctrl);
1742         nvme_put_ns_from_disk(head, srcu_idx);
1743
1744         switch (cmd) {
1745         case NVME_IOCTL_ADMIN_CMD:
1746                 ret = nvme_user_cmd(ctrl, NULL, argp);
1747                 break;
1748         case NVME_IOCTL_ADMIN64_CMD:
1749                 ret = nvme_user_cmd64(ctrl, NULL, argp);
1750                 break;
1751         default:
1752                 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1753                 break;
1754         }
1755         nvme_put_ctrl(ctrl);
1756         return ret;
1757 }
1758
1759 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1760                 unsigned int cmd, unsigned long arg)
1761 {
1762         struct nvme_ns_head *head = NULL;
1763         void __user *argp = (void __user *)arg;
1764         struct nvme_ns *ns;
1765         int srcu_idx, ret;
1766
1767         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1768         if (unlikely(!ns))
1769                 return -EWOULDBLOCK;
1770
1771         /*
1772          * Handle ioctls that apply to the controller instead of the namespace
1773          * seperately and drop the ns SRCU reference early.  This avoids a
1774          * deadlock when deleting namespaces using the passthrough interface.
1775          */
1776         if (is_ctrl_ioctl(cmd))
1777                 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1778
1779         switch (cmd) {
1780         case NVME_IOCTL_ID:
1781                 force_successful_syscall_return();
1782                 ret = ns->head->ns_id;
1783                 break;
1784         case NVME_IOCTL_IO_CMD:
1785                 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1786                 break;
1787         case NVME_IOCTL_SUBMIT_IO:
1788                 ret = nvme_submit_io(ns, argp);
1789                 break;
1790         case NVME_IOCTL_IO64_CMD:
1791                 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1792                 break;
1793         default:
1794                 if (ns->ndev)
1795                         ret = nvme_nvm_ioctl(ns, cmd, arg);
1796                 else
1797                         ret = -ENOTTY;
1798         }
1799
1800         nvme_put_ns_from_disk(head, srcu_idx);
1801         return ret;
1802 }
1803
1804 #ifdef CONFIG_COMPAT
1805 struct nvme_user_io32 {
1806         __u8    opcode;
1807         __u8    flags;
1808         __u16   control;
1809         __u16   nblocks;
1810         __u16   rsvd;
1811         __u64   metadata;
1812         __u64   addr;
1813         __u64   slba;
1814         __u32   dsmgmt;
1815         __u32   reftag;
1816         __u16   apptag;
1817         __u16   appmask;
1818 } __attribute__((__packed__));
1819
1820 #define NVME_IOCTL_SUBMIT_IO32  _IOW('N', 0x42, struct nvme_user_io32)
1821
1822 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1823                 unsigned int cmd, unsigned long arg)
1824 {
1825         /*
1826          * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1827          * between 32 bit programs and 64 bit kernel.
1828          * The cause is that the results of sizeof(struct nvme_user_io),
1829          * which is used to define NVME_IOCTL_SUBMIT_IO,
1830          * are not same between 32 bit compiler and 64 bit compiler.
1831          * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1832          * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1833          * Other IOCTL numbers are same between 32 bit and 64 bit.
1834          * So there is nothing to do regarding to other IOCTL numbers.
1835          */
1836         if (cmd == NVME_IOCTL_SUBMIT_IO32)
1837                 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1838
1839         return nvme_ioctl(bdev, mode, cmd, arg);
1840 }
1841 #else
1842 #define nvme_compat_ioctl       NULL
1843 #endif /* CONFIG_COMPAT */
1844
1845 static int nvme_open(struct block_device *bdev, fmode_t mode)
1846 {
1847         struct nvme_ns *ns = bdev->bd_disk->private_data;
1848
1849 #ifdef CONFIG_NVME_MULTIPATH
1850         /* should never be called due to GENHD_FL_HIDDEN */
1851         if (WARN_ON_ONCE(ns->head->disk))
1852                 goto fail;
1853 #endif
1854         if (!kref_get_unless_zero(&ns->kref))
1855                 goto fail;
1856         if (!try_module_get(ns->ctrl->ops->module))
1857                 goto fail_put_ns;
1858
1859         return 0;
1860
1861 fail_put_ns:
1862         nvme_put_ns(ns);
1863 fail:
1864         return -ENXIO;
1865 }
1866
1867 static void nvme_release(struct gendisk *disk, fmode_t mode)
1868 {
1869         struct nvme_ns *ns = disk->private_data;
1870
1871         module_put(ns->ctrl->ops->module);
1872         nvme_put_ns(ns);
1873 }
1874
1875 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1876 {
1877         /* some standard values */
1878         geo->heads = 1 << 6;
1879         geo->sectors = 1 << 5;
1880         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1881         return 0;
1882 }
1883
1884 #ifdef CONFIG_BLK_DEV_INTEGRITY
1885 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1886                                 u32 max_integrity_segments)
1887 {
1888         struct blk_integrity integrity;
1889
1890         memset(&integrity, 0, sizeof(integrity));
1891         switch (pi_type) {
1892         case NVME_NS_DPS_PI_TYPE3:
1893                 integrity.profile = &t10_pi_type3_crc;
1894                 integrity.tag_size = sizeof(u16) + sizeof(u32);
1895                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1896                 break;
1897         case NVME_NS_DPS_PI_TYPE1:
1898         case NVME_NS_DPS_PI_TYPE2:
1899                 integrity.profile = &t10_pi_type1_crc;
1900                 integrity.tag_size = sizeof(u16);
1901                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1902                 break;
1903         default:
1904                 integrity.profile = NULL;
1905                 break;
1906         }
1907         integrity.tuple_size = ms;
1908         blk_integrity_register(disk, &integrity);
1909         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1910 }
1911 #else
1912 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1913                                 u32 max_integrity_segments)
1914 {
1915 }
1916 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1917
1918 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1919 {
1920         struct nvme_ctrl *ctrl = ns->ctrl;
1921         struct request_queue *queue = disk->queue;
1922         u32 size = queue_logical_block_size(queue);
1923
1924         if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1925                 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1926                 return;
1927         }
1928
1929         if (ctrl->nr_streams && ns->sws && ns->sgs)
1930                 size *= ns->sws * ns->sgs;
1931
1932         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1933                         NVME_DSM_MAX_RANGES);
1934
1935         queue->limits.discard_alignment = 0;
1936         queue->limits.discard_granularity = size;
1937
1938         /* If discard is already enabled, don't reset queue limits */
1939         if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1940                 return;
1941
1942         blk_queue_max_discard_sectors(queue, UINT_MAX);
1943         blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1944
1945         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1946                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1947 }
1948
1949 /*
1950  * Even though NVMe spec explicitly states that MDTS is not applicable to the
1951  * write-zeroes, we are cautious and limit the size to the controllers
1952  * max_hw_sectors value, which is based on the MDTS field and possibly other
1953  * limiting factors.
1954  */
1955 static void nvme_config_write_zeroes(struct request_queue *q,
1956                 struct nvme_ctrl *ctrl)
1957 {
1958         if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
1959             !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1960                 blk_queue_max_write_zeroes_sectors(q, ctrl->max_hw_sectors);
1961 }
1962
1963 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1964 {
1965         return !uuid_is_null(&ids->uuid) ||
1966                 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1967                 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1968 }
1969
1970 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1971 {
1972         return uuid_equal(&a->uuid, &b->uuid) &&
1973                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1974                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1975                 a->csi == b->csi;
1976 }
1977
1978 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1979                                  u32 *phys_bs, u32 *io_opt)
1980 {
1981         struct streams_directive_params s;
1982         int ret;
1983
1984         if (!ctrl->nr_streams)
1985                 return 0;
1986
1987         ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1988         if (ret)
1989                 return ret;
1990
1991         ns->sws = le32_to_cpu(s.sws);
1992         ns->sgs = le16_to_cpu(s.sgs);
1993
1994         if (ns->sws) {
1995                 *phys_bs = ns->sws * (1 << ns->lba_shift);
1996                 if (ns->sgs)
1997                         *io_opt = *phys_bs * ns->sgs;
1998         }
1999
2000         return 0;
2001 }
2002
2003 static int nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
2004 {
2005         struct nvme_ctrl *ctrl = ns->ctrl;
2006
2007         /*
2008          * The PI implementation requires the metadata size to be equal to the
2009          * t10 pi tuple size.
2010          */
2011         ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
2012         if (ns->ms == sizeof(struct t10_pi_tuple))
2013                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
2014         else
2015                 ns->pi_type = 0;
2016
2017         ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
2018         if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
2019                 return 0;
2020         if (ctrl->ops->flags & NVME_F_FABRICS) {
2021                 /*
2022                  * The NVMe over Fabrics specification only supports metadata as
2023                  * part of the extended data LBA.  We rely on HCA/HBA support to
2024                  * remap the separate metadata buffer from the block layer.
2025                  */
2026                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
2027                         return -EINVAL;
2028                 if (ctrl->max_integrity_segments)
2029                         ns->features |=
2030                                 (NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
2031         } else {
2032                 /*
2033                  * For PCIe controllers, we can't easily remap the separate
2034                  * metadata buffer from the block layer and thus require a
2035                  * separate metadata buffer for block layer metadata/PI support.
2036                  * We allow extended LBAs for the passthrough interface, though.
2037                  */
2038                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
2039                         ns->features |= NVME_NS_EXT_LBAS;
2040                 else
2041                         ns->features |= NVME_NS_METADATA_SUPPORTED;
2042         }
2043
2044         return 0;
2045 }
2046
2047 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2048                 struct request_queue *q)
2049 {
2050         bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
2051
2052         if (ctrl->max_hw_sectors) {
2053                 u32 max_segments =
2054                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
2055
2056                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
2057                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
2058                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
2059         }
2060         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
2061         blk_queue_dma_alignment(q, 3);
2062         blk_queue_write_cache(q, vwc, vwc);
2063 }
2064
2065 static void nvme_update_disk_info(struct gendisk *disk,
2066                 struct nvme_ns *ns, struct nvme_id_ns *id)
2067 {
2068         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
2069         unsigned short bs = 1 << ns->lba_shift;
2070         u32 atomic_bs, phys_bs, io_opt = 0;
2071
2072         /*
2073          * The block layer can't support LBA sizes larger than the page size
2074          * or smaller than a sector size yet, so catch this early and don't
2075          * allow block I/O.
2076          */
2077         if (ns->lba_shift > PAGE_SHIFT || ns->lba_shift < SECTOR_SHIFT) {
2078                 capacity = 0;
2079                 bs = (1 << 9);
2080         }
2081
2082         blk_integrity_unregister(disk);
2083
2084         atomic_bs = phys_bs = bs;
2085         nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
2086         if (id->nabo == 0) {
2087                 /*
2088                  * Bit 1 indicates whether NAWUPF is defined for this namespace
2089                  * and whether it should be used instead of AWUPF. If NAWUPF ==
2090                  * 0 then AWUPF must be used instead.
2091                  */
2092                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
2093                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
2094                 else
2095                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
2096         }
2097
2098         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
2099                 /* NPWG = Namespace Preferred Write Granularity */
2100                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2101                 /* NOWS = Namespace Optimal Write Size */
2102                 io_opt = bs * (1 + le16_to_cpu(id->nows));
2103         }
2104
2105         blk_queue_logical_block_size(disk->queue, bs);
2106         /*
2107          * Linux filesystems assume writing a single physical block is
2108          * an atomic operation. Hence limit the physical block size to the
2109          * value of the Atomic Write Unit Power Fail parameter.
2110          */
2111         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
2112         blk_queue_io_min(disk->queue, phys_bs);
2113         blk_queue_io_opt(disk->queue, io_opt);
2114
2115         /*
2116          * Register a metadata profile for PI, or the plain non-integrity NVMe
2117          * metadata masquerading as Type 0 if supported, otherwise reject block
2118          * I/O to namespaces with metadata except when the namespace supports
2119          * PI, as it can strip/insert in that case.
2120          */
2121         if (ns->ms) {
2122                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2123                     (ns->features & NVME_NS_METADATA_SUPPORTED))
2124                         nvme_init_integrity(disk, ns->ms, ns->pi_type,
2125                                             ns->ctrl->max_integrity_segments);
2126                 else if (!nvme_ns_has_pi(ns))
2127                         capacity = 0;
2128         }
2129
2130         set_capacity_revalidate_and_notify(disk, capacity, false);
2131
2132         nvme_config_discard(disk, ns);
2133         nvme_config_write_zeroes(disk->queue, ns->ctrl);
2134
2135         if (id->nsattr & NVME_NS_ATTR_RO)
2136                 set_disk_ro(disk, true);
2137 }
2138
2139 static inline bool nvme_first_scan(struct gendisk *disk)
2140 {
2141         /* nvme_alloc_ns() scans the disk prior to adding it */
2142         return !(disk->flags & GENHD_FL_UP);
2143 }
2144
2145 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
2146 {
2147         struct nvme_ctrl *ctrl = ns->ctrl;
2148         u32 iob;
2149
2150         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2151             is_power_of_2(ctrl->max_hw_sectors))
2152                 iob = ctrl->max_hw_sectors;
2153         else
2154                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2155
2156         if (!iob)
2157                 return;
2158
2159         if (!is_power_of_2(iob)) {
2160                 if (nvme_first_scan(ns->disk))
2161                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2162                                 ns->disk->disk_name, iob);
2163                 return;
2164         }
2165
2166         if (blk_queue_is_zoned(ns->disk->queue)) {
2167                 if (nvme_first_scan(ns->disk))
2168                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
2169                                 ns->disk->disk_name);
2170                 return;
2171         }
2172
2173         blk_queue_chunk_sectors(ns->queue, iob);
2174 }
2175
2176 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
2177 {
2178         unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2179         int ret;
2180
2181         blk_mq_freeze_queue(ns->disk->queue);
2182         ns->lba_shift = id->lbaf[lbaf].ds;
2183         nvme_set_queue_limits(ns->ctrl, ns->queue);
2184
2185         if (ns->head->ids.csi == NVME_CSI_ZNS) {
2186                 ret = nvme_update_zone_info(ns, lbaf);
2187                 if (ret)
2188                         goto out_unfreeze;
2189         }
2190
2191         ret = nvme_configure_metadata(ns, id);
2192         if (ret)
2193                 goto out_unfreeze;
2194         nvme_set_chunk_sectors(ns, id);
2195         nvme_update_disk_info(ns->disk, ns, id);
2196         blk_mq_unfreeze_queue(ns->disk->queue);
2197
2198         if (blk_queue_is_zoned(ns->queue)) {
2199                 ret = nvme_revalidate_zones(ns);
2200                 if (ret && !nvme_first_scan(ns->disk))
2201                         return ret;
2202         }
2203
2204 #ifdef CONFIG_NVME_MULTIPATH
2205         if (ns->head->disk) {
2206                 blk_mq_freeze_queue(ns->head->disk->queue);
2207                 nvme_update_disk_info(ns->head->disk, ns, id);
2208                 blk_stack_limits(&ns->head->disk->queue->limits,
2209                                  &ns->queue->limits, 0);
2210                 blk_queue_update_readahead(ns->head->disk->queue);
2211                 nvme_update_bdev_size(ns->head->disk);
2212                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2213         }
2214 #endif
2215         return 0;
2216
2217 out_unfreeze:
2218         blk_mq_unfreeze_queue(ns->disk->queue);
2219         return ret;
2220 }
2221
2222 static char nvme_pr_type(enum pr_type type)
2223 {
2224         switch (type) {
2225         case PR_WRITE_EXCLUSIVE:
2226                 return 1;
2227         case PR_EXCLUSIVE_ACCESS:
2228                 return 2;
2229         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2230                 return 3;
2231         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2232                 return 4;
2233         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2234                 return 5;
2235         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2236                 return 6;
2237         default:
2238                 return 0;
2239         }
2240 };
2241
2242 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2243                                 u64 key, u64 sa_key, u8 op)
2244 {
2245         struct nvme_ns_head *head = NULL;
2246         struct nvme_ns *ns;
2247         struct nvme_command c;
2248         int srcu_idx, ret;
2249         u8 data[16] = { 0, };
2250
2251         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2252         if (unlikely(!ns))
2253                 return -EWOULDBLOCK;
2254
2255         put_unaligned_le64(key, &data[0]);
2256         put_unaligned_le64(sa_key, &data[8]);
2257
2258         memset(&c, 0, sizeof(c));
2259         c.common.opcode = op;
2260         c.common.nsid = cpu_to_le32(ns->head->ns_id);
2261         c.common.cdw10 = cpu_to_le32(cdw10);
2262
2263         ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
2264         nvme_put_ns_from_disk(head, srcu_idx);
2265         return ret;
2266 }
2267
2268 static int nvme_pr_register(struct block_device *bdev, u64 old,
2269                 u64 new, unsigned flags)
2270 {
2271         u32 cdw10;
2272
2273         if (flags & ~PR_FL_IGNORE_KEY)
2274                 return -EOPNOTSUPP;
2275
2276         cdw10 = old ? 2 : 0;
2277         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2278         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2279         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2280 }
2281
2282 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2283                 enum pr_type type, unsigned flags)
2284 {
2285         u32 cdw10;
2286
2287         if (flags & ~PR_FL_IGNORE_KEY)
2288                 return -EOPNOTSUPP;
2289
2290         cdw10 = nvme_pr_type(type) << 8;
2291         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2292         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2293 }
2294
2295 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2296                 enum pr_type type, bool abort)
2297 {
2298         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2299
2300         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2301 }
2302
2303 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2304 {
2305         u32 cdw10 = 1 | (key ? 0 : 1 << 3);
2306
2307         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2308 }
2309
2310 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2311 {
2312         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3);
2313
2314         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2315 }
2316
2317 static const struct pr_ops nvme_pr_ops = {
2318         .pr_register    = nvme_pr_register,
2319         .pr_reserve     = nvme_pr_reserve,
2320         .pr_release     = nvme_pr_release,
2321         .pr_preempt     = nvme_pr_preempt,
2322         .pr_clear       = nvme_pr_clear,
2323 };
2324
2325 #ifdef CONFIG_BLK_SED_OPAL
2326 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2327                 bool send)
2328 {
2329         struct nvme_ctrl *ctrl = data;
2330         struct nvme_command cmd;
2331
2332         memset(&cmd, 0, sizeof(cmd));
2333         if (send)
2334                 cmd.common.opcode = nvme_admin_security_send;
2335         else
2336                 cmd.common.opcode = nvme_admin_security_recv;
2337         cmd.common.nsid = 0;
2338         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2339         cmd.common.cdw11 = cpu_to_le32(len);
2340
2341         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2342                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2343 }
2344 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2345 #endif /* CONFIG_BLK_SED_OPAL */
2346
2347 static const struct block_device_operations nvme_fops = {
2348         .owner          = THIS_MODULE,
2349         .ioctl          = nvme_ioctl,
2350         .compat_ioctl   = nvme_compat_ioctl,
2351         .open           = nvme_open,
2352         .release        = nvme_release,
2353         .getgeo         = nvme_getgeo,
2354         .report_zones   = nvme_report_zones,
2355         .pr_ops         = &nvme_pr_ops,
2356 };
2357
2358 #ifdef CONFIG_NVME_MULTIPATH
2359 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2360 {
2361         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2362
2363         if (!kref_get_unless_zero(&head->ref))
2364                 return -ENXIO;
2365         return 0;
2366 }
2367
2368 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2369 {
2370         nvme_put_ns_head(disk->private_data);
2371 }
2372
2373 const struct block_device_operations nvme_ns_head_ops = {
2374         .owner          = THIS_MODULE,
2375         .submit_bio     = nvme_ns_head_submit_bio,
2376         .open           = nvme_ns_head_open,
2377         .release        = nvme_ns_head_release,
2378         .ioctl          = nvme_ioctl,
2379         .compat_ioctl   = nvme_compat_ioctl,
2380         .getgeo         = nvme_getgeo,
2381         .report_zones   = nvme_report_zones,
2382         .pr_ops         = &nvme_pr_ops,
2383 };
2384 #endif /* CONFIG_NVME_MULTIPATH */
2385
2386 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2387 {
2388         unsigned long timeout =
2389                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2390         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2391         int ret;
2392
2393         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2394                 if (csts == ~0)
2395                         return -ENODEV;
2396                 if ((csts & NVME_CSTS_RDY) == bit)
2397                         break;
2398
2399                 usleep_range(1000, 2000);
2400                 if (fatal_signal_pending(current))
2401                         return -EINTR;
2402                 if (time_after(jiffies, timeout)) {
2403                         dev_err(ctrl->device,
2404                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2405                                 enabled ? "initialisation" : "reset", csts);
2406                         return -ENODEV;
2407                 }
2408         }
2409
2410         return ret;
2411 }
2412
2413 /*
2414  * If the device has been passed off to us in an enabled state, just clear
2415  * the enabled bit.  The spec says we should set the 'shutdown notification
2416  * bits', but doing so may cause the device to complete commands to the
2417  * admin queue ... and we don't know what memory that might be pointing at!
2418  */
2419 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2420 {
2421         int ret;
2422
2423         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2424         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2425
2426         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2427         if (ret)
2428                 return ret;
2429
2430         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2431                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2432
2433         return nvme_wait_ready(ctrl, ctrl->cap, false);
2434 }
2435 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2436
2437 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2438 {
2439         unsigned dev_page_min;
2440         int ret;
2441
2442         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2443         if (ret) {
2444                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2445                 return ret;
2446         }
2447         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2448
2449         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2450                 dev_err(ctrl->device,
2451                         "Minimum device page size %u too large for host (%u)\n",
2452                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2453                 return -ENODEV;
2454         }
2455
2456         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2457                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2458         else
2459                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2460         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2461         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2462         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2463         ctrl->ctrl_config |= NVME_CC_ENABLE;
2464
2465         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2466         if (ret)
2467                 return ret;
2468         return nvme_wait_ready(ctrl, ctrl->cap, true);
2469 }
2470 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2471
2472 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2473 {
2474         unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2475         u32 csts;
2476         int ret;
2477
2478         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2479         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2480
2481         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2482         if (ret)
2483                 return ret;
2484
2485         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2486                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2487                         break;
2488
2489                 msleep(100);
2490                 if (fatal_signal_pending(current))
2491                         return -EINTR;
2492                 if (time_after(jiffies, timeout)) {
2493                         dev_err(ctrl->device,
2494                                 "Device shutdown incomplete; abort shutdown\n");
2495                         return -ENODEV;
2496                 }
2497         }
2498
2499         return ret;
2500 }
2501 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2502
2503 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2504 {
2505         __le64 ts;
2506         int ret;
2507
2508         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2509                 return 0;
2510
2511         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2512         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2513                         NULL);
2514         if (ret)
2515                 dev_warn_once(ctrl->device,
2516                         "could not set timestamp (%d)\n", ret);
2517         return ret;
2518 }
2519
2520 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2521 {
2522         struct nvme_feat_host_behavior *host;
2523         int ret;
2524
2525         /* Don't bother enabling the feature if retry delay is not reported */
2526         if (!ctrl->crdt[0])
2527                 return 0;
2528
2529         host = kzalloc(sizeof(*host), GFP_KERNEL);
2530         if (!host)
2531                 return 0;
2532
2533         host->acre = NVME_ENABLE_ACRE;
2534         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2535                                 host, sizeof(*host), NULL);
2536         kfree(host);
2537         return ret;
2538 }
2539
2540 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2541 {
2542         /*
2543          * APST (Autonomous Power State Transition) lets us program a
2544          * table of power state transitions that the controller will
2545          * perform automatically.  We configure it with a simple
2546          * heuristic: we are willing to spend at most 2% of the time
2547          * transitioning between power states.  Therefore, when running
2548          * in any given state, we will enter the next lower-power
2549          * non-operational state after waiting 50 * (enlat + exlat)
2550          * microseconds, as long as that state's exit latency is under
2551          * the requested maximum latency.
2552          *
2553          * We will not autonomously enter any non-operational state for
2554          * which the total latency exceeds ps_max_latency_us.  Users
2555          * can set ps_max_latency_us to zero to turn off APST.
2556          */
2557
2558         unsigned apste;
2559         struct nvme_feat_auto_pst *table;
2560         u64 max_lat_us = 0;
2561         int max_ps = -1;
2562         int ret;
2563
2564         /*
2565          * If APST isn't supported or if we haven't been initialized yet,
2566          * then don't do anything.
2567          */
2568         if (!ctrl->apsta)
2569                 return 0;
2570
2571         if (ctrl->npss > 31) {
2572                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2573                 return 0;
2574         }
2575
2576         table = kzalloc(sizeof(*table), GFP_KERNEL);
2577         if (!table)
2578                 return 0;
2579
2580         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2581                 /* Turn off APST. */
2582                 apste = 0;
2583                 dev_dbg(ctrl->device, "APST disabled\n");
2584         } else {
2585                 __le64 target = cpu_to_le64(0);
2586                 int state;
2587
2588                 /*
2589                  * Walk through all states from lowest- to highest-power.
2590                  * According to the spec, lower-numbered states use more
2591                  * power.  NPSS, despite the name, is the index of the
2592                  * lowest-power state, not the number of states.
2593                  */
2594                 for (state = (int)ctrl->npss; state >= 0; state--) {
2595                         u64 total_latency_us, exit_latency_us, transition_ms;
2596
2597                         if (target)
2598                                 table->entries[state] = target;
2599
2600                         /*
2601                          * Don't allow transitions to the deepest state
2602                          * if it's quirked off.
2603                          */
2604                         if (state == ctrl->npss &&
2605                             (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2606                                 continue;
2607
2608                         /*
2609                          * Is this state a useful non-operational state for
2610                          * higher-power states to autonomously transition to?
2611                          */
2612                         if (!(ctrl->psd[state].flags &
2613                               NVME_PS_FLAGS_NON_OP_STATE))
2614                                 continue;
2615
2616                         exit_latency_us =
2617                                 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2618                         if (exit_latency_us > ctrl->ps_max_latency_us)
2619                                 continue;
2620
2621                         total_latency_us =
2622                                 exit_latency_us +
2623                                 le32_to_cpu(ctrl->psd[state].entry_lat);
2624
2625                         /*
2626                          * This state is good.  Use it as the APST idle
2627                          * target for higher power states.
2628                          */
2629                         transition_ms = total_latency_us + 19;
2630                         do_div(transition_ms, 20);
2631                         if (transition_ms > (1 << 24) - 1)
2632                                 transition_ms = (1 << 24) - 1;
2633
2634                         target = cpu_to_le64((state << 3) |
2635                                              (transition_ms << 8));
2636
2637                         if (max_ps == -1)
2638                                 max_ps = state;
2639
2640                         if (total_latency_us > max_lat_us)
2641                                 max_lat_us = total_latency_us;
2642                 }
2643
2644                 apste = 1;
2645
2646                 if (max_ps == -1) {
2647                         dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2648                 } else {
2649                         dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2650                                 max_ps, max_lat_us, (int)sizeof(*table), table);
2651                 }
2652         }
2653
2654         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2655                                 table, sizeof(*table), NULL);
2656         if (ret)
2657                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2658
2659         kfree(table);
2660         return ret;
2661 }
2662
2663 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2664 {
2665         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2666         u64 latency;
2667
2668         switch (val) {
2669         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2670         case PM_QOS_LATENCY_ANY:
2671                 latency = U64_MAX;
2672                 break;
2673
2674         default:
2675                 latency = val;
2676         }
2677
2678         if (ctrl->ps_max_latency_us != latency) {
2679                 ctrl->ps_max_latency_us = latency;
2680                 if (ctrl->state == NVME_CTRL_LIVE)
2681                         nvme_configure_apst(ctrl);
2682         }
2683 }
2684
2685 struct nvme_core_quirk_entry {
2686         /*
2687          * NVMe model and firmware strings are padded with spaces.  For
2688          * simplicity, strings in the quirk table are padded with NULLs
2689          * instead.
2690          */
2691         u16 vid;
2692         const char *mn;
2693         const char *fr;
2694         unsigned long quirks;
2695 };
2696
2697 static const struct nvme_core_quirk_entry core_quirks[] = {
2698         {
2699                 /*
2700                  * This Toshiba device seems to die using any APST states.  See:
2701                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2702                  */
2703                 .vid = 0x1179,
2704                 .mn = "THNSF5256GPUK TOSHIBA",
2705                 .quirks = NVME_QUIRK_NO_APST,
2706         },
2707         {
2708                 /*
2709                  * This LiteON CL1-3D*-Q11 firmware version has a race
2710                  * condition associated with actions related to suspend to idle
2711                  * LiteON has resolved the problem in future firmware
2712                  */
2713                 .vid = 0x14a4,
2714                 .fr = "22301111",
2715                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2716         },
2717         {
2718                 /*
2719                  * This Kioxia CD6-V Series / HPE PE8030 device times out and
2720                  * aborts I/O during any load, but more easily reproducible
2721                  * with discards (fstrim).
2722                  *
2723                  * The device is left in a state where it is also not possible
2724                  * to use "nvme set-feature" to disable APST, but booting with
2725                  * nvme_core.default_ps_max_latency=0 works.
2726                  */
2727                 .vid = 0x1e0f,
2728                 .mn = "KCD6XVUL6T40",
2729                 .quirks = NVME_QUIRK_NO_APST,
2730         },
2731         {
2732                 /*
2733                  * The external Samsung X5 SSD fails initialization without a
2734                  * delay before checking if it is ready and has a whole set of
2735                  * other problems.  To make this even more interesting, it
2736                  * shares the PCI ID with internal Samsung 970 Evo Plus that
2737                  * does not need or want these quirks.
2738                  */
2739                 .vid = 0x144d,
2740                 .mn = "Samsung Portable SSD X5",
2741                 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2742                           NVME_QUIRK_NO_DEEPEST_PS |
2743                           NVME_QUIRK_IGNORE_DEV_SUBNQN,
2744         }
2745 };
2746
2747 /* match is null-terminated but idstr is space-padded. */
2748 static bool string_matches(const char *idstr, const char *match, size_t len)
2749 {
2750         size_t matchlen;
2751
2752         if (!match)
2753                 return true;
2754
2755         matchlen = strlen(match);
2756         WARN_ON_ONCE(matchlen > len);
2757
2758         if (memcmp(idstr, match, matchlen))
2759                 return false;
2760
2761         for (; matchlen < len; matchlen++)
2762                 if (idstr[matchlen] != ' ')
2763                         return false;
2764
2765         return true;
2766 }
2767
2768 static bool quirk_matches(const struct nvme_id_ctrl *id,
2769                           const struct nvme_core_quirk_entry *q)
2770 {
2771         return q->vid == le16_to_cpu(id->vid) &&
2772                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2773                 string_matches(id->fr, q->fr, sizeof(id->fr));
2774 }
2775
2776 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2777                 struct nvme_id_ctrl *id)
2778 {
2779         size_t nqnlen;
2780         int off;
2781
2782         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2783                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2784                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2785                         strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2786                         return;
2787                 }
2788
2789                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2790                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2791         }
2792
2793         /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2794         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2795                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2796                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2797         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2798         off += sizeof(id->sn);
2799         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2800         off += sizeof(id->mn);
2801         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2802 }
2803
2804 static void nvme_release_subsystem(struct device *dev)
2805 {
2806         struct nvme_subsystem *subsys =
2807                 container_of(dev, struct nvme_subsystem, dev);
2808
2809         if (subsys->instance >= 0)
2810                 ida_simple_remove(&nvme_instance_ida, subsys->instance);
2811         kfree(subsys);
2812 }
2813
2814 static void nvme_destroy_subsystem(struct kref *ref)
2815 {
2816         struct nvme_subsystem *subsys =
2817                         container_of(ref, struct nvme_subsystem, ref);
2818
2819         mutex_lock(&nvme_subsystems_lock);
2820         list_del(&subsys->entry);
2821         mutex_unlock(&nvme_subsystems_lock);
2822
2823         ida_destroy(&subsys->ns_ida);
2824         device_del(&subsys->dev);
2825         put_device(&subsys->dev);
2826 }
2827
2828 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2829 {
2830         kref_put(&subsys->ref, nvme_destroy_subsystem);
2831 }
2832
2833 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2834 {
2835         struct nvme_subsystem *subsys;
2836
2837         lockdep_assert_held(&nvme_subsystems_lock);
2838
2839         /*
2840          * Fail matches for discovery subsystems. This results
2841          * in each discovery controller bound to a unique subsystem.
2842          * This avoids issues with validating controller values
2843          * that can only be true when there is a single unique subsystem.
2844          * There may be multiple and completely independent entities
2845          * that provide discovery controllers.
2846          */
2847         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2848                 return NULL;
2849
2850         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2851                 if (strcmp(subsys->subnqn, subsysnqn))
2852                         continue;
2853                 if (!kref_get_unless_zero(&subsys->ref))
2854                         continue;
2855                 return subsys;
2856         }
2857
2858         return NULL;
2859 }
2860
2861 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2862         struct device_attribute subsys_attr_##_name = \
2863                 __ATTR(_name, _mode, _show, NULL)
2864
2865 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2866                                     struct device_attribute *attr,
2867                                     char *buf)
2868 {
2869         struct nvme_subsystem *subsys =
2870                 container_of(dev, struct nvme_subsystem, dev);
2871
2872         return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2873 }
2874 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2875
2876 #define nvme_subsys_show_str_function(field)                            \
2877 static ssize_t subsys_##field##_show(struct device *dev,                \
2878                             struct device_attribute *attr, char *buf)   \
2879 {                                                                       \
2880         struct nvme_subsystem *subsys =                                 \
2881                 container_of(dev, struct nvme_subsystem, dev);          \
2882         return sysfs_emit(buf, "%.*s\n",                                \
2883                            (int)sizeof(subsys->field), subsys->field);  \
2884 }                                                                       \
2885 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2886
2887 nvme_subsys_show_str_function(model);
2888 nvme_subsys_show_str_function(serial);
2889 nvme_subsys_show_str_function(firmware_rev);
2890
2891 static struct attribute *nvme_subsys_attrs[] = {
2892         &subsys_attr_model.attr,
2893         &subsys_attr_serial.attr,
2894         &subsys_attr_firmware_rev.attr,
2895         &subsys_attr_subsysnqn.attr,
2896 #ifdef CONFIG_NVME_MULTIPATH
2897         &subsys_attr_iopolicy.attr,
2898 #endif
2899         NULL,
2900 };
2901
2902 static struct attribute_group nvme_subsys_attrs_group = {
2903         .attrs = nvme_subsys_attrs,
2904 };
2905
2906 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2907         &nvme_subsys_attrs_group,
2908         NULL,
2909 };
2910
2911 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2912 {
2913         return ctrl->opts && ctrl->opts->discovery_nqn;
2914 }
2915
2916 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2917                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2918 {
2919         struct nvme_ctrl *tmp;
2920
2921         lockdep_assert_held(&nvme_subsystems_lock);
2922
2923         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2924                 if (nvme_state_terminal(tmp))
2925                         continue;
2926
2927                 if (tmp->cntlid == ctrl->cntlid) {
2928                         dev_err(ctrl->device,
2929                                 "Duplicate cntlid %u with %s, rejecting\n",
2930                                 ctrl->cntlid, dev_name(tmp->device));
2931                         return false;
2932                 }
2933
2934                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2935                     nvme_discovery_ctrl(ctrl))
2936                         continue;
2937
2938                 dev_err(ctrl->device,
2939                         "Subsystem does not support multiple controllers\n");
2940                 return false;
2941         }
2942
2943         return true;
2944 }
2945
2946 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2947 {
2948         struct nvme_subsystem *subsys, *found;
2949         int ret;
2950
2951         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2952         if (!subsys)
2953                 return -ENOMEM;
2954
2955         subsys->instance = -1;
2956         mutex_init(&subsys->lock);
2957         kref_init(&subsys->ref);
2958         INIT_LIST_HEAD(&subsys->ctrls);
2959         INIT_LIST_HEAD(&subsys->nsheads);
2960         nvme_init_subnqn(subsys, ctrl, id);
2961         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2962         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2963         subsys->vendor_id = le16_to_cpu(id->vid);
2964         subsys->cmic = id->cmic;
2965         subsys->awupf = le16_to_cpu(id->awupf);
2966 #ifdef CONFIG_NVME_MULTIPATH
2967         subsys->iopolicy = NVME_IOPOLICY_NUMA;
2968 #endif
2969
2970         subsys->dev.class = nvme_subsys_class;
2971         subsys->dev.release = nvme_release_subsystem;
2972         subsys->dev.groups = nvme_subsys_attrs_groups;
2973         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2974         device_initialize(&subsys->dev);
2975
2976         mutex_lock(&nvme_subsystems_lock);
2977         found = __nvme_find_get_subsystem(subsys->subnqn);
2978         if (found) {
2979                 put_device(&subsys->dev);
2980                 subsys = found;
2981
2982                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2983                         ret = -EINVAL;
2984                         goto out_put_subsystem;
2985                 }
2986         } else {
2987                 ret = device_add(&subsys->dev);
2988                 if (ret) {
2989                         dev_err(ctrl->device,
2990                                 "failed to register subsystem device.\n");
2991                         put_device(&subsys->dev);
2992                         goto out_unlock;
2993                 }
2994                 ida_init(&subsys->ns_ida);
2995                 list_add_tail(&subsys->entry, &nvme_subsystems);
2996         }
2997
2998         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2999                                 dev_name(ctrl->device));
3000         if (ret) {
3001                 dev_err(ctrl->device,
3002                         "failed to create sysfs link from subsystem.\n");
3003                 goto out_put_subsystem;
3004         }
3005
3006         if (!found)
3007                 subsys->instance = ctrl->instance;
3008         ctrl->subsys = subsys;
3009         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
3010         mutex_unlock(&nvme_subsystems_lock);
3011         return 0;
3012
3013 out_put_subsystem:
3014         nvme_put_subsystem(subsys);
3015 out_unlock:
3016         mutex_unlock(&nvme_subsystems_lock);
3017         return ret;
3018 }
3019
3020 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
3021                 void *log, size_t size, u64 offset)
3022 {
3023         struct nvme_command c = { };
3024         u32 dwlen = nvme_bytes_to_numd(size);
3025
3026         c.get_log_page.opcode = nvme_admin_get_log_page;
3027         c.get_log_page.nsid = cpu_to_le32(nsid);
3028         c.get_log_page.lid = log_page;
3029         c.get_log_page.lsp = lsp;
3030         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
3031         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
3032         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3033         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3034         c.get_log_page.csi = csi;
3035
3036         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3037 }
3038
3039 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3040                                 struct nvme_effects_log **log)
3041 {
3042         struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
3043         int ret;
3044
3045         if (cel)
3046                 goto out;
3047
3048         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3049         if (!cel)
3050                 return -ENOMEM;
3051
3052         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3053                         cel, sizeof(*cel), 0);
3054         if (ret) {
3055                 kfree(cel);
3056                 return ret;
3057         }
3058
3059         xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3060 out:
3061         *log = cel;
3062         return 0;
3063 }
3064
3065 /*
3066  * Initialize the cached copies of the Identify data and various controller
3067  * register in our nvme_ctrl structure.  This should be called as soon as
3068  * the admin queue is fully up and running.
3069  */
3070 int nvme_init_identify(struct nvme_ctrl *ctrl)
3071 {
3072         struct nvme_id_ctrl *id;
3073         int ret, page_shift;
3074         u32 max_hw_sectors;
3075         bool prev_apst_enabled;
3076
3077         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3078         if (ret) {
3079                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3080                 return ret;
3081         }
3082         page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
3083         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3084
3085         if (ctrl->vs >= NVME_VS(1, 1, 0))
3086                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3087
3088         ret = nvme_identify_ctrl(ctrl, &id);
3089         if (ret) {
3090                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3091                 return -EIO;
3092         }
3093
3094         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3095                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3096                 if (ret < 0)
3097                         goto out_free;
3098         }
3099
3100         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3101                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3102
3103         if (!ctrl->identified) {
3104                 int i;
3105
3106                 /*
3107                  * Check for quirks.  Quirk can depend on firmware version,
3108                  * so, in principle, the set of quirks present can change
3109                  * across a reset.  As a possible future enhancement, we
3110                  * could re-scan for quirks every time we reinitialize
3111                  * the device, but we'd have to make sure that the driver
3112                  * behaves intelligently if the quirks change.
3113                  */
3114                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3115                         if (quirk_matches(id, &core_quirks[i]))
3116                                 ctrl->quirks |= core_quirks[i].quirks;
3117                 }
3118
3119                 ret = nvme_init_subsystem(ctrl, id);
3120                 if (ret)
3121                         goto out_free;
3122         }
3123         memcpy(ctrl->subsys->firmware_rev, id->fr,
3124                sizeof(ctrl->subsys->firmware_rev));
3125
3126         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3127                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3128                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3129         }
3130
3131         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3132         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3133         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3134
3135         ctrl->oacs = le16_to_cpu(id->oacs);
3136         ctrl->oncs = le16_to_cpu(id->oncs);
3137         ctrl->mtfa = le16_to_cpu(id->mtfa);
3138         ctrl->oaes = le32_to_cpu(id->oaes);
3139         ctrl->wctemp = le16_to_cpu(id->wctemp);
3140         ctrl->cctemp = le16_to_cpu(id->cctemp);
3141
3142         atomic_set(&ctrl->abort_limit, id->acl + 1);
3143         ctrl->vwc = id->vwc;
3144         if (id->mdts)
3145                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
3146         else
3147                 max_hw_sectors = UINT_MAX;
3148         ctrl->max_hw_sectors =
3149                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3150
3151         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3152         ctrl->sgls = le32_to_cpu(id->sgls);
3153         ctrl->kas = le16_to_cpu(id->kas);
3154         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3155         ctrl->ctratt = le32_to_cpu(id->ctratt);
3156
3157         if (id->rtd3e) {
3158                 /* us -> s */
3159                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3160
3161                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3162                                                  shutdown_timeout, 60);
3163
3164                 if (ctrl->shutdown_timeout != shutdown_timeout)
3165                         dev_info(ctrl->device,
3166                                  "Shutdown timeout set to %u seconds\n",
3167                                  ctrl->shutdown_timeout);
3168         } else
3169                 ctrl->shutdown_timeout = shutdown_timeout;
3170
3171         ctrl->npss = id->npss;
3172         ctrl->apsta = id->apsta;
3173         prev_apst_enabled = ctrl->apst_enabled;
3174         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3175                 if (force_apst && id->apsta) {
3176                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3177                         ctrl->apst_enabled = true;
3178                 } else {
3179                         ctrl->apst_enabled = false;
3180                 }
3181         } else {
3182                 ctrl->apst_enabled = id->apsta;
3183         }
3184         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3185
3186         if (ctrl->ops->flags & NVME_F_FABRICS) {
3187                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3188                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3189                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3190                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3191
3192                 /*
3193                  * In fabrics we need to verify the cntlid matches the
3194                  * admin connect
3195                  */
3196                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3197                         dev_err(ctrl->device,
3198                                 "Mismatching cntlid: Connect %u vs Identify "
3199                                 "%u, rejecting\n",
3200                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3201                         ret = -EINVAL;
3202                         goto out_free;
3203                 }
3204
3205                 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3206                         dev_err(ctrl->device,
3207                                 "keep-alive support is mandatory for fabrics\n");
3208                         ret = -EINVAL;
3209                         goto out_free;
3210                 }
3211         } else {
3212                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3213                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3214                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3215                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3216         }
3217
3218         ret = nvme_mpath_init_identify(ctrl, id);
3219         kfree(id);
3220
3221         if (ret < 0)
3222                 return ret;
3223
3224         if (ctrl->apst_enabled && !prev_apst_enabled)
3225                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3226         else if (!ctrl->apst_enabled && prev_apst_enabled)
3227                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3228
3229         ret = nvme_configure_apst(ctrl);
3230         if (ret < 0)
3231                 return ret;
3232
3233         ret = nvme_configure_timestamp(ctrl);
3234         if (ret < 0)
3235                 return ret;
3236
3237         ret = nvme_configure_directives(ctrl);
3238         if (ret < 0)
3239                 return ret;
3240
3241         ret = nvme_configure_acre(ctrl);
3242         if (ret < 0)
3243                 return ret;
3244
3245         if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3246                 /*
3247                  * Do not return errors unless we are in a controller reset,
3248                  * the controller works perfectly fine without hwmon.
3249                  */
3250                 ret = nvme_hwmon_init(ctrl);
3251                 if (ret == -EINTR)
3252                         return ret;
3253         }
3254
3255         ctrl->identified = true;
3256
3257         return 0;
3258
3259 out_free:
3260         kfree(id);
3261         return ret;
3262 }
3263 EXPORT_SYMBOL_GPL(nvme_init_identify);
3264
3265 static int nvme_dev_open(struct inode *inode, struct file *file)
3266 {
3267         struct nvme_ctrl *ctrl =
3268                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3269
3270         switch (ctrl->state) {
3271         case NVME_CTRL_LIVE:
3272                 break;
3273         default:
3274                 return -EWOULDBLOCK;
3275         }
3276
3277         nvme_get_ctrl(ctrl);
3278         if (!try_module_get(ctrl->ops->module)) {
3279                 nvme_put_ctrl(ctrl);
3280                 return -EINVAL;
3281         }
3282
3283         file->private_data = ctrl;
3284         return 0;
3285 }
3286
3287 static int nvme_dev_release(struct inode *inode, struct file *file)
3288 {
3289         struct nvme_ctrl *ctrl =
3290                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3291
3292         module_put(ctrl->ops->module);
3293         nvme_put_ctrl(ctrl);
3294         return 0;
3295 }
3296
3297 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3298 {
3299         struct nvme_ns *ns;
3300         int ret;
3301
3302         down_read(&ctrl->namespaces_rwsem);
3303         if (list_empty(&ctrl->namespaces)) {
3304                 ret = -ENOTTY;
3305                 goto out_unlock;
3306         }
3307
3308         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3309         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
3310                 dev_warn(ctrl->device,
3311                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3312                 ret = -EINVAL;
3313                 goto out_unlock;
3314         }
3315
3316         dev_warn(ctrl->device,
3317                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3318         kref_get(&ns->kref);
3319         up_read(&ctrl->namespaces_rwsem);
3320
3321         ret = nvme_user_cmd(ctrl, ns, argp);
3322         nvme_put_ns(ns);
3323         return ret;
3324
3325 out_unlock:
3326         up_read(&ctrl->namespaces_rwsem);
3327         return ret;
3328 }
3329
3330 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3331                 unsigned long arg)
3332 {
3333         struct nvme_ctrl *ctrl = file->private_data;
3334         void __user *argp = (void __user *)arg;
3335
3336         switch (cmd) {
3337         case NVME_IOCTL_ADMIN_CMD:
3338                 return nvme_user_cmd(ctrl, NULL, argp);
3339         case NVME_IOCTL_ADMIN64_CMD:
3340                 return nvme_user_cmd64(ctrl, NULL, argp);
3341         case NVME_IOCTL_IO_CMD:
3342                 return nvme_dev_user_cmd(ctrl, argp);
3343         case NVME_IOCTL_RESET:
3344                 if (!capable(CAP_SYS_ADMIN))
3345                         return -EACCES;
3346                 dev_warn(ctrl->device, "resetting controller\n");
3347                 return nvme_reset_ctrl_sync(ctrl);
3348         case NVME_IOCTL_SUBSYS_RESET:
3349                 if (!capable(CAP_SYS_ADMIN))
3350                         return -EACCES;
3351                 return nvme_reset_subsystem(ctrl);
3352         case NVME_IOCTL_RESCAN:
3353                 if (!capable(CAP_SYS_ADMIN))
3354                         return -EACCES;
3355                 nvme_queue_scan(ctrl);
3356                 return 0;
3357         default:
3358                 return -ENOTTY;
3359         }
3360 }
3361
3362 static const struct file_operations nvme_dev_fops = {
3363         .owner          = THIS_MODULE,
3364         .open           = nvme_dev_open,
3365         .release        = nvme_dev_release,
3366         .unlocked_ioctl = nvme_dev_ioctl,
3367         .compat_ioctl   = compat_ptr_ioctl,
3368 };
3369
3370 static ssize_t nvme_sysfs_reset(struct device *dev,
3371                                 struct device_attribute *attr, const char *buf,
3372                                 size_t count)
3373 {
3374         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3375         int ret;
3376
3377         ret = nvme_reset_ctrl_sync(ctrl);
3378         if (ret < 0)
3379                 return ret;
3380         return count;
3381 }
3382 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3383
3384 static ssize_t nvme_sysfs_rescan(struct device *dev,
3385                                 struct device_attribute *attr, const char *buf,
3386                                 size_t count)
3387 {
3388         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3389
3390         nvme_queue_scan(ctrl);
3391         return count;
3392 }
3393 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3394
3395 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3396 {
3397         struct gendisk *disk = dev_to_disk(dev);
3398
3399         if (disk->fops == &nvme_fops)
3400                 return nvme_get_ns_from_dev(dev)->head;
3401         else
3402                 return disk->private_data;
3403 }
3404
3405 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3406                 char *buf)
3407 {
3408         struct nvme_ns_head *head = dev_to_ns_head(dev);
3409         struct nvme_ns_ids *ids = &head->ids;
3410         struct nvme_subsystem *subsys = head->subsys;
3411         int serial_len = sizeof(subsys->serial);
3412         int model_len = sizeof(subsys->model);
3413
3414         if (!uuid_is_null(&ids->uuid))
3415                 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3416
3417         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3418                 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3419
3420         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3421                 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3422
3423         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3424                                   subsys->serial[serial_len - 1] == '\0'))
3425                 serial_len--;
3426         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3427                                  subsys->model[model_len - 1] == '\0'))
3428                 model_len--;
3429
3430         return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3431                 serial_len, subsys->serial, model_len, subsys->model,
3432                 head->ns_id);
3433 }
3434 static DEVICE_ATTR_RO(wwid);
3435
3436 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3437                 char *buf)
3438 {
3439         return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3440 }
3441 static DEVICE_ATTR_RO(nguid);
3442
3443 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3444                 char *buf)
3445 {
3446         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3447
3448         /* For backward compatibility expose the NGUID to userspace if
3449          * we have no UUID set
3450          */
3451         if (uuid_is_null(&ids->uuid)) {
3452                 dev_warn_ratelimited(dev,
3453                         "No UUID available providing old NGUID\n");
3454                 return sysfs_emit(buf, "%pU\n", ids->nguid);
3455         }
3456         return sysfs_emit(buf, "%pU\n", &ids->uuid);
3457 }
3458 static DEVICE_ATTR_RO(uuid);
3459
3460 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3461                 char *buf)
3462 {
3463         return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3464 }
3465 static DEVICE_ATTR_RO(eui);
3466
3467 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3468                 char *buf)
3469 {
3470         return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3471 }
3472 static DEVICE_ATTR_RO(nsid);
3473
3474 static struct attribute *nvme_ns_id_attrs[] = {
3475         &dev_attr_wwid.attr,
3476         &dev_attr_uuid.attr,
3477         &dev_attr_nguid.attr,
3478         &dev_attr_eui.attr,
3479         &dev_attr_nsid.attr,
3480 #ifdef CONFIG_NVME_MULTIPATH
3481         &dev_attr_ana_grpid.attr,
3482         &dev_attr_ana_state.attr,
3483 #endif
3484         NULL,
3485 };
3486
3487 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3488                 struct attribute *a, int n)
3489 {
3490         struct device *dev = container_of(kobj, struct device, kobj);
3491         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3492
3493         if (a == &dev_attr_uuid.attr) {
3494                 if (uuid_is_null(&ids->uuid) &&
3495                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3496                         return 0;
3497         }
3498         if (a == &dev_attr_nguid.attr) {
3499                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3500                         return 0;
3501         }
3502         if (a == &dev_attr_eui.attr) {
3503                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3504                         return 0;
3505         }
3506 #ifdef CONFIG_NVME_MULTIPATH
3507         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3508                 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3509                         return 0;
3510                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3511                         return 0;
3512         }
3513 #endif
3514         return a->mode;
3515 }
3516
3517 static const struct attribute_group nvme_ns_id_attr_group = {
3518         .attrs          = nvme_ns_id_attrs,
3519         .is_visible     = nvme_ns_id_attrs_are_visible,
3520 };
3521
3522 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3523         &nvme_ns_id_attr_group,
3524 #ifdef CONFIG_NVM
3525         &nvme_nvm_attr_group,
3526 #endif
3527         NULL,
3528 };
3529
3530 #define nvme_show_str_function(field)                                           \
3531 static ssize_t  field##_show(struct device *dev,                                \
3532                             struct device_attribute *attr, char *buf)           \
3533 {                                                                               \
3534         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3535         return sysfs_emit(buf, "%.*s\n",                                        \
3536                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3537 }                                                                               \
3538 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3539
3540 nvme_show_str_function(model);
3541 nvme_show_str_function(serial);
3542 nvme_show_str_function(firmware_rev);
3543
3544 #define nvme_show_int_function(field)                                           \
3545 static ssize_t  field##_show(struct device *dev,                                \
3546                             struct device_attribute *attr, char *buf)           \
3547 {                                                                               \
3548         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3549         return sysfs_emit(buf, "%d\n", ctrl->field);                            \
3550 }                                                                               \
3551 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3552
3553 nvme_show_int_function(cntlid);
3554 nvme_show_int_function(numa_node);
3555 nvme_show_int_function(queue_count);
3556 nvme_show_int_function(sqsize);
3557
3558 static ssize_t nvme_sysfs_delete(struct device *dev,
3559                                 struct device_attribute *attr, const char *buf,
3560                                 size_t count)
3561 {
3562         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3563
3564         if (device_remove_file_self(dev, attr))
3565                 nvme_delete_ctrl_sync(ctrl);
3566         return count;
3567 }
3568 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3569
3570 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3571                                          struct device_attribute *attr,
3572                                          char *buf)
3573 {
3574         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3575
3576         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3577 }
3578 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3579
3580 static ssize_t nvme_sysfs_show_state(struct device *dev,
3581                                      struct device_attribute *attr,
3582                                      char *buf)
3583 {
3584         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3585         static const char *const state_name[] = {
3586                 [NVME_CTRL_NEW]         = "new",
3587                 [NVME_CTRL_LIVE]        = "live",
3588                 [NVME_CTRL_RESETTING]   = "resetting",
3589                 [NVME_CTRL_CONNECTING]  = "connecting",
3590                 [NVME_CTRL_DELETING]    = "deleting",
3591                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3592                 [NVME_CTRL_DEAD]        = "dead",
3593         };
3594
3595         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3596             state_name[ctrl->state])
3597                 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3598
3599         return sysfs_emit(buf, "unknown state\n");
3600 }
3601
3602 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3603
3604 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3605                                          struct device_attribute *attr,
3606                                          char *buf)
3607 {
3608         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3609
3610         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3611 }
3612 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3613
3614 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3615                                         struct device_attribute *attr,
3616                                         char *buf)
3617 {
3618         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3619
3620         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3621 }
3622 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3623
3624 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3625                                         struct device_attribute *attr,
3626                                         char *buf)
3627 {
3628         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3629
3630         return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3631 }
3632 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3633
3634 static ssize_t nvme_sysfs_show_address(struct device *dev,
3635                                          struct device_attribute *attr,
3636                                          char *buf)
3637 {
3638         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3639
3640         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3641 }
3642 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3643
3644 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3645                 struct device_attribute *attr, char *buf)
3646 {
3647         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3648         struct nvmf_ctrl_options *opts = ctrl->opts;
3649
3650         if (ctrl->opts->max_reconnects == -1)
3651                 return sysfs_emit(buf, "off\n");
3652         return sysfs_emit(buf, "%d\n",
3653                           opts->max_reconnects * opts->reconnect_delay);
3654 }
3655
3656 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3657                 struct device_attribute *attr, const char *buf, size_t count)
3658 {
3659         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3660         struct nvmf_ctrl_options *opts = ctrl->opts;
3661         int ctrl_loss_tmo, err;
3662
3663         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3664         if (err)
3665                 return -EINVAL;
3666
3667         else if (ctrl_loss_tmo < 0)
3668                 opts->max_reconnects = -1;
3669         else
3670                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3671                                                 opts->reconnect_delay);
3672         return count;
3673 }
3674 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3675         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3676
3677 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3678                 struct device_attribute *attr, char *buf)
3679 {
3680         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3681
3682         if (ctrl->opts->reconnect_delay == -1)
3683                 return sysfs_emit(buf, "off\n");
3684         return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3685 }
3686
3687 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3688                 struct device_attribute *attr, const char *buf, size_t count)
3689 {
3690         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3691         unsigned int v;
3692         int err;
3693
3694         err = kstrtou32(buf, 10, &v);
3695         if (err)
3696                 return err;
3697
3698         ctrl->opts->reconnect_delay = v;
3699         return count;
3700 }
3701 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3702         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3703
3704 static struct attribute *nvme_dev_attrs[] = {
3705         &dev_attr_reset_controller.attr,
3706         &dev_attr_rescan_controller.attr,
3707         &dev_attr_model.attr,
3708         &dev_attr_serial.attr,
3709         &dev_attr_firmware_rev.attr,
3710         &dev_attr_cntlid.attr,
3711         &dev_attr_delete_controller.attr,
3712         &dev_attr_transport.attr,
3713         &dev_attr_subsysnqn.attr,
3714         &dev_attr_address.attr,
3715         &dev_attr_state.attr,
3716         &dev_attr_numa_node.attr,
3717         &dev_attr_queue_count.attr,
3718         &dev_attr_sqsize.attr,
3719         &dev_attr_hostnqn.attr,
3720         &dev_attr_hostid.attr,
3721         &dev_attr_ctrl_loss_tmo.attr,
3722         &dev_attr_reconnect_delay.attr,
3723         NULL
3724 };
3725
3726 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3727                 struct attribute *a, int n)
3728 {
3729         struct device *dev = container_of(kobj, struct device, kobj);
3730         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3731
3732         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3733                 return 0;
3734         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3735                 return 0;
3736         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3737                 return 0;
3738         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3739                 return 0;
3740         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3741                 return 0;
3742         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3743                 return 0;
3744
3745         return a->mode;
3746 }
3747
3748 static struct attribute_group nvme_dev_attrs_group = {
3749         .attrs          = nvme_dev_attrs,
3750         .is_visible     = nvme_dev_attrs_are_visible,
3751 };
3752
3753 static const struct attribute_group *nvme_dev_attr_groups[] = {
3754         &nvme_dev_attrs_group,
3755         NULL,
3756 };
3757
3758 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
3759                 unsigned nsid)
3760 {
3761         struct nvme_ns_head *h;
3762
3763         lockdep_assert_held(&subsys->lock);
3764
3765         list_for_each_entry(h, &subsys->nsheads, entry) {
3766                 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3767                         return h;
3768         }
3769
3770         return NULL;
3771 }
3772
3773 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3774                 struct nvme_ns_ids *ids)
3775 {
3776         struct nvme_ns_head *h;
3777
3778         lockdep_assert_held(&subsys->lock);
3779
3780         list_for_each_entry(h, &subsys->nsheads, entry) {
3781                 if (nvme_ns_ids_valid(ids) && nvme_ns_ids_equal(ids, &h->ids))
3782                         return -EINVAL;
3783         }
3784
3785         return 0;
3786 }
3787
3788 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3789                 unsigned nsid, struct nvme_ns_ids *ids)
3790 {
3791         struct nvme_ns_head *head;
3792         size_t size = sizeof(*head);
3793         int ret = -ENOMEM;
3794
3795 #ifdef CONFIG_NVME_MULTIPATH
3796         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3797 #endif
3798
3799         head = kzalloc(size, GFP_KERNEL);
3800         if (!head)
3801                 goto out;
3802         ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3803         if (ret < 0)
3804                 goto out_free_head;
3805         head->instance = ret;
3806         INIT_LIST_HEAD(&head->list);
3807         ret = init_srcu_struct(&head->srcu);
3808         if (ret)
3809                 goto out_ida_remove;
3810         head->subsys = ctrl->subsys;
3811         head->ns_id = nsid;
3812         head->ids = *ids;
3813         kref_init(&head->ref);
3814
3815         ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &head->ids);
3816         if (ret) {
3817                 dev_err(ctrl->device,
3818                         "duplicate IDs for nsid %d\n", nsid);
3819                 goto out_cleanup_srcu;
3820         }
3821
3822         if (head->ids.csi) {
3823                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3824                 if (ret)
3825                         goto out_cleanup_srcu;
3826         } else
3827                 head->effects = ctrl->effects;
3828
3829         ret = nvme_mpath_alloc_disk(ctrl, head);
3830         if (ret)
3831                 goto out_cleanup_srcu;
3832
3833         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3834
3835         kref_get(&ctrl->subsys->ref);
3836
3837         return head;
3838 out_cleanup_srcu:
3839         cleanup_srcu_struct(&head->srcu);
3840 out_ida_remove:
3841         ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3842 out_free_head:
3843         kfree(head);
3844 out:
3845         if (ret > 0)
3846                 ret = blk_status_to_errno(nvme_error_status(ret));
3847         return ERR_PTR(ret);
3848 }
3849
3850 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3851                 struct nvme_ns_ids *ids, bool is_shared)
3852 {
3853         struct nvme_ctrl *ctrl = ns->ctrl;
3854         struct nvme_ns_head *head = NULL;
3855         int ret = 0;
3856
3857         mutex_lock(&ctrl->subsys->lock);
3858         head = nvme_find_ns_head(ctrl->subsys, nsid);
3859         if (!head) {
3860                 head = nvme_alloc_ns_head(ctrl, nsid, ids);
3861                 if (IS_ERR(head)) {
3862                         ret = PTR_ERR(head);
3863                         goto out_unlock;
3864                 }
3865                 head->shared = is_shared;
3866         } else {
3867                 ret = -EINVAL;
3868                 if (!is_shared || !head->shared) {
3869                         dev_err(ctrl->device,
3870                                 "Duplicate unshared namespace %d\n", nsid);
3871                         goto out_put_ns_head;
3872                 }
3873                 if (!nvme_ns_ids_equal(&head->ids, ids)) {
3874                         dev_err(ctrl->device,
3875                                 "IDs don't match for shared namespace %d\n",
3876                                         nsid);
3877                         goto out_put_ns_head;
3878                 }
3879         }
3880
3881         list_add_tail(&ns->siblings, &head->list);
3882         ns->head = head;
3883         mutex_unlock(&ctrl->subsys->lock);
3884         return 0;
3885
3886 out_put_ns_head:
3887         nvme_put_ns_head(head);
3888 out_unlock:
3889         mutex_unlock(&ctrl->subsys->lock);
3890         return ret;
3891 }
3892
3893 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3894 {
3895         struct nvme_ns *ns, *ret = NULL;
3896
3897         down_read(&ctrl->namespaces_rwsem);
3898         list_for_each_entry(ns, &ctrl->namespaces, list) {
3899                 if (ns->head->ns_id == nsid) {
3900                         if (!kref_get_unless_zero(&ns->kref))
3901                                 continue;
3902                         ret = ns;
3903                         break;
3904                 }
3905                 if (ns->head->ns_id > nsid)
3906                         break;
3907         }
3908         up_read(&ctrl->namespaces_rwsem);
3909         return ret;
3910 }
3911 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3912
3913 /*
3914  * Add the namespace to the controller list while keeping the list ordered.
3915  */
3916 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3917 {
3918         struct nvme_ns *tmp;
3919
3920         list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
3921                 if (tmp->head->ns_id < ns->head->ns_id) {
3922                         list_add(&ns->list, &tmp->list);
3923                         return;
3924                 }
3925         }
3926         list_add(&ns->list, &ns->ctrl->namespaces);
3927 }
3928
3929 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
3930                 struct nvme_ns_ids *ids)
3931 {
3932         struct nvme_ns *ns;
3933         struct gendisk *disk;
3934         struct nvme_id_ns *id;
3935         char disk_name[DISK_NAME_LEN];
3936         int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3937
3938         if (nvme_identify_ns(ctrl, nsid, ids, &id))
3939                 return;
3940
3941         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3942         if (!ns)
3943                 goto out_free_id;
3944
3945         ns->queue = blk_mq_init_queue(ctrl->tagset);
3946         if (IS_ERR(ns->queue))
3947                 goto out_free_ns;
3948
3949         if (ctrl->opts && ctrl->opts->data_digest)
3950                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3951
3952         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3953         if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3954                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3955
3956         ns->queue->queuedata = ns;
3957         ns->ctrl = ctrl;
3958         kref_init(&ns->kref);
3959
3960         ret = nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED);
3961         if (ret)
3962                 goto out_free_queue;
3963         nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3964
3965         disk = alloc_disk_node(0, node);
3966         if (!disk)
3967                 goto out_unlink_ns;
3968
3969         disk->fops = &nvme_fops;
3970         disk->private_data = ns;
3971         disk->queue = ns->queue;
3972         disk->flags = flags;
3973         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3974         ns->disk = disk;
3975
3976         if (nvme_update_ns_info(ns, id))
3977                 goto out_put_disk;
3978
3979         if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3980                 ret = nvme_nvm_register(ns, disk_name, node);
3981                 if (ret) {
3982                         dev_warn(ctrl->device, "LightNVM init failure\n");
3983                         goto out_put_disk;
3984                 }
3985         }
3986
3987         down_write(&ctrl->namespaces_rwsem);
3988         nvme_ns_add_to_ctrl_list(ns);
3989         up_write(&ctrl->namespaces_rwsem);
3990         nvme_get_ctrl(ctrl);
3991
3992         device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3993
3994         nvme_mpath_add_disk(ns, id);
3995         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3996         kfree(id);
3997
3998         return;
3999  out_put_disk:
4000         /* prevent double queue cleanup */
4001         ns->disk->queue = NULL;
4002         put_disk(ns->disk);
4003  out_unlink_ns:
4004         mutex_lock(&ctrl->subsys->lock);
4005         list_del_rcu(&ns->siblings);
4006         if (list_empty(&ns->head->list))
4007                 list_del_init(&ns->head->entry);
4008         mutex_unlock(&ctrl->subsys->lock);
4009         nvme_put_ns_head(ns->head);
4010  out_free_queue:
4011         blk_cleanup_queue(ns->queue);
4012  out_free_ns:
4013         kfree(ns);
4014  out_free_id:
4015         kfree(id);
4016 }
4017
4018 static void nvme_ns_remove(struct nvme_ns *ns)
4019 {
4020         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4021                 return;
4022
4023         set_capacity(ns->disk, 0);
4024         nvme_fault_inject_fini(&ns->fault_inject);
4025
4026         mutex_lock(&ns->ctrl->subsys->lock);
4027         list_del_rcu(&ns->siblings);
4028         if (list_empty(&ns->head->list))
4029                 list_del_init(&ns->head->entry);
4030         mutex_unlock(&ns->ctrl->subsys->lock);
4031
4032         synchronize_rcu(); /* guarantee not available in head->list */
4033         nvme_mpath_clear_current_path(ns);
4034         synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
4035
4036         if (ns->disk->flags & GENHD_FL_UP) {
4037                 del_gendisk(ns->disk);
4038                 blk_cleanup_queue(ns->queue);
4039                 if (blk_get_integrity(ns->disk))
4040                         blk_integrity_unregister(ns->disk);
4041         }
4042
4043         down_write(&ns->ctrl->namespaces_rwsem);
4044         list_del_init(&ns->list);
4045         up_write(&ns->ctrl->namespaces_rwsem);
4046
4047         nvme_mpath_check_last_path(ns);
4048         nvme_put_ns(ns);
4049 }
4050
4051 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4052 {
4053         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4054
4055         if (ns) {
4056                 nvme_ns_remove(ns);
4057                 nvme_put_ns(ns);
4058         }
4059 }
4060
4061 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids)
4062 {
4063         struct nvme_id_ns *id;
4064         int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4065
4066         if (test_bit(NVME_NS_DEAD, &ns->flags))
4067                 goto out;
4068
4069         ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
4070         if (ret)
4071                 goto out;
4072
4073         ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4074         if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
4075                 dev_err(ns->ctrl->device,
4076                         "identifiers changed for nsid %d\n", ns->head->ns_id);
4077                 goto out_free_id;
4078         }
4079
4080         ret = nvme_update_ns_info(ns, id);
4081
4082 out_free_id:
4083         kfree(id);
4084 out:
4085         /*
4086          * Only remove the namespace if we got a fatal error back from the
4087          * device, otherwise ignore the error and just move on.
4088          *
4089          * TODO: we should probably schedule a delayed retry here.
4090          */
4091         if (ret > 0 && (ret & NVME_SC_DNR))
4092                 nvme_ns_remove(ns);
4093         else
4094                 revalidate_disk_size(ns->disk, true);
4095 }
4096
4097 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4098 {
4099         struct nvme_ns_ids ids = { };
4100         struct nvme_ns *ns;
4101
4102         if (nvme_identify_ns_descs(ctrl, nsid, &ids))
4103                 return;
4104
4105         ns = nvme_find_get_ns(ctrl, nsid);
4106         if (ns) {
4107                 nvme_validate_ns(ns, &ids);
4108                 nvme_put_ns(ns);
4109                 return;
4110         }
4111
4112         switch (ids.csi) {
4113         case NVME_CSI_NVM:
4114                 nvme_alloc_ns(ctrl, nsid, &ids);
4115                 break;
4116         case NVME_CSI_ZNS:
4117                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
4118                         dev_warn(ctrl->device,
4119                                 "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
4120                                 nsid);
4121                         break;
4122                 }
4123                 if (!nvme_multi_css(ctrl)) {
4124                         dev_warn(ctrl->device,
4125                                 "command set not reported for nsid: %d\n",
4126                                 nsid);
4127                         break;
4128                 }
4129                 nvme_alloc_ns(ctrl, nsid, &ids);
4130                 break;
4131         default:
4132                 dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
4133                         ids.csi, nsid);
4134                 break;
4135         }
4136 }
4137
4138 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4139                                         unsigned nsid)
4140 {
4141         struct nvme_ns *ns, *next;
4142         LIST_HEAD(rm_list);
4143
4144         down_write(&ctrl->namespaces_rwsem);
4145         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4146                 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4147                         list_move_tail(&ns->list, &rm_list);
4148         }
4149         up_write(&ctrl->namespaces_rwsem);
4150
4151         list_for_each_entry_safe(ns, next, &rm_list, list)
4152                 nvme_ns_remove(ns);
4153
4154 }
4155
4156 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4157 {
4158         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4159         __le32 *ns_list;
4160         u32 prev = 0;
4161         int ret = 0, i;
4162
4163         if (nvme_ctrl_limited_cns(ctrl))
4164                 return -EOPNOTSUPP;
4165
4166         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4167         if (!ns_list)
4168                 return -ENOMEM;
4169
4170         for (;;) {
4171                 struct nvme_command cmd = {
4172                         .identify.opcode        = nvme_admin_identify,
4173                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4174                         .identify.nsid          = cpu_to_le32(prev),
4175                 };
4176
4177                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4178                                             NVME_IDENTIFY_DATA_SIZE);
4179                 if (ret)
4180                         goto free;
4181
4182                 for (i = 0; i < nr_entries; i++) {
4183                         u32 nsid = le32_to_cpu(ns_list[i]);
4184
4185                         if (!nsid)      /* end of the list? */
4186                                 goto out;
4187                         nvme_validate_or_alloc_ns(ctrl, nsid);
4188                         while (++prev < nsid)
4189                                 nvme_ns_remove_by_nsid(ctrl, prev);
4190                 }
4191         }
4192  out:
4193         nvme_remove_invalid_namespaces(ctrl, prev);
4194  free:
4195         kfree(ns_list);
4196         return ret;
4197 }
4198
4199 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4200 {
4201         struct nvme_id_ctrl *id;
4202         u32 nn, i;
4203
4204         if (nvme_identify_ctrl(ctrl, &id))
4205                 return;
4206         nn = le32_to_cpu(id->nn);
4207         kfree(id);
4208
4209         for (i = 1; i <= nn; i++)
4210                 nvme_validate_or_alloc_ns(ctrl, i);
4211
4212         nvme_remove_invalid_namespaces(ctrl, nn);
4213 }
4214
4215 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4216 {
4217         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4218         __le32 *log;
4219         int error;
4220
4221         log = kzalloc(log_size, GFP_KERNEL);
4222         if (!log)
4223                 return;
4224
4225         /*
4226          * We need to read the log to clear the AEN, but we don't want to rely
4227          * on it for the changed namespace information as userspace could have
4228          * raced with us in reading the log page, which could cause us to miss
4229          * updates.
4230          */
4231         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4232                         NVME_CSI_NVM, log, log_size, 0);
4233         if (error)
4234                 dev_warn(ctrl->device,
4235                         "reading changed ns log failed: %d\n", error);
4236
4237         kfree(log);
4238 }
4239
4240 static void nvme_scan_work(struct work_struct *work)
4241 {
4242         struct nvme_ctrl *ctrl =
4243                 container_of(work, struct nvme_ctrl, scan_work);
4244
4245         /* No tagset on a live ctrl means IO queues could not created */
4246         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4247                 return;
4248
4249         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4250                 dev_info(ctrl->device, "rescanning namespaces.\n");
4251                 nvme_clear_changed_ns_log(ctrl);
4252         }
4253
4254         mutex_lock(&ctrl->scan_lock);
4255         if (nvme_scan_ns_list(ctrl) != 0)
4256                 nvme_scan_ns_sequential(ctrl);
4257         mutex_unlock(&ctrl->scan_lock);
4258 }
4259
4260 /*
4261  * This function iterates the namespace list unlocked to allow recovery from
4262  * controller failure. It is up to the caller to ensure the namespace list is
4263  * not modified by scan work while this function is executing.
4264  */
4265 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4266 {
4267         struct nvme_ns *ns, *next;
4268         LIST_HEAD(ns_list);
4269
4270         /*
4271          * make sure to requeue I/O to all namespaces as these
4272          * might result from the scan itself and must complete
4273          * for the scan_work to make progress
4274          */
4275         nvme_mpath_clear_ctrl_paths(ctrl);
4276
4277         /* prevent racing with ns scanning */
4278         flush_work(&ctrl->scan_work);
4279
4280         /*
4281          * The dead states indicates the controller was not gracefully
4282          * disconnected. In that case, we won't be able to flush any data while
4283          * removing the namespaces' disks; fail all the queues now to avoid
4284          * potentially having to clean up the failed sync later.
4285          */
4286         if (ctrl->state == NVME_CTRL_DEAD)
4287                 nvme_kill_queues(ctrl);
4288
4289         /* this is a no-op when called from the controller reset handler */
4290         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4291
4292         down_write(&ctrl->namespaces_rwsem);
4293         list_splice_init(&ctrl->namespaces, &ns_list);
4294         up_write(&ctrl->namespaces_rwsem);
4295
4296         list_for_each_entry_safe(ns, next, &ns_list, list)
4297                 nvme_ns_remove(ns);
4298 }
4299 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4300
4301 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4302 {
4303         struct nvme_ctrl *ctrl =
4304                 container_of(dev, struct nvme_ctrl, ctrl_device);
4305         struct nvmf_ctrl_options *opts = ctrl->opts;
4306         int ret;
4307
4308         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4309         if (ret)
4310                 return ret;
4311
4312         if (opts) {
4313                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4314                 if (ret)
4315                         return ret;
4316
4317                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4318                                 opts->trsvcid ?: "none");
4319                 if (ret)
4320                         return ret;
4321
4322                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4323                                 opts->host_traddr ?: "none");
4324         }
4325         return ret;
4326 }
4327
4328 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4329 {
4330         char *envp[2] = { NULL, NULL };
4331         u32 aen_result = ctrl->aen_result;
4332
4333         ctrl->aen_result = 0;
4334         if (!aen_result)
4335                 return;
4336
4337         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4338         if (!envp[0])
4339                 return;
4340         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4341         kfree(envp[0]);
4342 }
4343
4344 static void nvme_async_event_work(struct work_struct *work)
4345 {
4346         struct nvme_ctrl *ctrl =
4347                 container_of(work, struct nvme_ctrl, async_event_work);
4348
4349         nvme_aen_uevent(ctrl);
4350
4351         /*
4352          * The transport drivers must guarantee AER submission here is safe by
4353          * flushing ctrl async_event_work after changing the controller state
4354          * from LIVE and before freeing the admin queue.
4355         */
4356         if (ctrl->state == NVME_CTRL_LIVE)
4357                 ctrl->ops->submit_async_event(ctrl);
4358 }
4359
4360 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4361 {
4362
4363         u32 csts;
4364
4365         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4366                 return false;
4367
4368         if (csts == ~0)
4369                 return false;
4370
4371         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4372 }
4373
4374 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4375 {
4376         struct nvme_fw_slot_info_log *log;
4377
4378         log = kmalloc(sizeof(*log), GFP_KERNEL);
4379         if (!log)
4380                 return;
4381
4382         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4383                         log, sizeof(*log), 0))
4384                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4385         kfree(log);
4386 }
4387
4388 static void nvme_fw_act_work(struct work_struct *work)
4389 {
4390         struct nvme_ctrl *ctrl = container_of(work,
4391                                 struct nvme_ctrl, fw_act_work);
4392         unsigned long fw_act_timeout;
4393
4394         if (ctrl->mtfa)
4395                 fw_act_timeout = jiffies +
4396                                 msecs_to_jiffies(ctrl->mtfa * 100);
4397         else
4398                 fw_act_timeout = jiffies +
4399                                 msecs_to_jiffies(admin_timeout * 1000);
4400
4401         nvme_stop_queues(ctrl);
4402         while (nvme_ctrl_pp_status(ctrl)) {
4403                 if (time_after(jiffies, fw_act_timeout)) {
4404                         dev_warn(ctrl->device,
4405                                 "Fw activation timeout, reset controller\n");
4406                         nvme_try_sched_reset(ctrl);
4407                         return;
4408                 }
4409                 msleep(100);
4410         }
4411
4412         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4413                 return;
4414
4415         nvme_start_queues(ctrl);
4416         /* read FW slot information to clear the AER */
4417         nvme_get_fw_slot_info(ctrl);
4418 }
4419
4420 static u32 nvme_aer_type(u32 result)
4421 {
4422         return result & 0x7;
4423 }
4424
4425 static u32 nvme_aer_subtype(u32 result)
4426 {
4427         return (result & 0xff00) >> 8;
4428 }
4429
4430 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4431 {
4432         u32 aer_notice_type = nvme_aer_subtype(result);
4433
4434         switch (aer_notice_type) {
4435         case NVME_AER_NOTICE_NS_CHANGED:
4436                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4437                 nvme_queue_scan(ctrl);
4438                 break;
4439         case NVME_AER_NOTICE_FW_ACT_STARTING:
4440                 /*
4441                  * We are (ab)using the RESETTING state to prevent subsequent
4442                  * recovery actions from interfering with the controller's
4443                  * firmware activation.
4444                  */
4445                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4446                         queue_work(nvme_wq, &ctrl->fw_act_work);
4447                 break;
4448 #ifdef CONFIG_NVME_MULTIPATH
4449         case NVME_AER_NOTICE_ANA:
4450                 if (!ctrl->ana_log_buf)
4451                         break;
4452                 queue_work(nvme_wq, &ctrl->ana_work);
4453                 break;
4454 #endif
4455         case NVME_AER_NOTICE_DISC_CHANGED:
4456                 ctrl->aen_result = result;
4457                 break;
4458         default:
4459                 dev_warn(ctrl->device, "async event result %08x\n", result);
4460         }
4461 }
4462
4463 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4464 {
4465         dev_warn(ctrl->device, "resetting controller due to AER\n");
4466         nvme_reset_ctrl(ctrl);
4467 }
4468
4469 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4470                 volatile union nvme_result *res)
4471 {
4472         u32 result = le32_to_cpu(res->u32);
4473         u32 aer_type = nvme_aer_type(result);
4474         u32 aer_subtype = nvme_aer_subtype(result);
4475
4476         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4477                 return;
4478
4479         trace_nvme_async_event(ctrl, result);
4480         switch (aer_type) {
4481         case NVME_AER_NOTICE:
4482                 nvme_handle_aen_notice(ctrl, result);
4483                 break;
4484         case NVME_AER_ERROR:
4485                 /*
4486                  * For a persistent internal error, don't run async_event_work
4487                  * to submit a new AER. The controller reset will do it.
4488                  */
4489                 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4490                         nvme_handle_aer_persistent_error(ctrl);
4491                         return;
4492                 }
4493                 fallthrough;
4494         case NVME_AER_SMART:
4495         case NVME_AER_CSS:
4496         case NVME_AER_VS:
4497                 ctrl->aen_result = result;
4498                 break;
4499         default:
4500                 break;
4501         }
4502         queue_work(nvme_wq, &ctrl->async_event_work);
4503 }
4504 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4505
4506 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4507 {
4508         nvme_mpath_stop(ctrl);
4509         nvme_stop_keep_alive(ctrl);
4510         flush_work(&ctrl->async_event_work);
4511         cancel_work_sync(&ctrl->fw_act_work);
4512         if (ctrl->ops->stop_ctrl)
4513                 ctrl->ops->stop_ctrl(ctrl);
4514 }
4515 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4516
4517 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4518 {
4519         nvme_start_keep_alive(ctrl);
4520
4521         nvme_enable_aen(ctrl);
4522
4523         if (ctrl->queue_count > 1) {
4524                 nvme_queue_scan(ctrl);
4525                 nvme_start_queues(ctrl);
4526                 nvme_mpath_update(ctrl);
4527         }
4528 }
4529 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4530
4531 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4532 {
4533         nvme_hwmon_exit(ctrl);
4534         nvme_fault_inject_fini(&ctrl->fault_inject);
4535         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4536         cdev_device_del(&ctrl->cdev, ctrl->device);
4537         nvme_put_ctrl(ctrl);
4538 }
4539 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4540
4541 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4542 {
4543         struct nvme_effects_log *cel;
4544         unsigned long i;
4545
4546         xa_for_each (&ctrl->cels, i, cel) {
4547                 xa_erase(&ctrl->cels, i);
4548                 kfree(cel);
4549         }
4550
4551         xa_destroy(&ctrl->cels);
4552 }
4553
4554 static void nvme_free_ctrl(struct device *dev)
4555 {
4556         struct nvme_ctrl *ctrl =
4557                 container_of(dev, struct nvme_ctrl, ctrl_device);
4558         struct nvme_subsystem *subsys = ctrl->subsys;
4559
4560         if (!subsys || ctrl->instance != subsys->instance)
4561                 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4562
4563         nvme_free_cels(ctrl);
4564         nvme_mpath_uninit(ctrl);
4565         __free_page(ctrl->discard_page);
4566
4567         if (subsys) {
4568                 mutex_lock(&nvme_subsystems_lock);
4569                 list_del(&ctrl->subsys_entry);
4570                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4571                 mutex_unlock(&nvme_subsystems_lock);
4572         }
4573
4574         ctrl->ops->free_ctrl(ctrl);
4575
4576         if (subsys)
4577                 nvme_put_subsystem(subsys);
4578 }
4579
4580 /*
4581  * Initialize a NVMe controller structures.  This needs to be called during
4582  * earliest initialization so that we have the initialized structured around
4583  * during probing.
4584  */
4585 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4586                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4587 {
4588         int ret;
4589
4590         ctrl->state = NVME_CTRL_NEW;
4591         spin_lock_init(&ctrl->lock);
4592         mutex_init(&ctrl->scan_lock);
4593         INIT_LIST_HEAD(&ctrl->namespaces);
4594         xa_init(&ctrl->cels);
4595         init_rwsem(&ctrl->namespaces_rwsem);
4596         ctrl->dev = dev;
4597         ctrl->ops = ops;
4598         ctrl->quirks = quirks;
4599         ctrl->numa_node = NUMA_NO_NODE;
4600         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4601         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4602         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4603         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4604         init_waitqueue_head(&ctrl->state_wq);
4605
4606         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4607         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4608         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4609
4610         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4611                         PAGE_SIZE);
4612         ctrl->discard_page = alloc_page(GFP_KERNEL);
4613         if (!ctrl->discard_page) {
4614                 ret = -ENOMEM;
4615                 goto out;
4616         }
4617
4618         ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4619         if (ret < 0)
4620                 goto out;
4621         ctrl->instance = ret;
4622
4623         device_initialize(&ctrl->ctrl_device);
4624         ctrl->device = &ctrl->ctrl_device;
4625         ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4626         ctrl->device->class = nvme_class;
4627         ctrl->device->parent = ctrl->dev;
4628         ctrl->device->groups = nvme_dev_attr_groups;
4629         ctrl->device->release = nvme_free_ctrl;
4630         dev_set_drvdata(ctrl->device, ctrl);
4631         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4632         if (ret)
4633                 goto out_release_instance;
4634
4635         nvme_get_ctrl(ctrl);
4636         cdev_init(&ctrl->cdev, &nvme_dev_fops);
4637         ctrl->cdev.owner = ops->module;
4638         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4639         if (ret)
4640                 goto out_free_name;
4641
4642         /*
4643          * Initialize latency tolerance controls.  The sysfs files won't
4644          * be visible to userspace unless the device actually supports APST.
4645          */
4646         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4647         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4648                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4649
4650         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4651         nvme_mpath_init_ctrl(ctrl);
4652
4653         return 0;
4654 out_free_name:
4655         nvme_put_ctrl(ctrl);
4656         kfree_const(ctrl->device->kobj.name);
4657 out_release_instance:
4658         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4659 out:
4660         if (ctrl->discard_page)
4661                 __free_page(ctrl->discard_page);
4662         return ret;
4663 }
4664 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4665
4666 /**
4667  * nvme_kill_queues(): Ends all namespace queues
4668  * @ctrl: the dead controller that needs to end
4669  *
4670  * Call this function when the driver determines it is unable to get the
4671  * controller in a state capable of servicing IO.
4672  */
4673 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4674 {
4675         struct nvme_ns *ns;
4676
4677         down_read(&ctrl->namespaces_rwsem);
4678
4679         /* Forcibly unquiesce queues to avoid blocking dispatch */
4680         if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4681                 blk_mq_unquiesce_queue(ctrl->admin_q);
4682
4683         list_for_each_entry(ns, &ctrl->namespaces, list)
4684                 nvme_set_queue_dying(ns);
4685
4686         up_read(&ctrl->namespaces_rwsem);
4687 }
4688 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4689
4690 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4691 {
4692         struct nvme_ns *ns;
4693
4694         down_read(&ctrl->namespaces_rwsem);
4695         list_for_each_entry(ns, &ctrl->namespaces, list)
4696                 blk_mq_unfreeze_queue(ns->queue);
4697         up_read(&ctrl->namespaces_rwsem);
4698 }
4699 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4700
4701 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4702 {
4703         struct nvme_ns *ns;
4704
4705         down_read(&ctrl->namespaces_rwsem);
4706         list_for_each_entry(ns, &ctrl->namespaces, list) {
4707                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4708                 if (timeout <= 0)
4709                         break;
4710         }
4711         up_read(&ctrl->namespaces_rwsem);
4712         return timeout;
4713 }
4714 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4715
4716 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4717 {
4718         struct nvme_ns *ns;
4719
4720         down_read(&ctrl->namespaces_rwsem);
4721         list_for_each_entry(ns, &ctrl->namespaces, list)
4722                 blk_mq_freeze_queue_wait(ns->queue);
4723         up_read(&ctrl->namespaces_rwsem);
4724 }
4725 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4726
4727 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4728 {
4729         struct nvme_ns *ns;
4730
4731         down_read(&ctrl->namespaces_rwsem);
4732         list_for_each_entry(ns, &ctrl->namespaces, list)
4733                 blk_freeze_queue_start(ns->queue);
4734         up_read(&ctrl->namespaces_rwsem);
4735 }
4736 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4737
4738 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4739 {
4740         struct nvme_ns *ns;
4741
4742         down_read(&ctrl->namespaces_rwsem);
4743         list_for_each_entry(ns, &ctrl->namespaces, list)
4744                 blk_mq_quiesce_queue(ns->queue);
4745         up_read(&ctrl->namespaces_rwsem);
4746 }
4747 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4748
4749 void nvme_start_queues(struct nvme_ctrl *ctrl)
4750 {
4751         struct nvme_ns *ns;
4752
4753         down_read(&ctrl->namespaces_rwsem);
4754         list_for_each_entry(ns, &ctrl->namespaces, list)
4755                 blk_mq_unquiesce_queue(ns->queue);
4756         up_read(&ctrl->namespaces_rwsem);
4757 }
4758 EXPORT_SYMBOL_GPL(nvme_start_queues);
4759
4760 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
4761 {
4762         struct nvme_ns *ns;
4763
4764         down_read(&ctrl->namespaces_rwsem);
4765         list_for_each_entry(ns, &ctrl->namespaces, list)
4766                 blk_sync_queue(ns->queue);
4767         up_read(&ctrl->namespaces_rwsem);
4768 }
4769 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
4770
4771 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4772 {
4773         nvme_sync_io_queues(ctrl);
4774         if (ctrl->admin_q)
4775                 blk_sync_queue(ctrl->admin_q);
4776 }
4777 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4778
4779 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4780 {
4781         if (file->f_op != &nvme_dev_fops)
4782                 return NULL;
4783         return file->private_data;
4784 }
4785 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4786
4787 /*
4788  * Check we didn't inadvertently grow the command structure sizes:
4789  */
4790 static inline void _nvme_check_size(void)
4791 {
4792         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4793         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4794         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4795         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4796         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4797         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4798         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4799         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4800         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4801         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4802         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4803         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4804         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4805         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4806         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4807         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4808         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4809         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4810         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4811 }
4812
4813
4814 static int __init nvme_core_init(void)
4815 {
4816         int result = -ENOMEM;
4817
4818         _nvme_check_size();
4819
4820         nvme_wq = alloc_workqueue("nvme-wq",
4821                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4822         if (!nvme_wq)
4823                 goto out;
4824
4825         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4826                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4827         if (!nvme_reset_wq)
4828                 goto destroy_wq;
4829
4830         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4831                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4832         if (!nvme_delete_wq)
4833                 goto destroy_reset_wq;
4834
4835         result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4836         if (result < 0)
4837                 goto destroy_delete_wq;
4838
4839         nvme_class = class_create(THIS_MODULE, "nvme");
4840         if (IS_ERR(nvme_class)) {
4841                 result = PTR_ERR(nvme_class);
4842                 goto unregister_chrdev;
4843         }
4844         nvme_class->dev_uevent = nvme_class_uevent;
4845
4846         nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4847         if (IS_ERR(nvme_subsys_class)) {
4848                 result = PTR_ERR(nvme_subsys_class);
4849                 goto destroy_class;
4850         }
4851         return 0;
4852
4853 destroy_class:
4854         class_destroy(nvme_class);
4855 unregister_chrdev:
4856         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4857 destroy_delete_wq:
4858         destroy_workqueue(nvme_delete_wq);
4859 destroy_reset_wq:
4860         destroy_workqueue(nvme_reset_wq);
4861 destroy_wq:
4862         destroy_workqueue(nvme_wq);
4863 out:
4864         return result;
4865 }
4866
4867 static void __exit nvme_core_exit(void)
4868 {
4869         class_destroy(nvme_subsys_class);
4870         class_destroy(nvme_class);
4871         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4872         destroy_workqueue(nvme_delete_wq);
4873         destroy_workqueue(nvme_reset_wq);
4874         destroy_workqueue(nvme_wq);
4875         ida_destroy(&nvme_instance_ida);
4876 }
4877
4878 MODULE_LICENSE("GPL");
4879 MODULE_VERSION("1.0");
4880 module_init(nvme_core_init);
4881 module_exit(nvme_core_exit);