2 * CXL Flash Device Driver
4 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
5 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
7 * Copyright (C) 2015 IBM Corporation
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
15 #include <linux/delay.h>
16 #include <linux/file.h>
17 #include <linux/syscalls.h>
19 #include <asm/unaligned.h>
21 #include <scsi/scsi.h>
22 #include <scsi/scsi_host.h>
23 #include <scsi/scsi_cmnd.h>
24 #include <scsi/scsi_eh.h>
25 #include <uapi/scsi/cxlflash_ioctl.h>
30 #include "superpipe.h"
32 struct cxlflash_global global;
35 * marshal_rele_to_resize() - translate release to resize structure
36 * @rele: Source structure from which to translate/copy.
37 * @resize: Destination structure for the translate/copy.
39 static void marshal_rele_to_resize(struct dk_cxlflash_release *release,
40 struct dk_cxlflash_resize *resize)
42 resize->hdr = release->hdr;
43 resize->context_id = release->context_id;
44 resize->rsrc_handle = release->rsrc_handle;
48 * marshal_det_to_rele() - translate detach to release structure
49 * @detach: Destination structure for the translate/copy.
50 * @rele: Source structure from which to translate/copy.
52 static void marshal_det_to_rele(struct dk_cxlflash_detach *detach,
53 struct dk_cxlflash_release *release)
55 release->hdr = detach->hdr;
56 release->context_id = detach->context_id;
60 * marshal_udir_to_rele() - translate udirect to release structure
61 * @udirect: Source structure from which to translate/copy.
62 * @release: Destination structure for the translate/copy.
64 static void marshal_udir_to_rele(struct dk_cxlflash_udirect *udirect,
65 struct dk_cxlflash_release *release)
67 release->hdr = udirect->hdr;
68 release->context_id = udirect->context_id;
69 release->rsrc_handle = udirect->rsrc_handle;
73 * cxlflash_free_errpage() - frees resources associated with global error page
75 void cxlflash_free_errpage(void)
78 mutex_lock(&global.mutex);
79 if (global.err_page) {
80 __free_page(global.err_page);
81 global.err_page = NULL;
83 mutex_unlock(&global.mutex);
87 * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts
88 * @cfg: Internal structure associated with the host.
90 * When the host needs to go down, all users must be quiesced and their
91 * memory freed. This is accomplished by putting the contexts in error
92 * state which will notify the user and let them 'drive' the tear down.
93 * Meanwhile, this routine camps until all user contexts have been removed.
95 * Note that the main loop in this routine will always execute at least once
96 * to flush the reset_waitq.
98 void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg)
100 struct device *dev = &cfg->dev->dev;
103 cxlflash_mark_contexts_error(cfg);
106 for (i = 0; i < MAX_CONTEXT; i++)
107 if (cfg->ctx_tbl[i]) {
112 if (!found && list_empty(&cfg->ctx_err_recovery))
115 dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n",
117 wake_up_all(&cfg->reset_waitq);
124 * find_error_context() - locates a context by cookie on the error recovery list
125 * @cfg: Internal structure associated with the host.
126 * @rctxid: Desired context by id.
127 * @file: Desired context by file.
129 * Return: Found context on success, NULL on failure
131 static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid,
134 struct ctx_info *ctxi;
136 list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list)
137 if ((ctxi->ctxid == rctxid) || (ctxi->file == file))
144 * get_context() - obtains a validated and locked context reference
145 * @cfg: Internal structure associated with the host.
146 * @rctxid: Desired context (raw, un-decoded format).
147 * @arg: LUN information or file associated with request.
148 * @ctx_ctrl: Control information to 'steer' desired lookup.
150 * NOTE: despite the name pid, in linux, current->pid actually refers
151 * to the lightweight process id (tid) and can change if the process is
152 * multi threaded. The tgid remains constant for the process and only changes
153 * when the process of fork. For all intents and purposes, think of tgid
154 * as a pid in the traditional sense.
156 * Return: Validated context on success, NULL on failure
158 struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid,
159 void *arg, enum ctx_ctrl ctx_ctrl)
161 struct device *dev = &cfg->dev->dev;
162 struct ctx_info *ctxi = NULL;
163 struct lun_access *lun_access = NULL;
164 struct file *file = NULL;
165 struct llun_info *lli = arg;
166 u64 ctxid = DECODE_CTXID(rctxid);
168 pid_t pid = current->tgid, ctxpid = 0;
170 if (ctx_ctrl & CTX_CTRL_FILE) {
172 file = (struct file *)arg;
175 if (ctx_ctrl & CTX_CTRL_CLONE)
176 pid = current->parent->tgid;
178 if (likely(ctxid < MAX_CONTEXT)) {
180 mutex_lock(&cfg->ctx_tbl_list_mutex);
181 ctxi = cfg->ctx_tbl[ctxid];
183 if ((file && (ctxi->file != file)) ||
184 (!file && (ctxi->ctxid != rctxid)))
187 if ((ctx_ctrl & CTX_CTRL_ERR) ||
188 (!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK)))
189 ctxi = find_error_context(cfg, rctxid, file);
191 mutex_unlock(&cfg->ctx_tbl_list_mutex);
196 * Need to acquire ownership of the context while still
197 * under the table/list lock to serialize with a remove
198 * thread. Use the 'try' to avoid stalling the
199 * table/list lock for a single context.
201 * Note that the lock order is:
203 * cfg->ctx_tbl_list_mutex -> ctxi->mutex
205 * Therefore release ctx_tbl_list_mutex before retrying.
207 rc = mutex_trylock(&ctxi->mutex);
208 mutex_unlock(&cfg->ctx_tbl_list_mutex);
210 break; /* got the context's lock! */
217 if (likely(!(ctx_ctrl & CTX_CTRL_NOPID)))
222 list_for_each_entry(lun_access, &ctxi->luns, list)
223 if (lun_access->lli == lli)
230 dev_dbg(dev, "%s: rctxid=%016llx ctxinfo=%p ctxpid=%u pid=%u "
231 "ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid,
237 mutex_unlock(&ctxi->mutex);
243 * put_context() - release a context that was retrieved from get_context()
244 * @ctxi: Context to release.
246 * For now, releasing the context equates to unlocking it's mutex.
248 void put_context(struct ctx_info *ctxi)
250 mutex_unlock(&ctxi->mutex);
254 * afu_attach() - attach a context to the AFU
255 * @cfg: Internal structure associated with the host.
256 * @ctxi: Context to attach.
258 * Upon setting the context capabilities, they must be confirmed with
259 * a read back operation as the context might have been closed since
260 * the mailbox was unlocked. When this occurs, registration is failed.
262 * Return: 0 on success, -errno on failure
264 static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
266 struct device *dev = &cfg->dev->dev;
267 struct afu *afu = cfg->afu;
268 struct sisl_ctrl_map __iomem *ctrl_map = ctxi->ctrl_map;
270 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
273 /* Unlock cap and restrict user to read/write cmds in translated mode */
274 readq_be(&ctrl_map->mbox_r);
275 val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD);
276 writeq_be(val, &ctrl_map->ctx_cap);
277 val = readq_be(&ctrl_map->ctx_cap);
278 if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) {
279 dev_err(dev, "%s: ctx may be closed val=%016llx\n",
285 /* Set up MMIO registers pointing to the RHT */
286 writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
287 val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
288 writeq_be(val, &ctrl_map->rht_cnt_id);
290 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
295 * read_cap16() - issues a SCSI READ_CAP16 command
296 * @sdev: SCSI device associated with LUN.
297 * @lli: LUN destined for capacity request.
299 * The READ_CAP16 can take quite a while to complete. Should an EEH occur while
300 * in scsi_execute(), the EEH handler will attempt to recover. As part of the
301 * recovery, the handler drains all currently running ioctls, waiting until they
302 * have completed before proceeding with a reset. As this routine is used on the
303 * ioctl path, this can create a condition where the EEH handler becomes stuck,
304 * infinitely waiting for this ioctl thread. To avoid this behavior, temporarily
305 * unmark this thread as an ioctl thread by releasing the ioctl read semaphore.
306 * This will allow the EEH handler to proceed with a recovery while this thread
307 * is still running. Once the scsi_execute() returns, reacquire the ioctl read
308 * semaphore and check the adapter state in case it changed while inside of
309 * scsi_execute(). The state check will wait if the adapter is still being
310 * recovered or return a failure if the recovery failed. In the event that the
311 * adapter reset failed, simply return the failure as the ioctl would be unable
314 * Note that the above puts a requirement on this routine to only be called on
317 * Return: 0 on success, -errno on failure
319 static int read_cap16(struct scsi_device *sdev, struct llun_info *lli)
321 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
322 struct device *dev = &cfg->dev->dev;
323 struct glun_info *gli = lli->parent;
324 struct scsi_sense_hdr sshdr;
327 u8 *sense_buf = NULL;
331 u32 to = CMD_TIMEOUT * HZ;
334 cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);
335 scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);
336 sense_buf = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
337 if (unlikely(!cmd_buf || !scsi_cmd || !sense_buf)) {
342 scsi_cmd[0] = SERVICE_ACTION_IN_16; /* read cap(16) */
343 scsi_cmd[1] = SAI_READ_CAPACITY_16; /* service action */
344 put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]);
346 dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,
347 retry_cnt ? "re" : "", scsi_cmd[0]);
349 /* Drop the ioctl read semahpore across lengthy call */
350 up_read(&cfg->ioctl_rwsem);
351 result = scsi_execute(sdev, scsi_cmd, DMA_FROM_DEVICE, cmd_buf,
352 CMD_BUFSIZE, sense_buf, &sshdr, to, CMD_RETRIES,
354 down_read(&cfg->ioctl_rwsem);
355 rc = check_state(cfg);
357 dev_err(dev, "%s: Failed state result=%08x\n",
363 if (driver_byte(result) == DRIVER_SENSE) {
364 result &= ~(0xFF<<24); /* DRIVER_SENSE is not an error */
365 if (result & SAM_STAT_CHECK_CONDITION) {
366 switch (sshdr.sense_key) {
368 case RECOVERED_ERROR:
371 result &= ~SAM_STAT_CHECK_CONDITION;
375 case 0x29: /* Power on Reset or Device Reset */
377 case 0x2A: /* Device capacity changed */
378 case 0x3F: /* Report LUNs changed */
379 /* Retry the command once more */
380 if (retry_cnt++ < 1) {
395 dev_err(dev, "%s: command failed, result=%08x\n",
402 * Read cap was successful, grab values from the buffer;
403 * note that we don't need to worry about unaligned access
404 * as the buffer is allocated on an aligned boundary.
406 mutex_lock(&gli->mutex);
407 gli->max_lba = be64_to_cpu(*((__be64 *)&cmd_buf[0]));
408 gli->blk_len = be32_to_cpu(*((__be32 *)&cmd_buf[8]));
409 mutex_unlock(&gli->mutex);
416 dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n",
417 __func__, gli->max_lba, gli->blk_len, rc);
422 * get_rhte() - obtains validated resource handle table entry reference
423 * @ctxi: Context owning the resource handle.
424 * @rhndl: Resource handle associated with entry.
425 * @lli: LUN associated with request.
427 * Return: Validated RHTE on success, NULL on failure
429 struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl,
430 struct llun_info *lli)
432 struct cxlflash_cfg *cfg = ctxi->cfg;
433 struct device *dev = &cfg->dev->dev;
434 struct sisl_rht_entry *rhte = NULL;
436 if (unlikely(!ctxi->rht_start)) {
437 dev_dbg(dev, "%s: Context does not have allocated RHT\n",
442 if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) {
443 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
448 if (unlikely(ctxi->rht_lun[rhndl] != lli)) {
449 dev_dbg(dev, "%s: Bad resource handle LUN rhndl=%d\n",
454 rhte = &ctxi->rht_start[rhndl];
455 if (unlikely(rhte->nmask == 0)) {
456 dev_dbg(dev, "%s: Unopened resource handle rhndl=%d\n",
467 * rhte_checkout() - obtains free/empty resource handle table entry
468 * @ctxi: Context owning the resource handle.
469 * @lli: LUN associated with request.
471 * Return: Free RHTE on success, NULL on failure
473 struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi,
474 struct llun_info *lli)
476 struct cxlflash_cfg *cfg = ctxi->cfg;
477 struct device *dev = &cfg->dev->dev;
478 struct sisl_rht_entry *rhte = NULL;
481 /* Find a free RHT entry */
482 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
483 if (ctxi->rht_start[i].nmask == 0) {
484 rhte = &ctxi->rht_start[i];
490 ctxi->rht_lun[i] = lli;
492 dev_dbg(dev, "%s: returning rhte=%p index=%d\n", __func__, rhte, i);
497 * rhte_checkin() - releases a resource handle table entry
498 * @ctxi: Context owning the resource handle.
499 * @rhte: RHTE to release.
501 void rhte_checkin(struct ctx_info *ctxi,
502 struct sisl_rht_entry *rhte)
504 u32 rsrc_handle = rhte - ctxi->rht_start;
509 ctxi->rht_lun[rsrc_handle] = NULL;
510 ctxi->rht_needs_ws[rsrc_handle] = false;
514 * rhte_format1() - populates a RHTE for format 1
515 * @rhte: RHTE to populate.
516 * @lun_id: LUN ID of LUN associated with RHTE.
517 * @perm: Desired permissions for RHTE.
518 * @port_sel: Port selection mask
520 static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm,
524 * Populate the Format 1 RHT entry for direct access (physical
525 * LUN) using the synchronization sequence defined in the
526 * SISLite specification.
528 struct sisl_rht_entry_f1 dummy = { 0 };
529 struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
531 memset(rhte_f1, 0, sizeof(*rhte_f1));
532 rhte_f1->fp = SISL_RHT_FP(1U, 0);
533 dma_wmb(); /* Make setting of format bit visible */
535 rhte_f1->lun_id = lun_id;
536 dma_wmb(); /* Make setting of LUN id visible */
539 * Use a dummy RHT Format 1 entry to build the second dword
540 * of the entry that must be populated in a single write when
541 * enabled (valid bit set to TRUE).
544 dummy.fp = SISL_RHT_FP(1U, perm);
545 dummy.port_sel = port_sel;
546 rhte_f1->dw = dummy.dw;
548 dma_wmb(); /* Make remaining RHT entry fields visible */
552 * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode
553 * @gli: LUN to attach.
554 * @mode: Desired mode of the LUN.
555 * @locked: Mutex status on current thread.
557 * Return: 0 on success, -errno on failure
559 int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked)
564 mutex_lock(&gli->mutex);
566 if (gli->mode == MODE_NONE)
568 else if (gli->mode != mode) {
569 pr_debug("%s: gli_mode=%d requested_mode=%d\n",
570 __func__, gli->mode, mode);
576 WARN_ON(gli->users <= 0);
578 pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",
579 __func__, rc, gli->mode, gli->users);
581 mutex_unlock(&gli->mutex);
586 * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode
587 * @gli: LUN to detach.
589 * When resetting the mode, terminate block allocation resources as they
590 * are no longer required (service is safe to call even when block allocation
591 * resources were not present - such as when transitioning from physical mode).
592 * These resources will be reallocated when needed (subsequent transition to
595 void cxlflash_lun_detach(struct glun_info *gli)
597 mutex_lock(&gli->mutex);
598 WARN_ON(gli->mode == MODE_NONE);
599 if (--gli->users == 0) {
600 gli->mode = MODE_NONE;
601 cxlflash_ba_terminate(&gli->blka.ba_lun);
603 pr_debug("%s: gli->users=%u\n", __func__, gli->users);
604 WARN_ON(gli->users < 0);
605 mutex_unlock(&gli->mutex);
609 * _cxlflash_disk_release() - releases the specified resource entry
610 * @sdev: SCSI device associated with LUN.
611 * @ctxi: Context owning resources.
612 * @release: Release ioctl data structure.
614 * For LUNs in virtual mode, the virtual LUN associated with the specified
615 * resource handle is resized to 0 prior to releasing the RHTE. Note that the
616 * AFU sync should _not_ be performed when the context is sitting on the error
617 * recovery list. A context on the error recovery list is not known to the AFU
618 * due to reset. When the context is recovered, it will be reattached and made
619 * known again to the AFU.
621 * Return: 0 on success, -errno on failure
623 int _cxlflash_disk_release(struct scsi_device *sdev,
624 struct ctx_info *ctxi,
625 struct dk_cxlflash_release *release)
627 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
628 struct device *dev = &cfg->dev->dev;
629 struct llun_info *lli = sdev->hostdata;
630 struct glun_info *gli = lli->parent;
631 struct afu *afu = cfg->afu;
632 bool put_ctx = false;
634 struct dk_cxlflash_resize size;
635 res_hndl_t rhndl = release->rsrc_handle;
639 u64 ctxid = DECODE_CTXID(release->context_id),
640 rctxid = release->context_id;
642 struct sisl_rht_entry *rhte;
643 struct sisl_rht_entry_f1 *rhte_f1;
645 dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu gli->mode=%u gli->users=%u\n",
646 __func__, ctxid, release->rsrc_handle, gli->mode, gli->users);
649 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
650 if (unlikely(!ctxi)) {
651 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
660 rhte = get_rhte(ctxi, rhndl, lli);
661 if (unlikely(!rhte)) {
662 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
669 * Resize to 0 for virtual LUNS by setting the size
670 * to 0. This will clear LXT_START and LXT_CNT fields
671 * in the RHT entry and properly sync with the AFU.
673 * Afterwards we clear the remaining fields.
677 marshal_rele_to_resize(release, &size);
679 rc = _cxlflash_vlun_resize(sdev, ctxi, &size);
681 dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc);
688 * Clear the Format 1 RHT entry for direct access
689 * (physical LUN) using the synchronization sequence
690 * defined in the SISLite specification.
692 rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
695 dma_wmb(); /* Make revocation of RHT entry visible */
698 dma_wmb(); /* Make clearing of LUN id visible */
701 dma_wmb(); /* Make RHT entry bottom-half clearing visible */
703 if (!ctxi->err_recovery_active) {
704 rcr = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
706 dev_dbg(dev, "%s: AFU sync failed rc=%d\n",
711 WARN(1, "Unsupported LUN mode!");
715 rhte_checkin(ctxi, rhte);
716 cxlflash_lun_detach(gli);
721 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
725 int cxlflash_disk_release(struct scsi_device *sdev,
726 struct dk_cxlflash_release *release)
728 return _cxlflash_disk_release(sdev, NULL, release);
732 * destroy_context() - releases a context
733 * @cfg: Internal structure associated with the host.
734 * @ctxi: Context to release.
736 * This routine is safe to be called with a a non-initialized context.
737 * Also note that the routine conditionally checks for the existence
738 * of the context control map before clearing the RHT registers and
739 * context capabilities because it is possible to destroy a context
740 * while the context is in the error state (previous mapping was
741 * removed [so there is no need to worry about clearing] and context
742 * is waiting for a new mapping).
744 static void destroy_context(struct cxlflash_cfg *cfg,
745 struct ctx_info *ctxi)
747 struct afu *afu = cfg->afu;
749 if (ctxi->initialized) {
750 WARN_ON(!list_empty(&ctxi->luns));
752 /* Clear RHT registers and drop all capabilities for context */
753 if (afu->afu_map && ctxi->ctrl_map) {
754 writeq_be(0, &ctxi->ctrl_map->rht_start);
755 writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);
756 writeq_be(0, &ctxi->ctrl_map->ctx_cap);
760 /* Free memory associated with context */
761 free_page((ulong)ctxi->rht_start);
762 kfree(ctxi->rht_needs_ws);
763 kfree(ctxi->rht_lun);
768 * create_context() - allocates and initializes a context
769 * @cfg: Internal structure associated with the host.
771 * Return: Allocated context on success, NULL on failure
773 static struct ctx_info *create_context(struct cxlflash_cfg *cfg)
775 struct device *dev = &cfg->dev->dev;
776 struct ctx_info *ctxi = NULL;
777 struct llun_info **lli = NULL;
779 struct sisl_rht_entry *rhte;
781 ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL);
782 lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL);
783 ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL);
784 if (unlikely(!ctxi || !lli || !ws)) {
785 dev_err(dev, "%s: Unable to allocate context\n", __func__);
789 rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL);
790 if (unlikely(!rhte)) {
791 dev_err(dev, "%s: Unable to allocate RHT\n", __func__);
796 ctxi->rht_needs_ws = ws;
797 ctxi->rht_start = rhte;
810 * init_context() - initializes a previously allocated context
811 * @ctxi: Previously allocated context
812 * @cfg: Internal structure associated with the host.
813 * @ctx: Previously obtained CXL context reference.
814 * @ctxid: Previously obtained process element associated with CXL context.
815 * @file: Previously obtained file associated with CXL context.
816 * @perms: User-specified permissions.
818 static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,
819 struct cxl_context *ctx, int ctxid, struct file *file,
822 struct afu *afu = cfg->afu;
824 ctxi->rht_perms = perms;
825 ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
826 ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
827 ctxi->pid = current->tgid; /* tgid = pid */
831 ctxi->initialized = true;
832 mutex_init(&ctxi->mutex);
833 kref_init(&ctxi->kref);
834 INIT_LIST_HEAD(&ctxi->luns);
835 INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */
839 * remove_context() - context kref release handler
840 * @kref: Kernel reference associated with context to be removed.
842 * When a context no longer has any references it can safely be removed
843 * from global access and destroyed. Note that it is assumed the thread
844 * relinquishing access to the context holds its mutex.
846 static void remove_context(struct kref *kref)
848 struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);
849 struct cxlflash_cfg *cfg = ctxi->cfg;
850 u64 ctxid = DECODE_CTXID(ctxi->ctxid);
852 /* Remove context from table/error list */
853 WARN_ON(!mutex_is_locked(&ctxi->mutex));
854 ctxi->unavail = true;
855 mutex_unlock(&ctxi->mutex);
856 mutex_lock(&cfg->ctx_tbl_list_mutex);
857 mutex_lock(&ctxi->mutex);
859 if (!list_empty(&ctxi->list))
860 list_del(&ctxi->list);
861 cfg->ctx_tbl[ctxid] = NULL;
862 mutex_unlock(&cfg->ctx_tbl_list_mutex);
863 mutex_unlock(&ctxi->mutex);
865 /* Context now completely uncoupled/unreachable */
866 destroy_context(cfg, ctxi);
870 * _cxlflash_disk_detach() - detaches a LUN from a context
871 * @sdev: SCSI device associated with LUN.
872 * @ctxi: Context owning resources.
873 * @detach: Detach ioctl data structure.
875 * As part of the detach, all per-context resources associated with the LUN
876 * are cleaned up. When detaching the last LUN for a context, the context
877 * itself is cleaned up and released.
879 * Return: 0 on success, -errno on failure
881 static int _cxlflash_disk_detach(struct scsi_device *sdev,
882 struct ctx_info *ctxi,
883 struct dk_cxlflash_detach *detach)
885 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
886 struct device *dev = &cfg->dev->dev;
887 struct llun_info *lli = sdev->hostdata;
888 struct lun_access *lun_access, *t;
889 struct dk_cxlflash_release rel;
890 bool put_ctx = false;
894 u64 ctxid = DECODE_CTXID(detach->context_id),
895 rctxid = detach->context_id;
897 dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid);
900 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
901 if (unlikely(!ctxi)) {
902 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
911 /* Cleanup outstanding resources tied to this LUN */
913 marshal_det_to_rele(detach, &rel);
914 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {
915 if (ctxi->rht_lun[i] == lli) {
917 _cxlflash_disk_release(sdev, ctxi, &rel);
920 /* No need to loop further if we're done */
921 if (ctxi->rht_out == 0)
926 /* Take our LUN out of context, free the node */
927 list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
928 if (lun_access->lli == lli) {
929 list_del(&lun_access->list);
936 * Release the context reference and the sdev reference that
937 * bound this LUN to the context.
939 if (kref_put(&ctxi->kref, remove_context))
941 scsi_device_put(sdev);
945 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
949 static int cxlflash_disk_detach(struct scsi_device *sdev,
950 struct dk_cxlflash_detach *detach)
952 return _cxlflash_disk_detach(sdev, NULL, detach);
956 * cxlflash_cxl_release() - release handler for adapter file descriptor
957 * @inode: File-system inode associated with fd.
958 * @file: File installed with adapter file descriptor.
960 * This routine is the release handler for the fops registered with
961 * the CXL services on an initial attach for a context. It is called
962 * when a close (explicity by the user or as part of a process tear
963 * down) is performed on the adapter file descriptor returned to the
964 * user. The user should be aware that explicitly performing a close
965 * considered catastrophic and subsequent usage of the superpipe API
966 * with previously saved off tokens will fail.
968 * This routine derives the context reference and calls detach for
969 * each LUN associated with the context.The final detach operation
970 * causes the context itself to be freed. With exception to when the
971 * CXL process element (context id) lookup fails (a case that should
972 * theoretically never occur), every call into this routine results
973 * in a complete freeing of a context.
975 * Return: 0 on success
977 static int cxlflash_cxl_release(struct inode *inode, struct file *file)
979 struct cxl_context *ctx = cxl_fops_get_context(file);
980 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
982 struct device *dev = &cfg->dev->dev;
983 struct ctx_info *ctxi = NULL;
984 struct dk_cxlflash_detach detach = { { 0 }, 0 };
985 struct lun_access *lun_access, *t;
986 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
989 ctxid = cxl_process_element(ctx);
990 if (unlikely(ctxid < 0)) {
991 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
992 __func__, ctx, ctxid);
996 ctxi = get_context(cfg, ctxid, file, ctrl);
997 if (unlikely(!ctxi)) {
998 ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE);
1000 dev_dbg(dev, "%s: ctxid=%d already free\n",
1005 dev_dbg(dev, "%s: Another process owns ctxid=%d\n",
1011 dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
1013 detach.context_id = ctxi->ctxid;
1014 list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
1015 _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
1017 cxl_fd_release(inode, file);
1019 dev_dbg(dev, "%s: returning\n", __func__);
1024 * unmap_context() - clears a previously established mapping
1025 * @ctxi: Context owning the mapping.
1027 * This routine is used to switch between the error notification page
1028 * (dummy page of all 1's) and the real mapping (established by the CXL
1031 static void unmap_context(struct ctx_info *ctxi)
1033 unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1);
1037 * get_err_page() - obtains and allocates the error notification page
1038 * @cfg: Internal structure associated with the host.
1040 * Return: error notification page on success, NULL on failure
1042 static struct page *get_err_page(struct cxlflash_cfg *cfg)
1044 struct page *err_page = global.err_page;
1045 struct device *dev = &cfg->dev->dev;
1047 if (unlikely(!err_page)) {
1048 err_page = alloc_page(GFP_KERNEL);
1049 if (unlikely(!err_page)) {
1050 dev_err(dev, "%s: Unable to allocate err_page\n",
1055 memset(page_address(err_page), -1, PAGE_SIZE);
1057 /* Serialize update w/ other threads to avoid a leak */
1058 mutex_lock(&global.mutex);
1059 if (likely(!global.err_page))
1060 global.err_page = err_page;
1062 __free_page(err_page);
1063 err_page = global.err_page;
1065 mutex_unlock(&global.mutex);
1069 dev_dbg(dev, "%s: returning err_page=%p\n", __func__, err_page);
1074 * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor
1075 * @vmf: VM fault associated with current fault.
1077 * To support error notification via MMIO, faults are 'caught' by this routine
1078 * that was inserted before passing back the adapter file descriptor on attach.
1079 * When a fault occurs, this routine evaluates if error recovery is active and
1080 * if so, installs the error page to 'notify' the user about the error state.
1081 * During normal operation, the fault is simply handled by the original fault
1082 * handler that was installed by CXL services as part of initializing the
1083 * adapter file descriptor. The VMA's page protection bits are toggled to
1084 * indicate cached/not-cached depending on the memory backing the fault.
1086 * Return: 0 on success, VM_FAULT_SIGBUS on failure
1088 static int cxlflash_mmap_fault(struct vm_fault *vmf)
1090 struct vm_area_struct *vma = vmf->vma;
1091 struct file *file = vma->vm_file;
1092 struct cxl_context *ctx = cxl_fops_get_context(file);
1093 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1095 struct device *dev = &cfg->dev->dev;
1096 struct ctx_info *ctxi = NULL;
1097 struct page *err_page = NULL;
1098 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1102 ctxid = cxl_process_element(ctx);
1103 if (unlikely(ctxid < 0)) {
1104 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1105 __func__, ctx, ctxid);
1109 ctxi = get_context(cfg, ctxid, file, ctrl);
1110 if (unlikely(!ctxi)) {
1111 dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1115 dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);
1117 if (likely(!ctxi->err_recovery_active)) {
1118 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1119 rc = ctxi->cxl_mmap_vmops->fault(vmf);
1121 dev_dbg(dev, "%s: err recovery active, use err_page\n",
1124 err_page = get_err_page(cfg);
1125 if (unlikely(!err_page)) {
1126 dev_err(dev, "%s: Could not get err_page\n", __func__);
1127 rc = VM_FAULT_RETRY;
1132 vmf->page = err_page;
1133 vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
1139 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
1143 rc = VM_FAULT_SIGBUS;
1148 * Local MMAP vmops to 'catch' faults
1150 static const struct vm_operations_struct cxlflash_mmap_vmops = {
1151 .fault = cxlflash_mmap_fault,
1155 * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor
1156 * @file: File installed with adapter file descriptor.
1157 * @vma: VM area associated with mapping.
1159 * Installs local mmap vmops to 'catch' faults for error notification support.
1161 * Return: 0 on success, -errno on failure
1163 static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)
1165 struct cxl_context *ctx = cxl_fops_get_context(file);
1166 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1168 struct device *dev = &cfg->dev->dev;
1169 struct ctx_info *ctxi = NULL;
1170 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1174 ctxid = cxl_process_element(ctx);
1175 if (unlikely(ctxid < 0)) {
1176 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1177 __func__, ctx, ctxid);
1182 ctxi = get_context(cfg, ctxid, file, ctrl);
1183 if (unlikely(!ctxi)) {
1184 dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1189 dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);
1191 rc = cxl_fd_mmap(file, vma);
1193 /* Insert ourself in the mmap fault handler path */
1194 ctxi->cxl_mmap_vmops = vma->vm_ops;
1195 vma->vm_ops = &cxlflash_mmap_vmops;
1204 const struct file_operations cxlflash_cxl_fops = {
1205 .owner = THIS_MODULE,
1206 .mmap = cxlflash_cxl_mmap,
1207 .release = cxlflash_cxl_release,
1211 * cxlflash_mark_contexts_error() - move contexts to error state and list
1212 * @cfg: Internal structure associated with the host.
1214 * A context is only moved over to the error list when there are no outstanding
1215 * references to it. This ensures that a running operation has completed.
1217 * Return: 0 on success, -errno on failure
1219 int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg)
1222 struct ctx_info *ctxi = NULL;
1224 mutex_lock(&cfg->ctx_tbl_list_mutex);
1226 for (i = 0; i < MAX_CONTEXT; i++) {
1227 ctxi = cfg->ctx_tbl[i];
1229 mutex_lock(&ctxi->mutex);
1230 cfg->ctx_tbl[i] = NULL;
1231 list_add(&ctxi->list, &cfg->ctx_err_recovery);
1232 ctxi->err_recovery_active = true;
1233 ctxi->ctrl_map = NULL;
1234 unmap_context(ctxi);
1235 mutex_unlock(&ctxi->mutex);
1239 mutex_unlock(&cfg->ctx_tbl_list_mutex);
1246 static const struct file_operations null_fops = {
1247 .owner = THIS_MODULE,
1251 * check_state() - checks and responds to the current adapter state
1252 * @cfg: Internal structure associated with the host.
1254 * This routine can block and should only be used on process context.
1255 * It assumes that the caller is an ioctl thread and holding the ioctl
1256 * read semaphore. This is temporarily let up across the wait to allow
1257 * for draining actively running ioctls. Also note that when waking up
1258 * from waiting in reset, the state is unknown and must be checked again
1259 * before proceeding.
1261 * Return: 0 on success, -errno on failure
1263 int check_state(struct cxlflash_cfg *cfg)
1265 struct device *dev = &cfg->dev->dev;
1269 switch (cfg->state) {
1271 dev_dbg(dev, "%s: Reset state, going to wait...\n", __func__);
1272 up_read(&cfg->ioctl_rwsem);
1273 rc = wait_event_interruptible(cfg->reset_waitq,
1274 cfg->state != STATE_RESET);
1275 down_read(&cfg->ioctl_rwsem);
1279 case STATE_FAILTERM:
1280 dev_dbg(dev, "%s: Failed/Terminating\n", __func__);
1291 * cxlflash_disk_attach() - attach a LUN to a context
1292 * @sdev: SCSI device associated with LUN.
1293 * @attach: Attach ioctl data structure.
1295 * Creates a context and attaches LUN to it. A LUN can only be attached
1296 * one time to a context (subsequent attaches for the same context/LUN pair
1297 * are not supported). Additional LUNs can be attached to a context by
1298 * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.
1300 * Return: 0 on success, -errno on failure
1302 static int cxlflash_disk_attach(struct scsi_device *sdev,
1303 struct dk_cxlflash_attach *attach)
1305 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1306 struct device *dev = &cfg->dev->dev;
1307 struct afu *afu = cfg->afu;
1308 struct llun_info *lli = sdev->hostdata;
1309 struct glun_info *gli = lli->parent;
1310 struct cxl_ioctl_start_work *work;
1311 struct ctx_info *ctxi = NULL;
1312 struct lun_access *lun_access = NULL;
1318 struct file *file = NULL;
1320 struct cxl_context *ctx = NULL;
1324 if (attach->num_interrupts > 4) {
1325 dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n",
1326 __func__, attach->num_interrupts);
1331 if (gli->max_lba == 0) {
1332 dev_dbg(dev, "%s: No capacity info for LUN=%016llx\n",
1333 __func__, lli->lun_id[sdev->channel]);
1334 rc = read_cap16(sdev, lli);
1336 dev_err(dev, "%s: Invalid device rc=%d\n",
1341 dev_dbg(dev, "%s: LBA = %016llx\n", __func__, gli->max_lba);
1342 dev_dbg(dev, "%s: BLK_LEN = %08x\n", __func__, gli->blk_len);
1345 if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) {
1346 rctxid = attach->context_id;
1347 ctxi = get_context(cfg, rctxid, NULL, 0);
1349 dev_dbg(dev, "%s: Bad context rctxid=%016llx\n",
1355 list_for_each_entry(lun_access, &ctxi->luns, list)
1356 if (lun_access->lli == lli) {
1357 dev_dbg(dev, "%s: Already attached\n",
1364 rc = scsi_device_get(sdev);
1366 dev_err(dev, "%s: Unable to get sdev reference\n", __func__);
1370 lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL);
1371 if (unlikely(!lun_access)) {
1372 dev_err(dev, "%s: Unable to allocate lun_access\n", __func__);
1377 lun_access->lli = lli;
1378 lun_access->sdev = sdev;
1380 /* Non-NULL context indicates reuse (another context reference) */
1382 dev_dbg(dev, "%s: Reusing context for LUN rctxid=%016llx\n",
1384 kref_get(&ctxi->kref);
1385 list_add(&lun_access->list, &ctxi->luns);
1389 ctxi = create_context(cfg);
1390 if (unlikely(!ctxi)) {
1391 dev_err(dev, "%s: Failed to create context ctxid=%d\n",
1397 ctx = cxl_dev_context_init(cfg->dev);
1398 if (IS_ERR_OR_NULL(ctx)) {
1399 dev_err(dev, "%s: Could not initialize context %p\n",
1406 work->num_interrupts = attach->num_interrupts;
1407 work->flags = CXL_START_WORK_NUM_IRQS;
1409 rc = cxl_start_work(ctx, work);
1411 dev_dbg(dev, "%s: Could not start context rc=%d\n",
1416 ctxid = cxl_process_element(ctx);
1417 if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1418 dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1423 file = cxl_get_fd(ctx, &cfg->cxl_fops, &fd);
1424 if (unlikely(fd < 0)) {
1426 dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1430 /* Translate read/write O_* flags from fcntl.h to AFU permission bits */
1431 perms = SISL_RHT_PERM(attach->hdr.flags + 1);
1433 /* Context mutex is locked upon return */
1434 init_context(ctxi, cfg, ctx, ctxid, file, perms);
1436 rc = afu_attach(cfg, ctxi);
1438 dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1443 * No error paths after this point. Once the fd is installed it's
1444 * visible to user space and can't be undone safely on this thread.
1445 * There is no need to worry about a deadlock here because no one
1446 * knows about us yet; we can be the only one holding our mutex.
1448 list_add(&lun_access->list, &ctxi->luns);
1449 mutex_lock(&cfg->ctx_tbl_list_mutex);
1450 mutex_lock(&ctxi->mutex);
1451 cfg->ctx_tbl[ctxid] = ctxi;
1452 mutex_unlock(&cfg->ctx_tbl_list_mutex);
1453 fd_install(fd, file);
1457 flags |= DK_CXLFLASH_APP_CLOSE_ADAP_FD;
1458 if (afu_is_sq_cmd_mode(afu))
1459 flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1461 attach->hdr.return_flags = flags;
1462 attach->context_id = ctxi->ctxid;
1463 attach->block_size = gli->blk_len;
1464 attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1465 attach->last_lba = gli->max_lba;
1466 attach->max_xfer = sdev->host->max_sectors * MAX_SECTOR_UNIT;
1467 attach->max_xfer /= gli->blk_len;
1470 attach->adap_fd = fd;
1475 dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",
1476 __func__, ctxid, fd, attach->block_size, rc, attach->last_lba);
1480 /* Cleanup CXL context; okay to 'stop' even if it was not started */
1481 if (!IS_ERR_OR_NULL(ctx)) {
1482 cxl_stop_context(ctx);
1483 cxl_release_context(ctx);
1488 * Here, we're overriding the fops with a dummy all-NULL fops because
1489 * fput() calls the release fop, which will cause us to mistakenly
1490 * call into the CXL code. Rather than try to add yet more complexity
1491 * to that routine (cxlflash_cxl_release) we should try to fix the
1495 file->f_op = &null_fops;
1502 /* Cleanup our context */
1504 destroy_context(cfg, ctxi);
1509 scsi_device_put(sdev);
1514 * recover_context() - recovers a context in error
1515 * @cfg: Internal structure associated with the host.
1516 * @ctxi: Context to release.
1517 * @adap_fd: Adapter file descriptor associated with new/recovered context.
1519 * Restablishes the state for a context-in-error.
1521 * Return: 0 on success, -errno on failure
1523 static int recover_context(struct cxlflash_cfg *cfg,
1524 struct ctx_info *ctxi,
1527 struct device *dev = &cfg->dev->dev;
1532 struct cxl_context *ctx;
1533 struct afu *afu = cfg->afu;
1535 ctx = cxl_dev_context_init(cfg->dev);
1536 if (IS_ERR_OR_NULL(ctx)) {
1537 dev_err(dev, "%s: Could not initialize context %p\n",
1543 rc = cxl_start_work(ctx, &ctxi->work);
1545 dev_dbg(dev, "%s: Could not start context rc=%d\n",
1550 ctxid = cxl_process_element(ctx);
1551 if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1552 dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1557 file = cxl_get_fd(ctx, &cfg->cxl_fops, &fd);
1558 if (unlikely(fd < 0)) {
1560 dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1564 /* Update with new MMIO area based on updated context id */
1565 ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
1567 rc = afu_attach(cfg, ctxi);
1569 dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1574 * No error paths after this point. Once the fd is installed it's
1575 * visible to user space and can't be undone safely on this thread.
1577 ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
1582 * Put context back in table (note the reinit of the context list);
1583 * we must first drop the context's mutex and then acquire it in
1584 * order with the table/list mutex to avoid a deadlock - safe to do
1585 * here because no one can find us at this moment in time.
1587 mutex_unlock(&ctxi->mutex);
1588 mutex_lock(&cfg->ctx_tbl_list_mutex);
1589 mutex_lock(&ctxi->mutex);
1590 list_del_init(&ctxi->list);
1591 cfg->ctx_tbl[ctxid] = ctxi;
1592 mutex_unlock(&cfg->ctx_tbl_list_mutex);
1593 fd_install(fd, file);
1596 dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",
1597 __func__, ctxid, fd, rc);
1604 cxl_stop_context(ctx);
1606 cxl_release_context(ctx);
1611 * cxlflash_afu_recover() - initiates AFU recovery
1612 * @sdev: SCSI device associated with LUN.
1613 * @recover: Recover ioctl data structure.
1615 * Only a single recovery is allowed at a time to avoid exhausting CXL
1616 * resources (leading to recovery failure) in the event that we're up
1617 * against the maximum number of contexts limit. For similar reasons,
1618 * a context recovery is retried if there are multiple recoveries taking
1619 * place at the same time and the failure was due to CXL services being
1620 * unable to keep up.
1622 * As this routine is called on ioctl context, it holds the ioctl r/w
1623 * semaphore that is used to drain ioctls in recovery scenarios. The
1624 * implementation to achieve the pacing described above (a local mutex)
1625 * requires that the ioctl r/w semaphore be dropped and reacquired to
1626 * avoid a 3-way deadlock when multiple process recoveries operate in
1629 * Because a user can detect an error condition before the kernel, it is
1630 * quite possible for this routine to act as the kernel's EEH detection
1631 * source (MMIO read of mbox_r). Because of this, there is a window of
1632 * time where an EEH might have been detected but not yet 'serviced'
1633 * (callback invoked, causing the device to enter reset state). To avoid
1634 * looping in this routine during that window, a 1 second sleep is in place
1635 * between the time the MMIO failure is detected and the time a wait on the
1636 * reset wait queue is attempted via check_state().
1638 * Return: 0 on success, -errno on failure
1640 static int cxlflash_afu_recover(struct scsi_device *sdev,
1641 struct dk_cxlflash_recover_afu *recover)
1643 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1644 struct device *dev = &cfg->dev->dev;
1645 struct llun_info *lli = sdev->hostdata;
1646 struct afu *afu = cfg->afu;
1647 struct ctx_info *ctxi = NULL;
1648 struct mutex *mutex = &cfg->ctx_recovery_mutex;
1649 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
1651 u64 ctxid = DECODE_CTXID(recover->context_id),
1652 rctxid = recover->context_id;
1655 int lretry = 20; /* up to 2 seconds */
1656 int new_adap_fd = -1;
1659 atomic_inc(&cfg->recovery_threads);
1660 up_read(&cfg->ioctl_rwsem);
1661 rc = mutex_lock_interruptible(mutex);
1662 down_read(&cfg->ioctl_rwsem);
1668 rc = check_state(cfg);
1670 dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);
1675 dev_dbg(dev, "%s: reason=%016llx rctxid=%016llx\n",
1676 __func__, recover->reason, rctxid);
1679 /* Ensure that this process is attached to the context */
1680 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
1681 if (unlikely(!ctxi)) {
1682 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1687 if (ctxi->err_recovery_active) {
1689 rc = recover_context(cfg, ctxi, &new_adap_fd);
1691 dev_err(dev, "%s: Recovery failed ctxid=%llu rc=%d\n",
1692 __func__, ctxid, rc);
1693 if ((rc == -ENODEV) &&
1694 ((atomic_read(&cfg->recovery_threads) > 1) ||
1696 dev_dbg(dev, "%s: Going to try again\n",
1698 mutex_unlock(mutex);
1700 rc = mutex_lock_interruptible(mutex);
1711 ctxi->err_recovery_active = false;
1713 flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |
1714 DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;
1715 if (afu_is_sq_cmd_mode(afu))
1716 flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1718 recover->hdr.return_flags = flags;
1719 recover->context_id = ctxi->ctxid;
1720 recover->adap_fd = new_adap_fd;
1721 recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1725 /* Test if in error state */
1726 reg = readq_be(&hwq->ctrl_map->mbox_r);
1728 dev_dbg(dev, "%s: MMIO fail, wait for recovery.\n", __func__);
1731 * Before checking the state, put back the context obtained with
1732 * get_context() as it is no longer needed and sleep for a short
1733 * period of time (see prolog notes).
1738 rc = check_state(cfg);
1744 dev_dbg(dev, "%s: MMIO working, no recovery required\n", __func__);
1749 mutex_unlock(mutex);
1750 atomic_dec_if_positive(&cfg->recovery_threads);
1755 * process_sense() - evaluates and processes sense data
1756 * @sdev: SCSI device associated with LUN.
1757 * @verify: Verify ioctl data structure.
1759 * Return: 0 on success, -errno on failure
1761 static int process_sense(struct scsi_device *sdev,
1762 struct dk_cxlflash_verify *verify)
1764 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1765 struct device *dev = &cfg->dev->dev;
1766 struct llun_info *lli = sdev->hostdata;
1767 struct glun_info *gli = lli->parent;
1768 u64 prev_lba = gli->max_lba;
1769 struct scsi_sense_hdr sshdr = { 0 };
1772 rc = scsi_normalize_sense((const u8 *)&verify->sense_data,
1773 DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr);
1775 dev_err(dev, "%s: Failed to normalize sense data\n", __func__);
1780 switch (sshdr.sense_key) {
1782 case RECOVERED_ERROR:
1786 case UNIT_ATTENTION:
1787 switch (sshdr.asc) {
1788 case 0x29: /* Power on Reset or Device Reset */
1790 case 0x2A: /* Device settings/capacity changed */
1791 rc = read_cap16(sdev, lli);
1796 if (prev_lba != gli->max_lba)
1797 dev_dbg(dev, "%s: Capacity changed old=%lld "
1798 "new=%lld\n", __func__, prev_lba,
1801 case 0x3F: /* Report LUNs changed, Rescan. */
1802 scsi_scan_host(cfg->host);
1814 dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__,
1815 sshdr.sense_key, sshdr.asc, sshdr.ascq, rc);
1820 * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes
1821 * @sdev: SCSI device associated with LUN.
1822 * @verify: Verify ioctl data structure.
1824 * Return: 0 on success, -errno on failure
1826 static int cxlflash_disk_verify(struct scsi_device *sdev,
1827 struct dk_cxlflash_verify *verify)
1830 struct ctx_info *ctxi = NULL;
1831 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1832 struct device *dev = &cfg->dev->dev;
1833 struct llun_info *lli = sdev->hostdata;
1834 struct glun_info *gli = lli->parent;
1835 struct sisl_rht_entry *rhte = NULL;
1836 res_hndl_t rhndl = verify->rsrc_handle;
1837 u64 ctxid = DECODE_CTXID(verify->context_id),
1838 rctxid = verify->context_id;
1841 dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llx, hint=%016llx, "
1842 "flags=%016llx\n", __func__, ctxid, verify->rsrc_handle,
1843 verify->hint, verify->hdr.flags);
1845 ctxi = get_context(cfg, rctxid, lli, 0);
1846 if (unlikely(!ctxi)) {
1847 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1852 rhte = get_rhte(ctxi, rhndl, lli);
1853 if (unlikely(!rhte)) {
1854 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
1861 * Look at the hint/sense to see if it requires us to redrive
1862 * inquiry (i.e. the Unit attention is due to the WWN changing).
1864 if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) {
1865 /* Can't hold mutex across process_sense/read_cap16,
1866 * since we could have an intervening EEH event.
1868 ctxi->unavail = true;
1869 mutex_unlock(&ctxi->mutex);
1870 rc = process_sense(sdev, verify);
1872 dev_err(dev, "%s: Failed to validate sense data (%d)\n",
1874 mutex_lock(&ctxi->mutex);
1875 ctxi->unavail = false;
1878 mutex_lock(&ctxi->mutex);
1879 ctxi->unavail = false;
1882 switch (gli->mode) {
1884 last_lba = gli->max_lba;
1887 /* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */
1888 last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len);
1889 last_lba /= CXLFLASH_BLOCK_SIZE;
1893 WARN(1, "Unsupported LUN mode!");
1896 verify->last_lba = last_lba;
1901 dev_dbg(dev, "%s: returning rc=%d llba=%llx\n",
1902 __func__, rc, verify->last_lba);
1907 * decode_ioctl() - translates an encoded ioctl to an easily identifiable string
1908 * @cmd: The ioctl command to decode.
1910 * Return: A string identifying the decoded ioctl.
1912 static char *decode_ioctl(int cmd)
1915 case DK_CXLFLASH_ATTACH:
1916 return __stringify_1(DK_CXLFLASH_ATTACH);
1917 case DK_CXLFLASH_USER_DIRECT:
1918 return __stringify_1(DK_CXLFLASH_USER_DIRECT);
1919 case DK_CXLFLASH_USER_VIRTUAL:
1920 return __stringify_1(DK_CXLFLASH_USER_VIRTUAL);
1921 case DK_CXLFLASH_VLUN_RESIZE:
1922 return __stringify_1(DK_CXLFLASH_VLUN_RESIZE);
1923 case DK_CXLFLASH_RELEASE:
1924 return __stringify_1(DK_CXLFLASH_RELEASE);
1925 case DK_CXLFLASH_DETACH:
1926 return __stringify_1(DK_CXLFLASH_DETACH);
1927 case DK_CXLFLASH_VERIFY:
1928 return __stringify_1(DK_CXLFLASH_VERIFY);
1929 case DK_CXLFLASH_VLUN_CLONE:
1930 return __stringify_1(DK_CXLFLASH_VLUN_CLONE);
1931 case DK_CXLFLASH_RECOVER_AFU:
1932 return __stringify_1(DK_CXLFLASH_RECOVER_AFU);
1933 case DK_CXLFLASH_MANAGE_LUN:
1934 return __stringify_1(DK_CXLFLASH_MANAGE_LUN);
1941 * cxlflash_disk_direct_open() - opens a direct (physical) disk
1942 * @sdev: SCSI device associated with LUN.
1943 * @arg: UDirect ioctl data structure.
1945 * On successful return, the user is informed of the resource handle
1946 * to be used to identify the direct lun and the size (in blocks) of
1947 * the direct lun in last LBA format.
1949 * Return: 0 on success, -errno on failure
1951 static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg)
1953 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1954 struct device *dev = &cfg->dev->dev;
1955 struct afu *afu = cfg->afu;
1956 struct llun_info *lli = sdev->hostdata;
1957 struct glun_info *gli = lli->parent;
1958 struct dk_cxlflash_release rel = { { 0 }, 0 };
1960 struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg;
1962 u64 ctxid = DECODE_CTXID(pphys->context_id),
1963 rctxid = pphys->context_id;
1966 u64 rsrc_handle = -1;
1967 u32 port = CHAN2PORTMASK(sdev->channel);
1971 struct ctx_info *ctxi = NULL;
1972 struct sisl_rht_entry *rhte = NULL;
1974 dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);
1976 rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false);
1978 dev_dbg(dev, "%s: Failed attach to LUN (PHYSICAL)\n", __func__);
1982 ctxi = get_context(cfg, rctxid, lli, 0);
1983 if (unlikely(!ctxi)) {
1984 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1989 rhte = rhte_checkout(ctxi, lli);
1990 if (unlikely(!rhte)) {
1991 dev_dbg(dev, "%s: Too many opens ctxid=%lld\n",
1993 rc = -EMFILE; /* too many opens */
1997 rsrc_handle = (rhte - ctxi->rht_start);
1999 rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port);
2001 last_lba = gli->max_lba;
2002 pphys->hdr.return_flags = 0;
2003 pphys->last_lba = last_lba;
2004 pphys->rsrc_handle = rsrc_handle;
2006 rc = cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC);
2008 dev_dbg(dev, "%s: AFU sync failed rc=%d\n", __func__, rc);
2015 dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",
2016 __func__, rsrc_handle, rc, last_lba);
2020 marshal_udir_to_rele(pphys, &rel);
2021 _cxlflash_disk_release(sdev, ctxi, &rel);
2024 cxlflash_lun_detach(gli);
2029 * ioctl_common() - common IOCTL handler for driver
2030 * @sdev: SCSI device associated with LUN.
2031 * @cmd: IOCTL command.
2033 * Handles common fencing operations that are valid for multiple ioctls. Always
2034 * allow through ioctls that are cleanup oriented in nature, even when operating
2035 * in a failed/terminating state.
2037 * Return: 0 on success, -errno on failure
2039 static int ioctl_common(struct scsi_device *sdev, int cmd)
2041 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2042 struct device *dev = &cfg->dev->dev;
2043 struct llun_info *lli = sdev->hostdata;
2046 if (unlikely(!lli)) {
2047 dev_dbg(dev, "%s: Unknown LUN\n", __func__);
2052 rc = check_state(cfg);
2053 if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) {
2055 case DK_CXLFLASH_VLUN_RESIZE:
2056 case DK_CXLFLASH_RELEASE:
2057 case DK_CXLFLASH_DETACH:
2058 dev_dbg(dev, "%s: Command override rc=%d\n",
2069 * cxlflash_ioctl() - IOCTL handler for driver
2070 * @sdev: SCSI device associated with LUN.
2071 * @cmd: IOCTL command.
2072 * @arg: Userspace ioctl data structure.
2074 * A read/write semaphore is used to implement a 'drain' of currently
2075 * running ioctls. The read semaphore is taken at the beginning of each
2076 * ioctl thread and released upon concluding execution. Additionally the
2077 * semaphore should be released and then reacquired in any ioctl execution
2078 * path which will wait for an event to occur that is outside the scope of
2079 * the ioctl (i.e. an adapter reset). To drain the ioctls currently running,
2080 * a thread simply needs to acquire the write semaphore.
2082 * Return: 0 on success, -errno on failure
2084 int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg)
2086 typedef int (*sioctl) (struct scsi_device *, void *);
2088 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2089 struct device *dev = &cfg->dev->dev;
2090 struct afu *afu = cfg->afu;
2091 struct dk_cxlflash_hdr *hdr;
2092 char buf[sizeof(union cxlflash_ioctls)];
2094 bool known_ioctl = false;
2097 struct Scsi_Host *shost = sdev->host;
2098 sioctl do_ioctl = NULL;
2100 static const struct {
2103 } ioctl_tbl[] = { /* NOTE: order matters here */
2104 {sizeof(struct dk_cxlflash_attach), (sioctl)cxlflash_disk_attach},
2105 {sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open},
2106 {sizeof(struct dk_cxlflash_release), (sioctl)cxlflash_disk_release},
2107 {sizeof(struct dk_cxlflash_detach), (sioctl)cxlflash_disk_detach},
2108 {sizeof(struct dk_cxlflash_verify), (sioctl)cxlflash_disk_verify},
2109 {sizeof(struct dk_cxlflash_recover_afu), (sioctl)cxlflash_afu_recover},
2110 {sizeof(struct dk_cxlflash_manage_lun), (sioctl)cxlflash_manage_lun},
2111 {sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open},
2112 {sizeof(struct dk_cxlflash_resize), (sioctl)cxlflash_vlun_resize},
2113 {sizeof(struct dk_cxlflash_clone), (sioctl)cxlflash_disk_clone},
2116 /* Hold read semaphore so we can drain if needed */
2117 down_read(&cfg->ioctl_rwsem);
2119 /* Restrict command set to physical support only for internal LUN */
2120 if (afu->internal_lun)
2122 case DK_CXLFLASH_RELEASE:
2123 case DK_CXLFLASH_USER_VIRTUAL:
2124 case DK_CXLFLASH_VLUN_RESIZE:
2125 case DK_CXLFLASH_VLUN_CLONE:
2126 dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n",
2127 __func__, decode_ioctl(cmd), afu->internal_lun);
2129 goto cxlflash_ioctl_exit;
2133 case DK_CXLFLASH_ATTACH:
2134 case DK_CXLFLASH_USER_DIRECT:
2135 case DK_CXLFLASH_RELEASE:
2136 case DK_CXLFLASH_DETACH:
2137 case DK_CXLFLASH_VERIFY:
2138 case DK_CXLFLASH_RECOVER_AFU:
2139 case DK_CXLFLASH_USER_VIRTUAL:
2140 case DK_CXLFLASH_VLUN_RESIZE:
2141 case DK_CXLFLASH_VLUN_CLONE:
2142 dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",
2143 __func__, decode_ioctl(cmd), cmd, shost->host_no,
2144 sdev->channel, sdev->id, sdev->lun);
2145 rc = ioctl_common(sdev, cmd);
2147 goto cxlflash_ioctl_exit;
2151 case DK_CXLFLASH_MANAGE_LUN:
2153 idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH);
2154 size = ioctl_tbl[idx].size;
2155 do_ioctl = ioctl_tbl[idx].ioctl;
2157 if (likely(do_ioctl))
2163 goto cxlflash_ioctl_exit;
2166 if (unlikely(copy_from_user(&buf, arg, size))) {
2167 dev_err(dev, "%s: copy_from_user() fail "
2168 "size=%lu cmd=%d (%s) arg=%p\n",
2169 __func__, size, cmd, decode_ioctl(cmd), arg);
2171 goto cxlflash_ioctl_exit;
2174 hdr = (struct dk_cxlflash_hdr *)&buf;
2175 if (hdr->version != DK_CXLFLASH_VERSION_0) {
2176 dev_dbg(dev, "%s: Version %u not supported for %s\n",
2177 __func__, hdr->version, decode_ioctl(cmd));
2179 goto cxlflash_ioctl_exit;
2182 if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) {
2183 dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__);
2185 goto cxlflash_ioctl_exit;
2188 rc = do_ioctl(sdev, (void *)&buf);
2190 if (unlikely(copy_to_user(arg, &buf, size))) {
2191 dev_err(dev, "%s: copy_to_user() fail "
2192 "size=%lu cmd=%d (%s) arg=%p\n",
2193 __func__, size, cmd, decode_ioctl(cmd), arg);
2197 /* fall through to exit */
2199 cxlflash_ioctl_exit:
2200 up_read(&cfg->ioctl_rwsem);
2201 if (unlikely(rc && known_ioctl))
2202 dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2203 "returned rc %d\n", __func__,
2204 decode_ioctl(cmd), cmd, shost->host_no,
2205 sdev->channel, sdev->id, sdev->lun, rc);
2207 dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2208 "returned rc %d\n", __func__, decode_ioctl(cmd),
2209 cmd, shost->host_no, sdev->channel, sdev->id,