drivers/infiniband/ulp/isert/ib_isert.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*******************************************************************************
   3  * This file contains iSCSI extentions for RDMA (iSER) Verbs
   4  *
   5  * (c) Copyright 2013 Datera, Inc.
   6  *
   7  * Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  ****************************************************************************/
  10
  11 #include <linux/string.h>
  12 #include <linux/module.h>
  13 #include <linux/scatterlist.h>
  14 #include <linux/socket.h>
  15 #include <linux/in.h>
  16 #include <linux/in6.h>
  17 #include <rdma/ib_verbs.h>
  18 #include <rdma/ib_cm.h>
  19 #include <rdma/rdma_cm.h>
  20 #include <target/target_core_base.h>
  21 #include <target/target_core_fabric.h>
  22 #include <target/iscsi/iscsi_transport.h>
  23 #include <linux/semaphore.h>
  24
  25 #include "ib_isert.h"
  26
  27 static int isert_debug_level;
  28 module_param_named(debug_level, isert_debug_level, int, 0644);
  29 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
  30
  31 static int isert_sg_tablesize_set(const char *val,
  32                                   const struct kernel_param *kp);
  33 static const struct kernel_param_ops sg_tablesize_ops = {
  34         .set = isert_sg_tablesize_set,
  35         .get = param_get_int,
  36 };
  37
  38 static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE;
  39 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644);
  40 MODULE_PARM_DESC(sg_tablesize,
  41                  "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)");
  42
  43 static DEFINE_MUTEX(device_list_mutex);
  44 static LIST_HEAD(device_list);
  45 static struct workqueue_struct *isert_comp_wq;
  46 static struct workqueue_struct *isert_release_wq;
  47
  48 static int
  49 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
  50 static int
  51 isert_login_post_recv(struct isert_conn *isert_conn);
  52 static int
  53 isert_rdma_accept(struct isert_conn *isert_conn);
  54 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
  55
  56 static void isert_release_work(struct work_struct *work);
  57 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
  58 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
  59 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
  60 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
  61
  62 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp)
  63 {
  64         int n = 0, ret;
  65
  66         ret = kstrtoint(val, 10, &n);
  67         if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE ||
  68             n > ISCSI_ISER_MAX_SG_TABLESIZE)
  69                 return -EINVAL;
  70
  71         return param_set_int(val, kp);
  72 }
  73
  74
  75 static inline bool
  76 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
  77 {
  78         return (conn->pi_support &&
  79                 cmd->prot_op != TARGET_PROT_NORMAL);
  80 }
  81
  82
  83 static void
  84 isert_qp_event_callback(struct ib_event *e, void *context)
  85 {
  86         struct isert_conn *isert_conn = context;
  87
  88         isert_err("%s (%d): conn %p\n",
  89                   ib_event_msg(e->event), e->event, isert_conn);
  90
  91         switch (e->event) {
  92         case IB_EVENT_COMM_EST:
  93                 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
  94                 break;
  95         case IB_EVENT_QP_LAST_WQE_REACHED:
  96                 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
  97                 break;
  98         default:
  99                 break;
 100         }
 101 }
 102
 103 static struct ib_qp *
 104 isert_create_qp(struct isert_conn *isert_conn,
 105                 struct rdma_cm_id *cma_id)
 106 {
 107         u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2;
 108         struct isert_device *device = isert_conn->device;
 109         struct ib_device *ib_dev = device->ib_device;
 110         struct ib_qp_init_attr attr;
 111         int ret, factor;
 112
 113         isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE);
 114         if (IS_ERR(isert_conn->cq)) {
 115                 isert_err("Unable to allocate cq\n");
 116                 ret = PTR_ERR(isert_conn->cq);
 117                 return ERR_PTR(ret);
 118         }
 119         isert_conn->cq_size = cq_size;
 120
 121         memset(&attr, 0, sizeof(struct ib_qp_init_attr));
 122         attr.event_handler = isert_qp_event_callback;
 123         attr.qp_context = isert_conn;
 124         attr.send_cq = isert_conn->cq;
 125         attr.recv_cq = isert_conn->cq;
 126         attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
 127         attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
 128         factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num,
 129                                    isert_sg_tablesize);
 130         attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor;
 131         attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
 132         attr.cap.max_recv_sge = 1;
 133         attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 134         attr.qp_type = IB_QPT_RC;
 135         if (device->pi_capable)
 136                 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
 137
 138         ret = rdma_create_qp(cma_id, device->pd, &attr);
 139         if (ret) {
 140                 isert_err("rdma_create_qp failed for cma_id %d\n", ret);
 141                 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
 142
 143                 return ERR_PTR(ret);
 144         }
 145
 146         return cma_id->qp;
 147 }
 148
 149 static int
 150 isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
 151 {
 152         struct isert_device *device = isert_conn->device;
 153         struct ib_device *ib_dev = device->ib_device;
 154         struct iser_rx_desc *rx_desc;
 155         struct ib_sge *rx_sg;
 156         u64 dma_addr;
 157         int i, j;
 158
 159         isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
 160                                        sizeof(struct iser_rx_desc),
 161                                        GFP_KERNEL);
 162         if (!isert_conn->rx_descs)
 163                 return -ENOMEM;
 164
 165         rx_desc = isert_conn->rx_descs;
 166
 167         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 168                 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf,
 169                                         ISER_RX_SIZE, DMA_FROM_DEVICE);
 170                 if (ib_dma_mapping_error(ib_dev, dma_addr))
 171                         goto dma_map_fail;
 172
 173                 rx_desc->dma_addr = dma_addr;
 174
 175                 rx_sg = &rx_desc->rx_sg;
 176                 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc);
 177                 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
 178                 rx_sg->lkey = device->pd->local_dma_lkey;
 179                 rx_desc->rx_cqe.done = isert_recv_done;
 180         }
 181
 182         return 0;
 183
 184 dma_map_fail:
 185         rx_desc = isert_conn->rx_descs;
 186         for (j = 0; j < i; j++, rx_desc++) {
 187                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 188                                     ISER_RX_SIZE, DMA_FROM_DEVICE);
 189         }
 190         kfree(isert_conn->rx_descs);
 191         isert_conn->rx_descs = NULL;
 192         isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
 193         return -ENOMEM;
 194 }
 195
 196 static void
 197 isert_free_rx_descriptors(struct isert_conn *isert_conn)
 198 {
 199         struct ib_device *ib_dev = isert_conn->device->ib_device;
 200         struct iser_rx_desc *rx_desc;
 201         int i;
 202
 203         if (!isert_conn->rx_descs)
 204                 return;
 205
 206         rx_desc = isert_conn->rx_descs;
 207         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 208                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 209                                     ISER_RX_SIZE, DMA_FROM_DEVICE);
 210         }
 211
 212         kfree(isert_conn->rx_descs);
 213         isert_conn->rx_descs = NULL;
 214 }
 215
 216 static int
 217 isert_create_device_ib_res(struct isert_device *device)
 218 {
 219         struct ib_device *ib_dev = device->ib_device;
 220         int ret;
 221
 222         isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
 223                   ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
 224         isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
 225
 226         device->pd = ib_alloc_pd(ib_dev, 0);
 227         if (IS_ERR(device->pd)) {
 228                 ret = PTR_ERR(device->pd);
 229                 isert_err("failed to allocate pd, device %p, ret=%d\n",
 230                           device, ret);
 231                 return ret;
 232         }
 233
 234         /* Check signature cap */
 235         device->pi_capable = ib_dev->attrs.device_cap_flags &
 236                              IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
 237
 238         return 0;
 239 }
 240
 241 static void
 242 isert_free_device_ib_res(struct isert_device *device)
 243 {
 244         isert_info("device %p\n", device);
 245
 246         ib_dealloc_pd(device->pd);
 247 }
 248
 249 static void
 250 isert_device_put(struct isert_device *device)
 251 {
 252         mutex_lock(&device_list_mutex);
 253         device->refcount--;
 254         isert_info("device %p refcount %d\n", device, device->refcount);
 255         if (!device->refcount) {
 256                 isert_free_device_ib_res(device);
 257                 list_del(&device->dev_node);
 258                 kfree(device);
 259         }
 260         mutex_unlock(&device_list_mutex);
 261 }
 262
 263 static struct isert_device *
 264 isert_device_get(struct rdma_cm_id *cma_id)
 265 {
 266         struct isert_device *device;
 267         int ret;
 268
 269         mutex_lock(&device_list_mutex);
 270         list_for_each_entry(device, &device_list, dev_node) {
 271                 if (device->ib_device->node_guid == cma_id->device->node_guid) {
 272                         device->refcount++;
 273                         isert_info("Found iser device %p refcount %d\n",
 274                                    device, device->refcount);
 275                         mutex_unlock(&device_list_mutex);
 276                         return device;
 277                 }
 278         }
 279
 280         device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
 281         if (!device) {
 282                 mutex_unlock(&device_list_mutex);
 283                 return ERR_PTR(-ENOMEM);
 284         }
 285
 286         INIT_LIST_HEAD(&device->dev_node);
 287
 288         device->ib_device = cma_id->device;
 289         ret = isert_create_device_ib_res(device);
 290         if (ret) {
 291                 kfree(device);
 292                 mutex_unlock(&device_list_mutex);
 293                 return ERR_PTR(ret);
 294         }
 295
 296         device->refcount++;
 297         list_add_tail(&device->dev_node, &device_list);
 298         isert_info("Created a new iser device %p refcount %d\n",
 299                    device, device->refcount);
 300         mutex_unlock(&device_list_mutex);
 301
 302         return device;
 303 }
 304
 305 static void
 306 isert_init_conn(struct isert_conn *isert_conn)
 307 {
 308         isert_conn->state = ISER_CONN_INIT;
 309         INIT_LIST_HEAD(&isert_conn->node);
 310         init_completion(&isert_conn->login_comp);
 311         init_completion(&isert_conn->login_req_comp);
 312         init_waitqueue_head(&isert_conn->rem_wait);
 313         kref_init(&isert_conn->kref);
 314         mutex_init(&isert_conn->mutex);
 315         INIT_WORK(&isert_conn->release_work, isert_release_work);
 316 }
 317
 318 static void
 319 isert_free_login_buf(struct isert_conn *isert_conn)
 320 {
 321         struct ib_device *ib_dev = isert_conn->device->ib_device;
 322
 323         ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
 324                             ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
 325         kfree(isert_conn->login_rsp_buf);
 326
 327         ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr,
 328                             ISER_RX_SIZE, DMA_FROM_DEVICE);
 329         kfree(isert_conn->login_desc);
 330 }
 331
 332 static int
 333 isert_alloc_login_buf(struct isert_conn *isert_conn,
 334                       struct ib_device *ib_dev)
 335 {
 336         int ret;
 337
 338         isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc),
 339                         GFP_KERNEL);
 340         if (!isert_conn->login_desc)
 341                 return -ENOMEM;
 342
 343         isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev,
 344                                 isert_conn->login_desc->buf,
 345                                 ISER_RX_SIZE, DMA_FROM_DEVICE);
 346         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr);
 347         if (ret) {
 348                 isert_err("login_desc dma mapping error: %d\n", ret);
 349                 isert_conn->login_desc->dma_addr = 0;
 350                 goto out_free_login_desc;
 351         }
 352
 353         isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
 354         if (!isert_conn->login_rsp_buf) {
 355                 ret = -ENOMEM;
 356                 goto out_unmap_login_desc;
 357         }
 358
 359         isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
 360                                         isert_conn->login_rsp_buf,
 361                                         ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
 362         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
 363         if (ret) {
 364                 isert_err("login_rsp_dma mapping error: %d\n", ret);
 365                 isert_conn->login_rsp_dma = 0;
 366                 goto out_free_login_rsp_buf;
 367         }
 368
 369         return 0;
 370
 371 out_free_login_rsp_buf:
 372         kfree(isert_conn->login_rsp_buf);
 373 out_unmap_login_desc:
 374         ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr,
 375                             ISER_RX_SIZE, DMA_FROM_DEVICE);
 376 out_free_login_desc:
 377         kfree(isert_conn->login_desc);
 378         return ret;
 379 }
 380
 381 static void
 382 isert_set_nego_params(struct isert_conn *isert_conn,
 383                       struct rdma_conn_param *param)
 384 {
 385         struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
 386
 387         /* Set max inflight RDMA READ requests */
 388         isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
 389                                 attr->max_qp_init_rd_atom);
 390         isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
 391
 392         if (param->private_data) {
 393                 u8 flags = *(u8 *)param->private_data;
 394
 395                 /*
 396                  * use remote invalidation if the both initiator
 397                  * and the HCA support it
 398                  */
 399                 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
 400                                           (attr->device_cap_flags &
 401                                            IB_DEVICE_MEM_MGT_EXTENSIONS);
 402                 if (isert_conn->snd_w_inv)
 403                         isert_info("Using remote invalidation\n");
 404         }
 405 }
 406
 407 static void
 408 isert_destroy_qp(struct isert_conn *isert_conn)
 409 {
 410         ib_destroy_qp(isert_conn->qp);
 411         ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
 412 }
 413
 414 static int
 415 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 416 {
 417         struct isert_np *isert_np = cma_id->context;
 418         struct iscsi_np *np = isert_np->np;
 419         struct isert_conn *isert_conn;
 420         struct isert_device *device;
 421         int ret = 0;
 422
 423         spin_lock_bh(&np->np_thread_lock);
 424         if (!np->enabled) {
 425                 spin_unlock_bh(&np->np_thread_lock);
 426                 isert_dbg("iscsi_np is not enabled, reject connect request\n");
 427                 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
 428         }
 429         spin_unlock_bh(&np->np_thread_lock);
 430
 431         isert_dbg("cma_id: %p, portal: %p\n",
 432                  cma_id, cma_id->context);
 433
 434         isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
 435         if (!isert_conn)
 436                 return -ENOMEM;
 437
 438         isert_init_conn(isert_conn);
 439         isert_conn->cm_id = cma_id;
 440
 441         device = isert_device_get(cma_id);
 442         if (IS_ERR(device)) {
 443                 ret = PTR_ERR(device);
 444                 goto out;
 445         }
 446         isert_conn->device = device;
 447
 448         ret = isert_alloc_login_buf(isert_conn, cma_id->device);
 449         if (ret)
 450                 goto out_conn_dev;
 451
 452         isert_set_nego_params(isert_conn, &event->param.conn);
 453
 454         isert_conn->qp = isert_create_qp(isert_conn, cma_id);
 455         if (IS_ERR(isert_conn->qp)) {
 456                 ret = PTR_ERR(isert_conn->qp);
 457                 goto out_rsp_dma_map;
 458         }
 459
 460         ret = isert_login_post_recv(isert_conn);
 461         if (ret)
 462                 goto out_destroy_qp;
 463
 464         ret = isert_rdma_accept(isert_conn);
 465         if (ret)
 466                 goto out_destroy_qp;
 467
 468         mutex_lock(&isert_np->mutex);
 469         list_add_tail(&isert_conn->node, &isert_np->accepted);
 470         mutex_unlock(&isert_np->mutex);
 471
 472         return 0;
 473
 474 out_destroy_qp:
 475         isert_destroy_qp(isert_conn);
 476 out_rsp_dma_map:
 477         isert_free_login_buf(isert_conn);
 478 out_conn_dev:
 479         isert_device_put(device);
 480 out:
 481         kfree(isert_conn);
 482         rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
 483         return ret;
 484 }
 485
 486 static void
 487 isert_connect_release(struct isert_conn *isert_conn)
 488 {
 489         struct isert_device *device = isert_conn->device;
 490
 491         isert_dbg("conn %p\n", isert_conn);
 492
 493         BUG_ON(!device);
 494
 495         isert_free_rx_descriptors(isert_conn);
 496         if (isert_conn->cm_id &&
 497             !isert_conn->dev_removed)
 498                 rdma_destroy_id(isert_conn->cm_id);
 499
 500         if (isert_conn->qp)
 501                 isert_destroy_qp(isert_conn);
 502
 503         if (isert_conn->login_desc)
 504                 isert_free_login_buf(isert_conn);
 505
 506         isert_device_put(device);
 507
 508         if (isert_conn->dev_removed)
 509                 wake_up_interruptible(&isert_conn->rem_wait);
 510         else
 511                 kfree(isert_conn);
 512 }
 513
 514 static void
 515 isert_connected_handler(struct rdma_cm_id *cma_id)
 516 {
 517         struct isert_conn *isert_conn = cma_id->qp->qp_context;
 518         struct isert_np *isert_np = cma_id->context;
 519
 520         isert_info("conn %p\n", isert_conn);
 521
 522         mutex_lock(&isert_conn->mutex);
 523         isert_conn->state = ISER_CONN_UP;
 524         kref_get(&isert_conn->kref);
 525         mutex_unlock(&isert_conn->mutex);
 526
 527         mutex_lock(&isert_np->mutex);
 528         list_move_tail(&isert_conn->node, &isert_np->pending);
 529         mutex_unlock(&isert_np->mutex);
 530
 531         isert_info("np %p: Allow accept_np to continue\n", isert_np);
 532         up(&isert_np->sem);
 533 }
 534
 535 static void
 536 isert_release_kref(struct kref *kref)
 537 {
 538         struct isert_conn *isert_conn = container_of(kref,
 539                                 struct isert_conn, kref);
 540
 541         isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
 542                    current->pid);
 543
 544         isert_connect_release(isert_conn);
 545 }
 546
 547 static void
 548 isert_put_conn(struct isert_conn *isert_conn)
 549 {
 550         kref_put(&isert_conn->kref, isert_release_kref);
 551 }
 552
 553 static void
 554 isert_handle_unbound_conn(struct isert_conn *isert_conn)
 555 {
 556         struct isert_np *isert_np = isert_conn->cm_id->context;
 557
 558         mutex_lock(&isert_np->mutex);
 559         if (!list_empty(&isert_conn->node)) {
 560                 /*
 561                  * This means iscsi doesn't know this connection
 562                  * so schedule a cleanup ourselves
 563                  */
 564                 list_del_init(&isert_conn->node);
 565                 isert_put_conn(isert_conn);
 566                 queue_work(isert_release_wq, &isert_conn->release_work);
 567         }
 568         mutex_unlock(&isert_np->mutex);
 569 }
 570
 571 /**
 572  * isert_conn_terminate() - Initiate connection termination
 573  * @isert_conn: isert connection struct
 574  *
 575  * Notes:
 576  * In case the connection state is BOUND, move state
 577  * to TEMINATING and start teardown sequence (rdma_disconnect).
 578  * In case the connection state is UP, complete flush as well.
 579  *
 580  * This routine must be called with mutex held. Thus it is
 581  * safe to call multiple times.
 582  */
 583 static void
 584 isert_conn_terminate(struct isert_conn *isert_conn)
 585 {
 586         int err;
 587
 588         if (isert_conn->state >= ISER_CONN_TERMINATING)
 589                 return;
 590
 591         isert_info("Terminating conn %p state %d\n",
 592                    isert_conn, isert_conn->state);
 593         isert_conn->state = ISER_CONN_TERMINATING;
 594         err = rdma_disconnect(isert_conn->cm_id);
 595         if (err)
 596                 isert_warn("Failed rdma_disconnect isert_conn %p\n",
 597                            isert_conn);
 598 }
 599
 600 static int
 601 isert_np_cma_handler(struct isert_np *isert_np,
 602                      enum rdma_cm_event_type event)
 603 {
 604         isert_dbg("%s (%d): isert np %p\n",
 605                   rdma_event_msg(event), event, isert_np);
 606
 607         switch (event) {
 608         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 609                 isert_np->cm_id = NULL;
 610                 break;
 611         case RDMA_CM_EVENT_ADDR_CHANGE:
 612                 isert_np->cm_id = isert_setup_id(isert_np);
 613                 if (IS_ERR(isert_np->cm_id)) {
 614                         isert_err("isert np %p setup id failed: %ld\n",
 615                                   isert_np, PTR_ERR(isert_np->cm_id));
 616                         isert_np->cm_id = NULL;
 617                 }
 618                 break;
 619         default:
 620                 isert_err("isert np %p Unexpected event %d\n",
 621                           isert_np, event);
 622         }
 623
 624         return -1;
 625 }
 626
 627 static int
 628 isert_disconnected_handler(struct rdma_cm_id *cma_id,
 629                            enum rdma_cm_event_type event)
 630 {
 631         struct isert_conn *isert_conn = cma_id->qp->qp_context;
 632
 633         mutex_lock(&isert_conn->mutex);
 634         switch (isert_conn->state) {
 635         case ISER_CONN_TERMINATING:
 636                 break;
 637         case ISER_CONN_UP:
 638                 isert_conn_terminate(isert_conn);
 639                 ib_drain_qp(isert_conn->qp);
 640                 isert_handle_unbound_conn(isert_conn);
 641                 break;
 642         case ISER_CONN_BOUND:
 643         case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
 644                 iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
 645                 break;
 646         default:
 647                 isert_warn("conn %p terminating in state %d\n",
 648                            isert_conn, isert_conn->state);
 649         }
 650         mutex_unlock(&isert_conn->mutex);
 651
 652         return 0;
 653 }
 654
 655 static int
 656 isert_connect_error(struct rdma_cm_id *cma_id)
 657 {
 658         struct isert_conn *isert_conn = cma_id->qp->qp_context;
 659
 660         ib_drain_qp(isert_conn->qp);
 661         list_del_init(&isert_conn->node);
 662         isert_conn->cm_id = NULL;
 663         isert_put_conn(isert_conn);
 664
 665         return -1;
 666 }
 667
 668 static int
 669 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 670 {
 671         struct isert_np *isert_np = cma_id->context;
 672         struct isert_conn *isert_conn;
 673         int ret = 0;
 674
 675         isert_info("%s (%d): status %d id %p np %p\n",
 676                    rdma_event_msg(event->event), event->event,
 677                    event->status, cma_id, cma_id->context);
 678
 679         if (isert_np->cm_id == cma_id)
 680                 return isert_np_cma_handler(cma_id->context, event->event);
 681
 682         switch (event->event) {
 683         case RDMA_CM_EVENT_CONNECT_REQUEST:
 684                 ret = isert_connect_request(cma_id, event);
 685                 if (ret)
 686                         isert_err("failed handle connect request %d\n", ret);
 687                 break;
 688         case RDMA_CM_EVENT_ESTABLISHED:
 689                 isert_connected_handler(cma_id);
 690                 break;
 691         case RDMA_CM_EVENT_ADDR_CHANGE:
 692         case RDMA_CM_EVENT_DISCONNECTED:
 693         case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
 694                 ret = isert_disconnected_handler(cma_id, event->event);
 695                 break;
 696         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 697                 isert_conn = cma_id->qp->qp_context;
 698                 isert_conn->dev_removed = true;
 699                 isert_disconnected_handler(cma_id, event->event);
 700                 wait_event_interruptible(isert_conn->rem_wait,
 701                                          isert_conn->state == ISER_CONN_DOWN);
 702                 kfree(isert_conn);
 703                 /*
 704                  * return non-zero from the callback to destroy
 705                  * the rdma cm id
 706                  */
 707                 return 1;
 708         case RDMA_CM_EVENT_REJECTED:
 709                 isert_info("Connection rejected: %s\n",
 710                            rdma_reject_msg(cma_id, event->status));
 711                 fallthrough;
 712         case RDMA_CM_EVENT_UNREACHABLE:
 713         case RDMA_CM_EVENT_CONNECT_ERROR:
 714                 ret = isert_connect_error(cma_id);
 715                 break;
 716         default:
 717                 isert_err("Unhandled RDMA CMA event: %d\n", event->event);
 718                 break;
 719         }
 720
 721         return ret;
 722 }
 723
 724 static int
 725 isert_post_recvm(struct isert_conn *isert_conn, u32 count)
 726 {
 727         struct ib_recv_wr *rx_wr;
 728         int i, ret;
 729         struct iser_rx_desc *rx_desc;
 730
 731         for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
 732                 rx_desc = &isert_conn->rx_descs[i];
 733
 734                 rx_wr->wr_cqe = &rx_desc->rx_cqe;
 735                 rx_wr->sg_list = &rx_desc->rx_sg;
 736                 rx_wr->num_sge = 1;
 737                 rx_wr->next = rx_wr + 1;
 738                 rx_desc->in_use = false;
 739         }
 740         rx_wr--;
 741         rx_wr->next = NULL; /* mark end of work requests list */
 742
 743         ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
 744         if (ret)
 745                 isert_err("ib_post_recv() failed with ret: %d\n", ret);
 746
 747         return ret;
 748 }
 749
 750 static int
 751 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
 752 {
 753         struct ib_recv_wr rx_wr;
 754         int ret;
 755
 756         if (!rx_desc->in_use) {
 757                 /*
 758                  * if the descriptor is not in-use we already reposted it
 759                  * for recv, so just silently return
 760                  */
 761                 return 0;
 762         }
 763
 764         rx_desc->in_use = false;
 765         rx_wr.wr_cqe = &rx_desc->rx_cqe;
 766         rx_wr.sg_list = &rx_desc->rx_sg;
 767         rx_wr.num_sge = 1;
 768         rx_wr.next = NULL;
 769
 770         ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
 771         if (ret)
 772                 isert_err("ib_post_recv() failed with ret: %d\n", ret);
 773
 774         return ret;
 775 }
 776
 777 static int
 778 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
 779 {
 780         struct ib_device *ib_dev = isert_conn->cm_id->device;
 781         struct ib_send_wr send_wr;
 782         int ret;
 783
 784         ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
 785                                       ISER_HEADERS_LEN, DMA_TO_DEVICE);
 786
 787         tx_desc->tx_cqe.done = isert_login_send_done;
 788
 789         send_wr.next    = NULL;
 790         send_wr.wr_cqe  = &tx_desc->tx_cqe;
 791         send_wr.sg_list = tx_desc->tx_sg;
 792         send_wr.num_sge = tx_desc->num_sge;
 793         send_wr.opcode  = IB_WR_SEND;
 794         send_wr.send_flags = IB_SEND_SIGNALED;
 795
 796         ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
 797         if (ret)
 798                 isert_err("ib_post_send() failed, ret: %d\n", ret);
 799
 800         return ret;
 801 }
 802
 803 static void
 804 __isert_create_send_desc(struct isert_device *device,
 805                          struct iser_tx_desc *tx_desc)
 806 {
 807
 808         memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
 809         tx_desc->iser_header.flags = ISCSI_CTRL;
 810
 811         tx_desc->num_sge = 1;
 812
 813         if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
 814                 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
 815                 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
 816         }
 817 }
 818
 819 static void
 820 isert_create_send_desc(struct isert_conn *isert_conn,
 821                        struct isert_cmd *isert_cmd,
 822                        struct iser_tx_desc *tx_desc)
 823 {
 824         struct isert_device *device = isert_conn->device;
 825         struct ib_device *ib_dev = device->ib_device;
 826
 827         ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
 828                                    ISER_HEADERS_LEN, DMA_TO_DEVICE);
 829
 830         __isert_create_send_desc(device, tx_desc);
 831 }
 832
 833 static int
 834 isert_init_tx_hdrs(struct isert_conn *isert_conn,
 835                    struct iser_tx_desc *tx_desc)
 836 {
 837         struct isert_device *device = isert_conn->device;
 838         struct ib_device *ib_dev = device->ib_device;
 839         u64 dma_addr;
 840
 841         dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
 842                         ISER_HEADERS_LEN, DMA_TO_DEVICE);
 843         if (ib_dma_mapping_error(ib_dev, dma_addr)) {
 844                 isert_err("ib_dma_mapping_error() failed\n");
 845                 return -ENOMEM;
 846         }
 847
 848         tx_desc->dma_addr = dma_addr;
 849         tx_desc->tx_sg[0].addr  = tx_desc->dma_addr;
 850         tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
 851         tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
 852
 853         isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
 854                   tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
 855                   tx_desc->tx_sg[0].lkey);
 856
 857         return 0;
 858 }
 859
 860 static void
 861 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
 862                    struct ib_send_wr *send_wr)
 863 {
 864         struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
 865
 866         tx_desc->tx_cqe.done = isert_send_done;
 867         send_wr->wr_cqe = &tx_desc->tx_cqe;
 868
 869         if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
 870                 send_wr->opcode  = IB_WR_SEND_WITH_INV;
 871                 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
 872         } else {
 873                 send_wr->opcode = IB_WR_SEND;
 874         }
 875
 876         send_wr->sg_list = &tx_desc->tx_sg[0];
 877         send_wr->num_sge = isert_cmd->tx_desc.num_sge;
 878         send_wr->send_flags = IB_SEND_SIGNALED;
 879 }
 880
 881 static int
 882 isert_login_post_recv(struct isert_conn *isert_conn)
 883 {
 884         struct ib_recv_wr rx_wr;
 885         struct ib_sge sge;
 886         int ret;
 887
 888         memset(&sge, 0, sizeof(struct ib_sge));
 889         sge.addr = isert_conn->login_desc->dma_addr +
 890                 isert_get_hdr_offset(isert_conn->login_desc);
 891         sge.length = ISER_RX_PAYLOAD_SIZE;
 892         sge.lkey = isert_conn->device->pd->local_dma_lkey;
 893
 894         isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
 895                 sge.addr, sge.length, sge.lkey);
 896
 897         isert_conn->login_desc->rx_cqe.done = isert_login_recv_done;
 898
 899         memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
 900         rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe;
 901         rx_wr.sg_list = &sge;
 902         rx_wr.num_sge = 1;
 903
 904         ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
 905         if (ret)
 906                 isert_err("ib_post_recv() failed: %d\n", ret);
 907
 908         return ret;
 909 }
 910
 911 static int
 912 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 913                    u32 length)
 914 {
 915         struct isert_conn *isert_conn = conn->context;
 916         struct isert_device *device = isert_conn->device;
 917         struct ib_device *ib_dev = device->ib_device;
 918         struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
 919         int ret;
 920
 921         __isert_create_send_desc(device, tx_desc);
 922
 923         memcpy(&tx_desc->iscsi_header, &login->rsp[0],
 924                sizeof(struct iscsi_hdr));
 925
 926         isert_init_tx_hdrs(isert_conn, tx_desc);
 927
 928         if (length > 0) {
 929                 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
 930
 931                 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
 932                                            length, DMA_TO_DEVICE);
 933
 934                 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
 935
 936                 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
 937                                               length, DMA_TO_DEVICE);
 938
 939                 tx_dsg->addr    = isert_conn->login_rsp_dma;
 940                 tx_dsg->length  = length;
 941                 tx_dsg->lkey    = isert_conn->device->pd->local_dma_lkey;
 942                 tx_desc->num_sge = 2;
 943         }
 944         if (!login->login_failed) {
 945                 if (login->login_complete) {
 946                         ret = isert_alloc_rx_descriptors(isert_conn);
 947                         if (ret)
 948                                 return ret;
 949
 950                         ret = isert_post_recvm(isert_conn,
 951                                                ISERT_QP_MAX_RECV_DTOS);
 952                         if (ret)
 953                                 return ret;
 954
 955                         /* Now we are in FULL_FEATURE phase */
 956                         mutex_lock(&isert_conn->mutex);
 957                         isert_conn->state = ISER_CONN_FULL_FEATURE;
 958                         mutex_unlock(&isert_conn->mutex);
 959                         goto post_send;
 960                 }
 961
 962                 ret = isert_login_post_recv(isert_conn);
 963                 if (ret)
 964                         return ret;
 965         }
 966 post_send:
 967         ret = isert_login_post_send(isert_conn, tx_desc);
 968         if (ret)
 969                 return ret;
 970
 971         return 0;
 972 }
 973
 974 static void
 975 isert_rx_login_req(struct isert_conn *isert_conn)
 976 {
 977         struct iser_rx_desc *rx_desc = isert_conn->login_desc;
 978         int rx_buflen = isert_conn->login_req_len;
 979         struct iscsi_conn *conn = isert_conn->conn;
 980         struct iscsi_login *login = conn->conn_login;
 981         int size;
 982
 983         isert_info("conn %p\n", isert_conn);
 984
 985         WARN_ON_ONCE(!login);
 986
 987         if (login->first_request) {
 988                 struct iscsi_login_req *login_req =
 989                         (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc);
 990                 /*
 991                  * Setup the initial iscsi_login values from the leading
 992                  * login request PDU.
 993                  */
 994                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
 995                 login->current_stage =
 996                         (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
 997                          >> 2;
 998                 login->version_min      = login_req->min_version;
 999                 login->version_max      = login_req->max_version;
1000                 memcpy(login->isid, login_req->isid, 6);
1001                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1002                 login->init_task_tag    = login_req->itt;
1003                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1004                 login->cid              = be16_to_cpu(login_req->cid);
1005                 login->tsih             = be16_to_cpu(login_req->tsih);
1006         }
1007
1008         memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN);
1009
1010         size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
1011         isert_dbg("Using login payload size: %d, rx_buflen: %d "
1012                   "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
1013                   MAX_KEY_VALUE_PAIRS);
1014         memcpy(login->req_buf, isert_get_data(rx_desc), size);
1015
1016         if (login->first_request) {
1017                 complete(&isert_conn->login_comp);
1018                 return;
1019         }
1020         schedule_delayed_work(&conn->login_work, 0);
1021 }
1022
1023 static struct iscsi_cmd
1024 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
1025 {
1026         struct isert_conn *isert_conn = conn->context;
1027         struct isert_cmd *isert_cmd;
1028         struct iscsi_cmd *cmd;
1029
1030         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
1031         if (!cmd) {
1032                 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1033                 return NULL;
1034         }
1035         isert_cmd = iscsit_priv_cmd(cmd);
1036         isert_cmd->conn = isert_conn;
1037         isert_cmd->iscsi_cmd = cmd;
1038         isert_cmd->rx_desc = rx_desc;
1039
1040         return cmd;
1041 }
1042
1043 static int
1044 isert_handle_scsi_cmd(struct isert_conn *isert_conn,
1045                       struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
1046                       struct iser_rx_desc *rx_desc, unsigned char *buf)
1047 {
1048         struct iscsi_conn *conn = isert_conn->conn;
1049         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1050         int imm_data, imm_data_len, unsol_data, sg_nents, rc;
1051         bool dump_payload = false;
1052         unsigned int data_len;
1053
1054         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1055         if (rc < 0)
1056                 return rc;
1057
1058         imm_data = cmd->immediate_data;
1059         imm_data_len = cmd->first_burst_len;
1060         unsol_data = cmd->unsolicited_data;
1061         data_len = cmd->se_cmd.data_length;
1062
1063         if (imm_data && imm_data_len == data_len)
1064                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1065         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1066         if (rc < 0) {
1067                 return 0;
1068         } else if (rc > 0) {
1069                 dump_payload = true;
1070                 goto sequence_cmd;
1071         }
1072
1073         if (!imm_data)
1074                 return 0;
1075
1076         if (imm_data_len != data_len) {
1077                 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
1078                 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
1079                                     isert_get_data(rx_desc), imm_data_len);
1080                 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
1081                           sg_nents, imm_data_len);
1082         } else {
1083                 sg_init_table(&isert_cmd->sg, 1);
1084                 cmd->se_cmd.t_data_sg = &isert_cmd->sg;
1085                 cmd->se_cmd.t_data_nents = 1;
1086                 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc),
1087                                 imm_data_len);
1088                 isert_dbg("Transfer Immediate imm_data_len: %d\n",
1089                           imm_data_len);
1090         }
1091
1092         cmd->write_data_done += imm_data_len;
1093
1094         if (cmd->write_data_done == cmd->se_cmd.data_length) {
1095                 spin_lock_bh(&cmd->istate_lock);
1096                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1097                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1098                 spin_unlock_bh(&cmd->istate_lock);
1099         }
1100
1101 sequence_cmd:
1102         rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1103
1104         if (!rc && dump_payload == false && unsol_data)
1105                 iscsit_set_unsolicited_dataout(cmd);
1106         else if (dump_payload && imm_data)
1107                 target_put_sess_cmd(&cmd->se_cmd);
1108
1109         return 0;
1110 }
1111
1112 static int
1113 isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1114                            struct iser_rx_desc *rx_desc, unsigned char *buf)
1115 {
1116         struct scatterlist *sg_start;
1117         struct iscsi_conn *conn = isert_conn->conn;
1118         struct iscsi_cmd *cmd = NULL;
1119         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1120         u32 unsol_data_len = ntoh24(hdr->dlength);
1121         int rc, sg_nents, sg_off, page_off;
1122
1123         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1124         if (rc < 0)
1125                 return rc;
1126         else if (!cmd)
1127                 return 0;
1128         /*
1129          * FIXME: Unexpected unsolicited_data out
1130          */
1131         if (!cmd->unsolicited_data) {
1132                 isert_err("Received unexpected solicited data payload\n");
1133                 dump_stack();
1134                 return -1;
1135         }
1136
1137         isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
1138                   "write_data_done: %u, data_length: %u\n",
1139                   unsol_data_len,  cmd->write_data_done,
1140                   cmd->se_cmd.data_length);
1141
1142         sg_off = cmd->write_data_done / PAGE_SIZE;
1143         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1144         sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1145         page_off = cmd->write_data_done % PAGE_SIZE;
1146         /*
1147          * FIXME: Non page-aligned unsolicited_data out
1148          */
1149         if (page_off) {
1150                 isert_err("unexpected non-page aligned data payload\n");
1151                 dump_stack();
1152                 return -1;
1153         }
1154         isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
1155                   "sg_nents: %u from %p %u\n", sg_start, sg_off,
1156                   sg_nents, isert_get_data(rx_desc), unsol_data_len);
1157
1158         sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc),
1159                             unsol_data_len);
1160
1161         rc = iscsit_check_dataout_payload(cmd, hdr, false);
1162         if (rc < 0)
1163                 return rc;
1164
1165         /*
1166          * multiple data-outs on the same command can arrive -
1167          * so post the buffer before hand
1168          */
1169         return isert_post_recv(isert_conn, rx_desc);
1170 }
1171
1172 static int
1173 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1174                      struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1175                      unsigned char *buf)
1176 {
1177         struct iscsi_conn *conn = isert_conn->conn;
1178         struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1179         int rc;
1180
1181         rc = iscsit_setup_nop_out(conn, cmd, hdr);
1182         if (rc < 0)
1183                 return rc;
1184         /*
1185          * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1186          */
1187
1188         return iscsit_process_nop_out(conn, cmd, hdr);
1189 }
1190
1191 static int
1192 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1193                       struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1194                       struct iscsi_text *hdr)
1195 {
1196         struct iscsi_conn *conn = isert_conn->conn;
1197         u32 payload_length = ntoh24(hdr->dlength);
1198         int rc;
1199         unsigned char *text_in = NULL;
1200
1201         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1202         if (rc < 0)
1203                 return rc;
1204
1205         if (payload_length) {
1206                 text_in = kzalloc(payload_length, GFP_KERNEL);
1207                 if (!text_in)
1208                         return -ENOMEM;
1209         }
1210         cmd->text_in_ptr = text_in;
1211
1212         memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length);
1213
1214         return iscsit_process_text_cmd(conn, cmd, hdr);
1215 }
1216
1217 static int
1218 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1219                 uint32_t read_stag, uint64_t read_va,
1220                 uint32_t write_stag, uint64_t write_va)
1221 {
1222         struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc);
1223         struct iscsi_conn *conn = isert_conn->conn;
1224         struct iscsi_cmd *cmd;
1225         struct isert_cmd *isert_cmd;
1226         int ret = -EINVAL;
1227         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1228
1229         if (conn->sess->sess_ops->SessionType &&
1230            (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
1231                 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1232                           " ignoring\n", opcode);
1233                 return 0;
1234         }
1235
1236         switch (opcode) {
1237         case ISCSI_OP_SCSI_CMD:
1238                 cmd = isert_allocate_cmd(conn, rx_desc);
1239                 if (!cmd)
1240                         break;
1241
1242                 isert_cmd = iscsit_priv_cmd(cmd);
1243                 isert_cmd->read_stag = read_stag;
1244                 isert_cmd->read_va = read_va;
1245                 isert_cmd->write_stag = write_stag;
1246                 isert_cmd->write_va = write_va;
1247                 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
1248
1249                 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
1250                                         rx_desc, (unsigned char *)hdr);
1251                 break;
1252         case ISCSI_OP_NOOP_OUT:
1253                 cmd = isert_allocate_cmd(conn, rx_desc);
1254                 if (!cmd)
1255                         break;
1256
1257                 isert_cmd = iscsit_priv_cmd(cmd);
1258                 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
1259                                            rx_desc, (unsigned char *)hdr);
1260                 break;
1261         case ISCSI_OP_SCSI_DATA_OUT:
1262                 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1263                                                 (unsigned char *)hdr);
1264                 break;
1265         case ISCSI_OP_SCSI_TMFUNC:
1266                 cmd = isert_allocate_cmd(conn, rx_desc);
1267                 if (!cmd)
1268                         break;
1269
1270                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1271                                                 (unsigned char *)hdr);
1272                 break;
1273         case ISCSI_OP_LOGOUT:
1274                 cmd = isert_allocate_cmd(conn, rx_desc);
1275                 if (!cmd)
1276                         break;
1277
1278                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1279                 break;
1280         case ISCSI_OP_TEXT:
1281                 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
1282                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1283                 else
1284                         cmd = isert_allocate_cmd(conn, rx_desc);
1285
1286                 if (!cmd)
1287                         break;
1288
1289                 isert_cmd = iscsit_priv_cmd(cmd);
1290                 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
1291                                             rx_desc, (struct iscsi_text *)hdr);
1292                 break;
1293         default:
1294                 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1295                 dump_stack();
1296                 break;
1297         }
1298
1299         return ret;
1300 }
1301
1302 static void
1303 isert_print_wc(struct ib_wc *wc, const char *type)
1304 {
1305         if (wc->status != IB_WC_WR_FLUSH_ERR)
1306                 isert_err("%s failure: %s (%d) vend_err %x\n", type,
1307                           ib_wc_status_msg(wc->status), wc->status,
1308                           wc->vendor_err);
1309         else
1310                 isert_dbg("%s failure: %s (%d)\n", type,
1311                           ib_wc_status_msg(wc->status), wc->status);
1312 }
1313
1314 static void
1315 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1316 {
1317         struct isert_conn *isert_conn = wc->qp->qp_context;
1318         struct ib_device *ib_dev = isert_conn->cm_id->device;
1319         struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe);
1320         struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc);
1321         struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc);
1322         uint64_t read_va = 0, write_va = 0;
1323         uint32_t read_stag = 0, write_stag = 0;
1324
1325         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1326                 isert_print_wc(wc, "recv");
1327                 if (wc->status != IB_WC_WR_FLUSH_ERR)
1328                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1329                 return;
1330         }
1331
1332         rx_desc->in_use = true;
1333
1334         ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr,
1335                         ISER_RX_SIZE, DMA_FROM_DEVICE);
1336
1337         isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1338                  rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags,
1339                  (int)(wc->byte_len - ISER_HEADERS_LEN));
1340
1341         switch (iser_ctrl->flags & 0xF0) {
1342         case ISCSI_CTRL:
1343                 if (iser_ctrl->flags & ISER_RSV) {
1344                         read_stag = be32_to_cpu(iser_ctrl->read_stag);
1345                         read_va = be64_to_cpu(iser_ctrl->read_va);
1346                         isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
1347                                   read_stag, (unsigned long long)read_va);
1348                 }
1349                 if (iser_ctrl->flags & ISER_WSV) {
1350                         write_stag = be32_to_cpu(iser_ctrl->write_stag);
1351                         write_va = be64_to_cpu(iser_ctrl->write_va);
1352                         isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
1353                                   write_stag, (unsigned long long)write_va);
1354                 }
1355
1356                 isert_dbg("ISER ISCSI_CTRL PDU\n");
1357                 break;
1358         case ISER_HELLO:
1359                 isert_err("iSER Hello message\n");
1360                 break;
1361         default:
1362                 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
1363                 break;
1364         }
1365
1366         isert_rx_opcode(isert_conn, rx_desc,
1367                         read_stag, read_va, write_stag, write_va);
1368
1369         ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr,
1370                         ISER_RX_SIZE, DMA_FROM_DEVICE);
1371 }
1372
1373 static void
1374 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1375 {
1376         struct isert_conn *isert_conn = wc->qp->qp_context;
1377         struct ib_device *ib_dev = isert_conn->device->ib_device;
1378
1379         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1380                 isert_print_wc(wc, "login recv");
1381                 return;
1382         }
1383
1384         ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr,
1385                         ISER_RX_SIZE, DMA_FROM_DEVICE);
1386
1387         isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN;
1388
1389         if (isert_conn->conn) {
1390                 struct iscsi_login *login = isert_conn->conn->conn_login;
1391
1392                 if (login && !login->first_request)
1393                         isert_rx_login_req(isert_conn);
1394         }
1395
1396         mutex_lock(&isert_conn->mutex);
1397         complete(&isert_conn->login_req_comp);
1398         mutex_unlock(&isert_conn->mutex);
1399
1400         ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr,
1401                                 ISER_RX_SIZE, DMA_FROM_DEVICE);
1402 }
1403
1404 static void
1405 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
1406 {
1407         struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
1408         enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
1409
1410         if (!cmd->rw.nr_ops)
1411                 return;
1412
1413         if (isert_prot_cmd(conn, se_cmd)) {
1414                 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp,
1415                                 conn->cm_id->port_num, se_cmd->t_data_sg,
1416                                 se_cmd->t_data_nents, se_cmd->t_prot_sg,
1417                                 se_cmd->t_prot_nents, dir);
1418         } else {
1419                 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num,
1420                                 se_cmd->t_data_sg, se_cmd->t_data_nents, dir);
1421         }
1422
1423         cmd->rw.nr_ops = 0;
1424 }
1425
1426 static void
1427 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
1428 {
1429         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1430         struct isert_conn *isert_conn = isert_cmd->conn;
1431         struct iscsi_conn *conn = isert_conn->conn;
1432         struct iscsi_text_rsp *hdr;
1433
1434         isert_dbg("Cmd %p\n", isert_cmd);
1435
1436         switch (cmd->iscsi_opcode) {
1437         case ISCSI_OP_SCSI_CMD:
1438                 spin_lock_bh(&conn->cmd_lock);
1439                 if (!list_empty(&cmd->i_conn_node))
1440                         list_del_init(&cmd->i_conn_node);
1441                 spin_unlock_bh(&conn->cmd_lock);
1442
1443                 if (cmd->data_direction == DMA_TO_DEVICE) {
1444                         iscsit_stop_dataout_timer(cmd);
1445                         /*
1446                          * Check for special case during comp_err where
1447                          * WRITE_PENDING has been handed off from core,
1448                          * but requires an extra target_put_sess_cmd()
1449                          * before transport_generic_free_cmd() below.
1450                          */
1451                         if (comp_err &&
1452                             cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1453                                 struct se_cmd *se_cmd = &cmd->se_cmd;
1454
1455                                 target_put_sess_cmd(se_cmd);
1456                         }
1457                 }
1458
1459                 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1460                 transport_generic_free_cmd(&cmd->se_cmd, 0);
1461                 break;
1462         case ISCSI_OP_SCSI_TMFUNC:
1463                 spin_lock_bh(&conn->cmd_lock);
1464                 if (!list_empty(&cmd->i_conn_node))
1465                         list_del_init(&cmd->i_conn_node);
1466                 spin_unlock_bh(&conn->cmd_lock);
1467
1468                 transport_generic_free_cmd(&cmd->se_cmd, 0);
1469                 break;
1470         case ISCSI_OP_REJECT:
1471         case ISCSI_OP_NOOP_OUT:
1472         case ISCSI_OP_TEXT:
1473                 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1474                 /* If the continue bit is on, keep the command alive */
1475                 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
1476                         break;
1477
1478                 spin_lock_bh(&conn->cmd_lock);
1479                 if (!list_empty(&cmd->i_conn_node))
1480                         list_del_init(&cmd->i_conn_node);
1481                 spin_unlock_bh(&conn->cmd_lock);
1482
1483                 /*
1484                  * Handle special case for REJECT when iscsi_add_reject*() has
1485                  * overwritten the original iscsi_opcode assignment, and the
1486                  * associated cmd->se_cmd needs to be released.
1487                  */
1488                 if (cmd->se_cmd.se_tfo != NULL) {
1489                         isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
1490                                  cmd->iscsi_opcode);
1491                         transport_generic_free_cmd(&cmd->se_cmd, 0);
1492                         break;
1493                 }
1494                 fallthrough;
1495         default:
1496                 iscsit_release_cmd(cmd);
1497                 break;
1498         }
1499 }
1500
1501 static void
1502 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1503 {
1504         if (tx_desc->dma_addr != 0) {
1505                 isert_dbg("unmap single for tx_desc->dma_addr\n");
1506                 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1507                                     ISER_HEADERS_LEN, DMA_TO_DEVICE);
1508                 tx_desc->dma_addr = 0;
1509         }
1510 }
1511
1512 static void
1513 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1514                      struct ib_device *ib_dev, bool comp_err)
1515 {
1516         if (isert_cmd->pdu_buf_dma != 0) {
1517                 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
1518                 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
1519                                     isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
1520                 isert_cmd->pdu_buf_dma = 0;
1521         }
1522
1523         isert_unmap_tx_desc(tx_desc, ib_dev);
1524         isert_put_cmd(isert_cmd, comp_err);
1525 }
1526
1527 static int
1528 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
1529 {
1530         struct ib_mr_status mr_status;
1531         int ret;
1532
1533         ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
1534         if (ret) {
1535                 isert_err("ib_check_mr_status failed, ret %d\n", ret);
1536                 goto fail_mr_status;
1537         }
1538
1539         if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
1540                 u64 sec_offset_err;
1541                 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
1542
1543                 switch (mr_status.sig_err.err_type) {
1544                 case IB_SIG_BAD_GUARD:
1545                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
1546                         break;
1547                 case IB_SIG_BAD_REFTAG:
1548                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
1549                         break;
1550                 case IB_SIG_BAD_APPTAG:
1551                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
1552                         break;
1553                 }
1554                 sec_offset_err = mr_status.sig_err.sig_err_offset;
1555                 do_div(sec_offset_err, block_size);
1556                 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;
1557
1558                 isert_err("PI error found type %d at sector 0x%llx "
1559                           "expected 0x%x vs actual 0x%x\n",
1560                           mr_status.sig_err.err_type,
1561                           (unsigned long long)se_cmd->bad_sector,
1562                           mr_status.sig_err.expected,
1563                           mr_status.sig_err.actual);
1564                 ret = 1;
1565         }
1566
1567 fail_mr_status:
1568         return ret;
1569 }
1570
1571 static void
1572 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
1573 {
1574         struct isert_conn *isert_conn = wc->qp->qp_context;
1575         struct isert_device *device = isert_conn->device;
1576         struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1577         struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1578         struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd;
1579         int ret = 0;
1580
1581         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1582                 isert_print_wc(wc, "rdma write");
1583                 if (wc->status != IB_WC_WR_FLUSH_ERR)
1584                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1585                 isert_completion_put(desc, isert_cmd, device->ib_device, true);
1586                 return;
1587         }
1588
1589         isert_dbg("Cmd %p\n", isert_cmd);
1590
1591         ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr);
1592         isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1593
1594         if (ret) {
1595                 /*
1596                  * transport_generic_request_failure() expects to have
1597                  * plus two references to handle queue-full, so re-add
1598                  * one here as target-core will have already dropped
1599                  * it after the first isert_put_datain() callback.
1600                  */
1601                 kref_get(&cmd->cmd_kref);
1602                 transport_generic_request_failure(cmd, cmd->pi_err);
1603         } else {
1604                 /*
1605                  * XXX: isert_put_response() failure is not retried.
1606                  */
1607                 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd);
1608                 if (ret)
1609                         pr_warn_ratelimited("isert_put_response() ret: %d\n", ret);
1610         }
1611 }
1612
1613 static void
1614 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1615 {
1616         struct isert_conn *isert_conn = wc->qp->qp_context;
1617         struct isert_device *device = isert_conn->device;
1618         struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1619         struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1620         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1621         struct se_cmd *se_cmd = &cmd->se_cmd;
1622         int ret = 0;
1623
1624         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1625                 isert_print_wc(wc, "rdma read");
1626                 if (wc->status != IB_WC_WR_FLUSH_ERR)
1627                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1628                 isert_completion_put(desc, isert_cmd, device->ib_device, true);
1629                 return;
1630         }
1631
1632         isert_dbg("Cmd %p\n", isert_cmd);
1633
1634         iscsit_stop_dataout_timer(cmd);
1635
1636         if (isert_prot_cmd(isert_conn, se_cmd))
1637                 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr);
1638         isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1639         cmd->write_data_done = 0;
1640
1641         isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
1642         spin_lock_bh(&cmd->istate_lock);
1643         cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1644         cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1645         spin_unlock_bh(&cmd->istate_lock);
1646
1647         /*
1648          * transport_generic_request_failure() will drop the extra
1649          * se_cmd->cmd_kref reference after T10-PI error, and handle
1650          * any non-zero ->queue_status() callback error retries.
1651          */
1652         if (ret)
1653                 transport_generic_request_failure(se_cmd, se_cmd->pi_err);
1654         else
1655                 target_execute_cmd(se_cmd);
1656 }
1657
1658 static void
1659 isert_do_control_comp(struct work_struct *work)
1660 {
1661         struct isert_cmd *isert_cmd = container_of(work,
1662                         struct isert_cmd, comp_work);
1663         struct isert_conn *isert_conn = isert_cmd->conn;
1664         struct ib_device *ib_dev = isert_conn->cm_id->device;
1665         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1666
1667         isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
1668
1669         switch (cmd->i_state) {
1670         case ISTATE_SEND_TASKMGTRSP:
1671                 iscsit_tmr_post_handler(cmd, cmd->conn);
1672                 fallthrough;
1673         case ISTATE_SEND_REJECT:
1674         case ISTATE_SEND_TEXTRSP:
1675                 cmd->i_state = ISTATE_SENT_STATUS;
1676                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
1677                                      ib_dev, false);
1678                 break;
1679         case ISTATE_SEND_LOGOUTRSP:
1680                 iscsit_logout_post_handler(cmd, cmd->conn);
1681                 break;
1682         default:
1683                 isert_err("Unknown i_state %d\n", cmd->i_state);
1684                 dump_stack();
1685                 break;
1686         }
1687 }
1688
1689 static void
1690 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
1691 {
1692         struct isert_conn *isert_conn = wc->qp->qp_context;
1693         struct ib_device *ib_dev = isert_conn->cm_id->device;
1694         struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1695
1696         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1697                 isert_print_wc(wc, "login send");
1698                 if (wc->status != IB_WC_WR_FLUSH_ERR)
1699                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1700         }
1701
1702         isert_unmap_tx_desc(tx_desc, ib_dev);
1703 }
1704
1705 static void
1706 isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
1707 {
1708         struct isert_conn *isert_conn = wc->qp->qp_context;
1709         struct ib_device *ib_dev = isert_conn->cm_id->device;
1710         struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1711         struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc);
1712
1713         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1714                 isert_print_wc(wc, "send");
1715                 if (wc->status != IB_WC_WR_FLUSH_ERR)
1716                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1717                 isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
1718                 return;
1719         }
1720
1721         isert_dbg("Cmd %p\n", isert_cmd);
1722
1723         switch (isert_cmd->iscsi_cmd->i_state) {
1724         case ISTATE_SEND_TASKMGTRSP:
1725         case ISTATE_SEND_LOGOUTRSP:
1726         case ISTATE_SEND_REJECT:
1727         case ISTATE_SEND_TEXTRSP:
1728                 isert_unmap_tx_desc(tx_desc, ib_dev);
1729
1730                 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1731                 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1732                 return;
1733         default:
1734                 isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS;
1735                 isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
1736                 break;
1737         }
1738 }
1739
1740 static int
1741 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1742 {
1743         int ret;
1744
1745         ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
1746         if (ret)
1747                 return ret;
1748
1749         ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL);
1750         if (ret) {
1751                 isert_err("ib_post_send failed with %d\n", ret);
1752                 return ret;
1753         }
1754         return ret;
1755 }
1756
1757 static int
1758 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1759 {
1760         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1761         struct isert_conn *isert_conn = conn->context;
1762         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1763         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1764                                 &isert_cmd->tx_desc.iscsi_header;
1765
1766         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1767         iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1768         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1769         /*
1770          * Attach SENSE DATA payload to iSCSI Response PDU
1771          */
1772         if (cmd->se_cmd.sense_buffer &&
1773             ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1774             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1775                 struct isert_device *device = isert_conn->device;
1776                 struct ib_device *ib_dev = device->ib_device;
1777                 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1778                 u32 padding, pdu_len;
1779
1780                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1781                                    cmd->sense_buffer);
1782                 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1783
1784                 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1785                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1786                 pdu_len = cmd->se_cmd.scsi_sense_length + padding;
1787
1788                 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1789                                 (void *)cmd->sense_buffer, pdu_len,
1790                                 DMA_TO_DEVICE);
1791                 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1792                         return -ENOMEM;
1793
1794                 isert_cmd->pdu_buf_len = pdu_len;
1795                 tx_dsg->addr    = isert_cmd->pdu_buf_dma;
1796                 tx_dsg->length  = pdu_len;
1797                 tx_dsg->lkey    = device->pd->local_dma_lkey;
1798                 isert_cmd->tx_desc.num_sge = 2;
1799         }
1800
1801         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1802
1803         isert_dbg("Posting SCSI Response\n");
1804
1805         return isert_post_response(isert_conn, isert_cmd);
1806 }
1807
1808 static void
1809 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1810 {
1811         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1812         struct isert_conn *isert_conn = conn->context;
1813
1814         spin_lock_bh(&conn->cmd_lock);
1815         if (!list_empty(&cmd->i_conn_node))
1816                 list_del_init(&cmd->i_conn_node);
1817         spin_unlock_bh(&conn->cmd_lock);
1818
1819         if (cmd->data_direction == DMA_TO_DEVICE)
1820                 iscsit_stop_dataout_timer(cmd);
1821         isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1822 }
1823
1824 static enum target_prot_op
1825 isert_get_sup_prot_ops(struct iscsi_conn *conn)
1826 {
1827         struct isert_conn *isert_conn = conn->context;
1828         struct isert_device *device = isert_conn->device;
1829
1830         if (conn->tpg->tpg_attrib.t10_pi) {
1831                 if (device->pi_capable) {
1832                         isert_info("conn %p PI offload enabled\n", isert_conn);
1833                         isert_conn->pi_support = true;
1834                         return TARGET_PROT_ALL;
1835                 }
1836         }
1837
1838         isert_info("conn %p PI offload disabled\n", isert_conn);
1839         isert_conn->pi_support = false;
1840
1841         return TARGET_PROT_NORMAL;
1842 }
1843
1844 static int
1845 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1846                 bool nopout_response)
1847 {
1848         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1849         struct isert_conn *isert_conn = conn->context;
1850         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1851
1852         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1853         iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1854                                &isert_cmd->tx_desc.iscsi_header,
1855                                nopout_response);
1856         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1857         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1858
1859         isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
1860
1861         return isert_post_response(isert_conn, isert_cmd);
1862 }
1863
1864 static int
1865 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1866 {
1867         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1868         struct isert_conn *isert_conn = conn->context;
1869         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1870
1871         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1872         iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1873                                 &isert_cmd->tx_desc.iscsi_header);
1874         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1875         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1876
1877         isert_dbg("conn %p Posting Logout Response\n", isert_conn);
1878
1879         return isert_post_response(isert_conn, isert_cmd);
1880 }
1881
1882 static int
1883 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1884 {
1885         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1886         struct isert_conn *isert_conn = conn->context;
1887         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1888
1889         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1890         iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1891                                   &isert_cmd->tx_desc.iscsi_header);
1892         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1893         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1894
1895         isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
1896
1897         return isert_post_response(isert_conn, isert_cmd);
1898 }
1899
1900 static int
1901 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1902 {
1903         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1904         struct isert_conn *isert_conn = conn->context;
1905         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1906         struct isert_device *device = isert_conn->device;
1907         struct ib_device *ib_dev = device->ib_device;
1908         struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1909         struct iscsi_reject *hdr =
1910                 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
1911
1912         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1913         iscsit_build_reject(cmd, conn, hdr);
1914         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1915
1916         hton24(hdr->dlength, ISCSI_HDR_LEN);
1917         isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1918                         (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
1919                         DMA_TO_DEVICE);
1920         if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1921                 return -ENOMEM;
1922         isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
1923         tx_dsg->addr    = isert_cmd->pdu_buf_dma;
1924         tx_dsg->length  = ISCSI_HDR_LEN;
1925         tx_dsg->lkey    = device->pd->local_dma_lkey;
1926         isert_cmd->tx_desc.num_sge = 2;
1927
1928         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1929
1930         isert_dbg("conn %p Posting Reject\n", isert_conn);
1931
1932         return isert_post_response(isert_conn, isert_cmd);
1933 }
1934
1935 static int
1936 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1937 {
1938         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1939         struct isert_conn *isert_conn = conn->context;
1940         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1941         struct iscsi_text_rsp *hdr =
1942                 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1943         u32 txt_rsp_len;
1944         int rc;
1945
1946         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1947         rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
1948         if (rc < 0)
1949                 return rc;
1950
1951         txt_rsp_len = rc;
1952         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1953
1954         if (txt_rsp_len) {
1955                 struct isert_device *device = isert_conn->device;
1956                 struct ib_device *ib_dev = device->ib_device;
1957                 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1958                 void *txt_rsp_buf = cmd->buf_ptr;
1959
1960                 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1961                                 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
1962                 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1963                         return -ENOMEM;
1964
1965                 isert_cmd->pdu_buf_len = txt_rsp_len;
1966                 tx_dsg->addr    = isert_cmd->pdu_buf_dma;
1967                 tx_dsg->length  = txt_rsp_len;
1968                 tx_dsg->lkey    = device->pd->local_dma_lkey;
1969                 isert_cmd->tx_desc.num_sge = 2;
1970         }
1971         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1972
1973         isert_dbg("conn %p Text Response\n", isert_conn);
1974
1975         return isert_post_response(isert_conn, isert_cmd);
1976 }
1977
1978 static inline void
1979 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain)
1980 {
1981         domain->sig_type = IB_SIG_TYPE_T10_DIF;
1982         domain->sig.dif.bg_type = IB_T10DIF_CRC;
1983         domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
1984         domain->sig.dif.ref_tag = se_cmd->reftag_seed;
1985         /*
1986          * At the moment we hard code those, but if in the future
1987          * the target core would like to use it, we will take it
1988          * from se_cmd.
1989          */
1990         domain->sig.dif.apptag_check_mask = 0xffff;
1991         domain->sig.dif.app_escape = true;
1992         domain->sig.dif.ref_escape = true;
1993         if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
1994             se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
1995                 domain->sig.dif.ref_remap = true;
1996 };
1997
1998 static int
1999 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
2000 {
2001         memset(sig_attrs, 0, sizeof(*sig_attrs));
2002
2003         switch (se_cmd->prot_op) {
2004         case TARGET_PROT_DIN_INSERT:
2005         case TARGET_PROT_DOUT_STRIP:
2006                 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2007                 isert_set_dif_domain(se_cmd, &sig_attrs->wire);
2008                 break;
2009         case TARGET_PROT_DOUT_INSERT:
2010         case TARGET_PROT_DIN_STRIP:
2011                 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2012                 isert_set_dif_domain(se_cmd, &sig_attrs->mem);
2013                 break;
2014         case TARGET_PROT_DIN_PASS:
2015         case TARGET_PROT_DOUT_PASS:
2016                 isert_set_dif_domain(se_cmd, &sig_attrs->wire);
2017                 isert_set_dif_domain(se_cmd, &sig_attrs->mem);
2018                 break;
2019         default:
2020                 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
2021                 return -EINVAL;
2022         }
2023
2024         if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD)
2025                 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
2026         if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG)
2027                 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
2028         if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG)
2029                 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
2030
2031         return 0;
2032 }
2033
2034 static int
2035 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
2036                 struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
2037 {
2038         struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
2039         enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
2040         u8 port_num = conn->cm_id->port_num;
2041         u64 addr;
2042         u32 rkey, offset;
2043         int ret;
2044
2045         if (cmd->ctx_init_done)
2046                 goto rdma_ctx_post;
2047
2048         if (dir == DMA_FROM_DEVICE) {
2049                 addr = cmd->write_va;
2050                 rkey = cmd->write_stag;
2051                 offset = cmd->iscsi_cmd->write_data_done;
2052         } else {
2053                 addr = cmd->read_va;
2054                 rkey = cmd->read_stag;
2055                 offset = 0;
2056         }
2057
2058         if (isert_prot_cmd(conn, se_cmd)) {
2059                 struct ib_sig_attrs sig_attrs;
2060
2061                 ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
2062                 if (ret)
2063                         return ret;
2064
2065                 WARN_ON_ONCE(offset);
2066                 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num,
2067                                 se_cmd->t_data_sg, se_cmd->t_data_nents,
2068                                 se_cmd->t_prot_sg, se_cmd->t_prot_nents,
2069                                 &sig_attrs, addr, rkey, dir);
2070         } else {
2071                 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num,
2072                                 se_cmd->t_data_sg, se_cmd->t_data_nents,
2073                                 offset, addr, rkey, dir);
2074         }
2075
2076         if (ret < 0) {
2077                 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
2078                 return ret;
2079         }
2080
2081         cmd->ctx_init_done = true;
2082
2083 rdma_ctx_post:
2084         ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
2085         if (ret < 0)
2086                 isert_err("Cmd: %p failed to post RDMA res\n", cmd);
2087         return ret;
2088 }
2089
2090 static int
2091 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2092 {
2093         struct se_cmd *se_cmd = &cmd->se_cmd;
2094         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2095         struct isert_conn *isert_conn = conn->context;
2096         struct ib_cqe *cqe = NULL;
2097         struct ib_send_wr *chain_wr = NULL;
2098         int rc;
2099
2100         isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
2101                  isert_cmd, se_cmd->data_length);
2102
2103         if (isert_prot_cmd(isert_conn, se_cmd)) {
2104                 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done;
2105                 cqe = &isert_cmd->tx_desc.tx_cqe;
2106         } else {
2107                 /*
2108                  * Build isert_conn->tx_desc for iSCSI response PDU and attach
2109                  */
2110                 isert_create_send_desc(isert_conn, isert_cmd,
2111                                        &isert_cmd->tx_desc);
2112                 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
2113                                      &isert_cmd->tx_desc.iscsi_header);
2114                 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2115                 isert_init_send_wr(isert_conn, isert_cmd,
2116                                    &isert_cmd->tx_desc.send_wr);
2117
2118                 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
2119                 if (rc)
2120                         return rc;
2121
2122                 chain_wr = &isert_cmd->tx_desc.send_wr;
2123         }
2124
2125         rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr);
2126         isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n",
2127                   isert_cmd, rc);
2128         return rc;
2129 }
2130
2131 static int
2132 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
2133 {
2134         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2135         int ret;
2136
2137         isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2138                  isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done);
2139
2140         isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done;
2141         ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context,
2142                                      &isert_cmd->tx_desc.tx_cqe, NULL);
2143
2144         isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n",
2145                  isert_cmd, ret);
2146         return ret;
2147 }
2148
2149 static int
2150 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2151 {
2152         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2153         int ret = 0;
2154
2155         switch (state) {
2156         case ISTATE_REMOVE:
2157                 spin_lock_bh(&conn->cmd_lock);
2158                 list_del_init(&cmd->i_conn_node);
2159                 spin_unlock_bh(&conn->cmd_lock);
2160                 isert_put_cmd(isert_cmd, true);
2161                 break;
2162         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2163                 ret = isert_put_nopin(cmd, conn, false);
2164                 break;
2165         default:
2166                 isert_err("Unknown immediate state: 0x%02x\n", state);
2167                 ret = -EINVAL;
2168                 break;
2169         }
2170
2171         return ret;
2172 }
2173
2174 static int
2175 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2176 {
2177         struct isert_conn *isert_conn = conn->context;
2178         int ret;
2179
2180         switch (state) {
2181         case ISTATE_SEND_LOGOUTRSP:
2182                 ret = isert_put_logout_rsp(cmd, conn);
2183                 if (!ret)
2184                         isert_conn->logout_posted = true;
2185                 break;
2186         case ISTATE_SEND_NOPIN:
2187                 ret = isert_put_nopin(cmd, conn, true);
2188                 break;
2189         case ISTATE_SEND_TASKMGTRSP:
2190                 ret = isert_put_tm_rsp(cmd, conn);
2191                 break;
2192         case ISTATE_SEND_REJECT:
2193                 ret = isert_put_reject(cmd, conn);
2194                 break;
2195         case ISTATE_SEND_TEXTRSP:
2196                 ret = isert_put_text_rsp(cmd, conn);
2197                 break;
2198         case ISTATE_SEND_STATUS:
2199                 /*
2200                  * Special case for sending non GOOD SCSI status from TX thread
2201                  * context during pre se_cmd excecution failure.
2202                  */
2203                 ret = isert_put_response(conn, cmd);
2204                 break;
2205         default:
2206                 isert_err("Unknown response state: 0x%02x\n", state);
2207                 ret = -EINVAL;
2208                 break;
2209         }
2210
2211         return ret;
2212 }
2213
2214 struct rdma_cm_id *
2215 isert_setup_id(struct isert_np *isert_np)
2216 {
2217         struct iscsi_np *np = isert_np->np;
2218         struct rdma_cm_id *id;
2219         struct sockaddr *sa;
2220         int ret;
2221
2222         sa = (struct sockaddr *)&np->np_sockaddr;
2223         isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
2224
2225         id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
2226                             RDMA_PS_TCP, IB_QPT_RC);
2227         if (IS_ERR(id)) {
2228                 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
2229                 ret = PTR_ERR(id);
2230                 goto out;
2231         }
2232         isert_dbg("id %p context %p\n", id, id->context);
2233
2234         ret = rdma_bind_addr(id, sa);
2235         if (ret) {
2236                 isert_err("rdma_bind_addr() failed: %d\n", ret);
2237                 goto out_id;
2238         }
2239
2240         ret = rdma_listen(id, 0);
2241         if (ret) {
2242                 isert_err("rdma_listen() failed: %d\n", ret);
2243                 goto out_id;
2244         }
2245
2246         return id;
2247 out_id:
2248         rdma_destroy_id(id);
2249 out:
2250         return ERR_PTR(ret);
2251 }
2252
2253 static int
2254 isert_setup_np(struct iscsi_np *np,
2255                struct sockaddr_storage *ksockaddr)
2256 {
2257         struct isert_np *isert_np;
2258         struct rdma_cm_id *isert_lid;
2259         int ret;
2260
2261         isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2262         if (!isert_np)
2263                 return -ENOMEM;
2264
2265         sema_init(&isert_np->sem, 0);
2266         mutex_init(&isert_np->mutex);
2267         INIT_LIST_HEAD(&isert_np->accepted);
2268         INIT_LIST_HEAD(&isert_np->pending);
2269         isert_np->np = np;
2270
2271         /*
2272          * Setup the np->np_sockaddr from the passed sockaddr setup
2273          * in iscsi_target_configfs.c code..
2274          */
2275         memcpy(&np->np_sockaddr, ksockaddr,
2276                sizeof(struct sockaddr_storage));
2277
2278         isert_lid = isert_setup_id(isert_np);
2279         if (IS_ERR(isert_lid)) {
2280                 ret = PTR_ERR(isert_lid);
2281                 goto out;
2282         }
2283
2284         isert_np->cm_id = isert_lid;
2285         np->np_context = isert_np;
2286
2287         return 0;
2288
2289 out:
2290         kfree(isert_np);
2291
2292         return ret;
2293 }
2294
2295 static int
2296 isert_rdma_accept(struct isert_conn *isert_conn)
2297 {
2298         struct rdma_cm_id *cm_id = isert_conn->cm_id;
2299         struct rdma_conn_param cp;
2300         int ret;
2301         struct iser_cm_hdr rsp_hdr;
2302
2303         memset(&cp, 0, sizeof(struct rdma_conn_param));
2304         cp.initiator_depth = isert_conn->initiator_depth;
2305         cp.retry_count = 7;
2306         cp.rnr_retry_count = 7;
2307
2308         memset(&rsp_hdr, 0, sizeof(rsp_hdr));
2309         rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
2310         if (!isert_conn->snd_w_inv)
2311                 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
2312         cp.private_data = (void *)&rsp_hdr;
2313         cp.private_data_len = sizeof(rsp_hdr);
2314
2315         ret = rdma_accept(cm_id, &cp);
2316         if (ret) {
2317                 isert_err("rdma_accept() failed with: %d\n", ret);
2318                 return ret;
2319         }
2320
2321         return 0;
2322 }
2323
2324 static int
2325 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2326 {
2327         struct isert_conn *isert_conn = conn->context;
2328         int ret;
2329
2330         isert_info("before login_req comp conn: %p\n", isert_conn);
2331         ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
2332         if (ret) {
2333                 isert_err("isert_conn %p interrupted before got login req\n",
2334                           isert_conn);
2335                 return ret;
2336         }
2337         reinit_completion(&isert_conn->login_req_comp);
2338
2339         /*
2340          * For login requests after the first PDU, isert_rx_login_req() will
2341          * kick schedule_delayed_work(&conn->login_work) as the packet is
2342          * received, which turns this callback from iscsi_target_do_login_rx()
2343          * into a NOP.
2344          */
2345         if (!login->first_request)
2346                 return 0;
2347
2348         isert_rx_login_req(isert_conn);
2349
2350         isert_info("before login_comp conn: %p\n", conn);
2351         ret = wait_for_completion_interruptible(&isert_conn->login_comp);
2352         if (ret)
2353                 return ret;
2354
2355         isert_info("processing login->req: %p\n", login->req);
2356
2357         return 0;
2358 }
2359
2360 static void
2361 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2362                     struct isert_conn *isert_conn)
2363 {
2364         struct rdma_cm_id *cm_id = isert_conn->cm_id;
2365         struct rdma_route *cm_route = &cm_id->route;
2366
2367         conn->login_family = np->np_sockaddr.ss_family;
2368
2369         conn->login_sockaddr = cm_route->addr.dst_addr;
2370         conn->local_sockaddr = cm_route->addr.src_addr;
2371 }
2372
2373 static int
2374 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2375 {
2376         struct isert_np *isert_np = np->np_context;
2377         struct isert_conn *isert_conn;
2378         int ret;
2379
2380 accept_wait:
2381         ret = down_interruptible(&isert_np->sem);
2382         if (ret)
2383                 return -ENODEV;
2384
2385         spin_lock_bh(&np->np_thread_lock);
2386         if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
2387                 spin_unlock_bh(&np->np_thread_lock);
2388                 isert_dbg("np_thread_state %d\n",
2389                          np->np_thread_state);
2390                 /**
2391                  * No point in stalling here when np_thread
2392                  * is in state RESET/SHUTDOWN/EXIT - bail
2393                  **/
2394                 return -ENODEV;
2395         }
2396         spin_unlock_bh(&np->np_thread_lock);
2397
2398         mutex_lock(&isert_np->mutex);
2399         if (list_empty(&isert_np->pending)) {
2400                 mutex_unlock(&isert_np->mutex);
2401                 goto accept_wait;
2402         }
2403         isert_conn = list_first_entry(&isert_np->pending,
2404                         struct isert_conn, node);
2405         list_del_init(&isert_conn->node);
2406         mutex_unlock(&isert_np->mutex);
2407
2408         conn->context = isert_conn;
2409         isert_conn->conn = conn;
2410         isert_conn->state = ISER_CONN_BOUND;
2411
2412         isert_set_conn_info(np, conn, isert_conn);
2413
2414         isert_dbg("Processing isert_conn: %p\n", isert_conn);
2415
2416         return 0;
2417 }
2418
2419 static void
2420 isert_free_np(struct iscsi_np *np)
2421 {
2422         struct isert_np *isert_np = np->np_context;
2423         struct isert_conn *isert_conn, *n;
2424
2425         if (isert_np->cm_id)
2426                 rdma_destroy_id(isert_np->cm_id);
2427
2428         /*
2429          * FIXME: At this point we don't have a good way to insure
2430          * that at this point we don't have hanging connections that
2431          * completed RDMA establishment but didn't start iscsi login
2432          * process. So work-around this by cleaning up what ever piled
2433          * up in accepted and pending lists.
2434          */
2435         mutex_lock(&isert_np->mutex);
2436         if (!list_empty(&isert_np->pending)) {
2437                 isert_info("Still have isert pending connections\n");
2438                 list_for_each_entry_safe(isert_conn, n,
2439                                          &isert_np->pending,
2440                                          node) {
2441                         isert_info("cleaning isert_conn %p state (%d)\n",
2442                                    isert_conn, isert_conn->state);
2443                         isert_connect_release(isert_conn);
2444                 }
2445         }
2446
2447         if (!list_empty(&isert_np->accepted)) {
2448                 isert_info("Still have isert accepted connections\n");
2449                 list_for_each_entry_safe(isert_conn, n,
2450                                          &isert_np->accepted,
2451                                          node) {
2452                         isert_info("cleaning isert_conn %p state (%d)\n",
2453                                    isert_conn, isert_conn->state);
2454                         isert_connect_release(isert_conn);
2455                 }
2456         }
2457         mutex_unlock(&isert_np->mutex);
2458
2459         np->np_context = NULL;
2460         kfree(isert_np);
2461 }
2462
2463 static void isert_release_work(struct work_struct *work)
2464 {
2465         struct isert_conn *isert_conn = container_of(work,
2466                                                      struct isert_conn,
2467                                                      release_work);
2468
2469         isert_info("Starting release conn %p\n", isert_conn);
2470
2471         mutex_lock(&isert_conn->mutex);
2472         isert_conn->state = ISER_CONN_DOWN;
2473         mutex_unlock(&isert_conn->mutex);
2474
2475         isert_info("Destroying conn %p\n", isert_conn);
2476         isert_put_conn(isert_conn);
2477 }
2478
2479 static void
2480 isert_wait4logout(struct isert_conn *isert_conn)
2481 {
2482         struct iscsi_conn *conn = isert_conn->conn;
2483
2484         isert_info("conn %p\n", isert_conn);
2485
2486         if (isert_conn->logout_posted) {
2487                 isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
2488                 wait_for_completion_timeout(&conn->conn_logout_comp,
2489                                             SECONDS_FOR_LOGOUT_COMP * HZ);
2490         }
2491 }
2492
2493 static void
2494 isert_wait4cmds(struct iscsi_conn *conn)
2495 {
2496         isert_info("iscsi_conn %p\n", conn);
2497
2498         if (conn->sess) {
2499                 target_sess_cmd_list_set_waiting(conn->sess->se_sess);
2500                 target_wait_for_sess_cmds(conn->sess->se_sess);
2501         }
2502 }
2503
2504 /**
2505  * isert_put_unsol_pending_cmds() - Drop commands waiting for
2506  *     unsolicitate dataout
2507  * @conn:    iscsi connection
2508  *
2509  * We might still have commands that are waiting for unsolicited
2510  * dataouts messages. We must put the extra reference on those
2511  * before blocking on the target_wait_for_session_cmds
2512  */
2513 static void
2514 isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
2515 {
2516         struct iscsi_cmd *cmd, *tmp;
2517         static LIST_HEAD(drop_cmd_list);
2518
2519         spin_lock_bh(&conn->cmd_lock);
2520         list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
2521                 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
2522                     (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
2523                     (cmd->write_data_done < cmd->se_cmd.data_length))
2524                         list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
2525         }
2526         spin_unlock_bh(&conn->cmd_lock);
2527
2528         list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
2529                 list_del_init(&cmd->i_conn_node);
2530                 if (cmd->i_state != ISTATE_REMOVE) {
2531                         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2532
2533                         isert_info("conn %p dropping cmd %p\n", conn, cmd);
2534                         isert_put_cmd(isert_cmd, true);
2535                 }
2536         }
2537 }
2538
2539 static void isert_wait_conn(struct iscsi_conn *conn)
2540 {
2541         struct isert_conn *isert_conn = conn->context;
2542
2543         isert_info("Starting conn %p\n", isert_conn);
2544
2545         mutex_lock(&isert_conn->mutex);
2546         isert_conn_terminate(isert_conn);
2547         mutex_unlock(&isert_conn->mutex);
2548
2549         ib_drain_qp(isert_conn->qp);
2550         isert_put_unsol_pending_cmds(conn);
2551         isert_wait4cmds(conn);
2552         isert_wait4logout(isert_conn);
2553
2554         queue_work(isert_release_wq, &isert_conn->release_work);
2555 }
2556
2557 static void isert_free_conn(struct iscsi_conn *conn)
2558 {
2559         struct isert_conn *isert_conn = conn->context;
2560
2561         ib_drain_qp(isert_conn->qp);
2562         isert_put_conn(isert_conn);
2563 }
2564
2565 static void isert_get_rx_pdu(struct iscsi_conn *conn)
2566 {
2567         struct completion comp;
2568
2569         init_completion(&comp);
2570
2571         wait_for_completion_interruptible(&comp);
2572 }
2573
2574 static struct iscsit_transport iser_target_transport = {
2575         .name                   = "IB/iSER",
2576         .transport_type         = ISCSI_INFINIBAND,
2577         .rdma_shutdown          = true,
2578         .priv_size              = sizeof(struct isert_cmd),
2579         .owner                  = THIS_MODULE,
2580         .iscsit_setup_np        = isert_setup_np,
2581         .iscsit_accept_np       = isert_accept_np,
2582         .iscsit_free_np         = isert_free_np,
2583         .iscsit_wait_conn       = isert_wait_conn,
2584         .iscsit_free_conn       = isert_free_conn,
2585         .iscsit_get_login_rx    = isert_get_login_rx,
2586         .iscsit_put_login_tx    = isert_put_login_tx,
2587         .iscsit_immediate_queue = isert_immediate_queue,
2588         .iscsit_response_queue  = isert_response_queue,
2589         .iscsit_get_dataout     = isert_get_dataout,
2590         .iscsit_queue_data_in   = isert_put_datain,
2591         .iscsit_queue_status    = isert_put_response,
2592         .iscsit_aborted_task    = isert_aborted_task,
2593         .iscsit_get_rx_pdu      = isert_get_rx_pdu,
2594         .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
2595 };
2596
2597 static int __init isert_init(void)
2598 {
2599         int ret;
2600
2601         isert_comp_wq = alloc_workqueue("isert_comp_wq",
2602                                         WQ_UNBOUND | WQ_HIGHPRI, 0);
2603         if (!isert_comp_wq) {
2604                 isert_err("Unable to allocate isert_comp_wq\n");
2605                 return -ENOMEM;
2606         }
2607
2608         isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
2609                                         WQ_UNBOUND_MAX_ACTIVE);
2610         if (!isert_release_wq) {
2611                 isert_err("Unable to allocate isert_release_wq\n");
2612                 ret = -ENOMEM;
2613                 goto destroy_comp_wq;
2614         }
2615
2616         iscsit_register_transport(&iser_target_transport);
2617         isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
2618
2619         return 0;
2620
2621 destroy_comp_wq:
2622         destroy_workqueue(isert_comp_wq);
2623
2624         return ret;
2625 }
2626
2627 static void __exit isert_exit(void)
2628 {
2629         flush_scheduled_work();
2630         destroy_workqueue(isert_release_wq);
2631         destroy_workqueue(isert_comp_wq);
2632         iscsit_unregister_transport(&iser_target_transport);
2633         isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
2634 }
2635
2636 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2637 MODULE_AUTHOR("nab@Linux-iSCSI.org");
2638 MODULE_LICENSE("GPL");
2639
2640 module_init(isert_init);
2641 module_exit(isert_exit);