drivers/target/iscsi/cxgbit/cxgbit_target.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016 Chelsio Communications, Inc.
   4  */
   5
   6 #include <linux/workqueue.h>
   7 #include <linux/kthread.h>
   8 #include <linux/sched/signal.h>
   9
  10 #include <asm/unaligned.h>
  11 #include <net/tcp.h>
  12 #include <target/target_core_base.h>
  13 #include <target/target_core_fabric.h>
  14 #include "cxgbit.h"
  15
  16 struct sge_opaque_hdr {
  17         void *dev;
  18         dma_addr_t addr[MAX_SKB_FRAGS + 1];
  19 };
  20
  21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
  22
  23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
  24                     sizeof(struct fw_ofld_tx_data_wr))
  25
  26 static struct sk_buff *
  27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
  28 {
  29         struct sk_buff *skb = NULL;
  30         u8 submode = 0;
  31         int errcode;
  32         static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
  33
  34         if (len) {
  35                 skb = alloc_skb_with_frags(hdr_len, len,
  36                                            0, &errcode,
  37                                            GFP_KERNEL);
  38                 if (!skb)
  39                         return NULL;
  40
  41                 skb_reserve(skb, TX_HDR_LEN);
  42                 skb_reset_transport_header(skb);
  43                 __skb_put(skb, ISCSI_HDR_LEN);
  44                 skb->data_len = len;
  45                 skb->len += len;
  46                 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
  47
  48         } else {
  49                 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
  50
  51                 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
  52                 if (!skb)
  53                         return NULL;
  54
  55                 skb_reserve(skb, TX_HDR_LEN + iso_len);
  56                 skb_reset_transport_header(skb);
  57                 __skb_put(skb, ISCSI_HDR_LEN);
  58         }
  59
  60         submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
  61         cxgbit_skcb_submode(skb) = submode;
  62         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
  63         cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
  64         return skb;
  65 }
  66
  67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
  68 {
  69         return __cxgbit_alloc_skb(csk, len, false);
  70 }
  71
  72 /*
  73  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
  74  * @skb: the packet
  75  *
  76  * Returns true if a packet can be sent as an offload WR with immediate
  77  * data.  We currently use the same limit as for Ethernet packets.
  78  */
  79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
  80 {
  81         int length = skb->len;
  82
  83         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
  84                 length += sizeof(struct fw_ofld_tx_data_wr);
  85
  86         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
  87                 length += sizeof(struct cpl_tx_data_iso);
  88
  89         return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
  90 }
  91
  92 /*
  93  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
  94  * @n: the number of SGL entries
  95  * Calculates the number of flits needed for a scatter/gather list that
  96  * can hold the given number of entries.
  97  */
  98 static inline unsigned int cxgbit_sgl_len(unsigned int n)
  99 {
 100         n--;
 101         return (3 * n) / 2 + (n & 1) + 2;
 102 }
 103
 104 /*
 105  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
 106  * @skb: the packet
 107  *
 108  * Returns the number of flits needed for the given offload packet.
 109  * These packets are already fully constructed and no additional headers
 110  * will be added.
 111  */
 112 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
 113 {
 114         unsigned int flits, cnt;
 115
 116         if (cxgbit_is_ofld_imm(skb))
 117                 return DIV_ROUND_UP(skb->len, 8);
 118         flits = skb_transport_offset(skb) / 8;
 119         cnt = skb_shinfo(skb)->nr_frags;
 120         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 121                 cnt++;
 122         return flits + cxgbit_sgl_len(cnt);
 123 }
 124
 125 #define CXGBIT_ISO_FSLICE 0x1
 126 #define CXGBIT_ISO_LSLICE 0x2
 127 static void
 128 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
 129 {
 130         struct cpl_tx_data_iso *cpl;
 131         unsigned int submode = cxgbit_skcb_submode(skb);
 132         unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
 133         unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
 134
 135         cpl = __skb_push(skb, sizeof(*cpl));
 136
 137         cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
 138                         CPL_TX_DATA_ISO_FIRST_V(fslice) |
 139                         CPL_TX_DATA_ISO_LAST_V(lslice) |
 140                         CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
 141                         CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
 142                         CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
 143                         CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
 144                         CPL_TX_DATA_ISO_SCSI_V(2));
 145
 146         cpl->ahs_len = 0;
 147         cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
 148         cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
 149         cpl->len = htonl(iso_info->len);
 150         cpl->reserved2_seglen_offset = htonl(0);
 151         cpl->datasn_offset = htonl(0);
 152         cpl->buffer_offset = htonl(0);
 153         cpl->reserved3 = 0;
 154
 155         __skb_pull(skb, sizeof(*cpl));
 156 }
 157
 158 static void
 159 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
 160                   u32 len, u32 credits, u32 compl)
 161 {
 162         struct fw_ofld_tx_data_wr *req;
 163         const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
 164         u32 submode = cxgbit_skcb_submode(skb);
 165         u32 wr_ulp_mode = 0;
 166         u32 hdr_size = sizeof(*req);
 167         u32 opcode = FW_OFLD_TX_DATA_WR;
 168         u32 immlen = 0;
 169         u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
 170                     T6_TX_FORCE_F;
 171
 172         if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
 173                 opcode = FW_ISCSI_TX_DATA_WR;
 174                 immlen += sizeof(struct cpl_tx_data_iso);
 175                 hdr_size += sizeof(struct cpl_tx_data_iso);
 176                 submode |= 8;
 177         }
 178
 179         if (cxgbit_is_ofld_imm(skb))
 180                 immlen += dlen;
 181
 182         req = __skb_push(skb, hdr_size);
 183         req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
 184                                         FW_WR_COMPL_V(compl) |
 185                                         FW_WR_IMMDLEN_V(immlen));
 186         req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 187                                         FW_WR_LEN16_V(credits));
 188         req->plen = htonl(len);
 189         wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
 190                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 191
 192         req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
 193                  FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
 194 }
 195
 196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 197 {
 198         kfree_skb(skb);
 199 }
 200
 201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
 202 {
 203         struct sk_buff *skb;
 204
 205         while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
 206                 u32 dlen = skb->len;
 207                 u32 len = skb->len;
 208                 u32 credits_needed;
 209                 u32 compl = 0;
 210                 u32 flowclen16 = 0;
 211                 u32 iso_cpl_len = 0;
 212
 213                 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
 214                         iso_cpl_len = sizeof(struct cpl_tx_data_iso);
 215
 216                 if (cxgbit_is_ofld_imm(skb))
 217                         credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
 218                 else
 219                         credits_needed = DIV_ROUND_UP((8 *
 220                                         cxgbit_calc_tx_flits_ofld(skb)) +
 221                                         iso_cpl_len, 16);
 222
 223                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
 224                         credits_needed += DIV_ROUND_UP(
 225                                 sizeof(struct fw_ofld_tx_data_wr), 16);
 226                 /*
 227                  * Assumes the initial credits is large enough to support
 228                  * fw_flowc_wr plus largest possible first payload
 229                  */
 230
 231                 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
 232                         flowclen16 = cxgbit_send_tx_flowc_wr(csk);
 233                         csk->wr_cred -= flowclen16;
 234                         csk->wr_una_cred += flowclen16;
 235                 }
 236
 237                 if (csk->wr_cred < credits_needed) {
 238                         pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
 239                                  csk, skb->len, skb->data_len,
 240                                  credits_needed, csk->wr_cred);
 241                         break;
 242                 }
 243                 __skb_unlink(skb, &csk->txq);
 244                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
 245                 skb->csum = (__force __wsum)(credits_needed + flowclen16);
 246                 csk->wr_cred -= credits_needed;
 247                 csk->wr_una_cred += credits_needed;
 248
 249                 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 250                          csk, skb->len, skb->data_len, credits_needed,
 251                          csk->wr_cred, csk->wr_una_cred);
 252
 253                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
 254                         len += cxgbit_skcb_tx_extralen(skb);
 255
 256                         if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
 257                             (!before(csk->write_seq,
 258                                      csk->snd_una + csk->snd_win))) {
 259                                 compl = 1;
 260                                 csk->wr_una_cred = 0;
 261                         }
 262
 263                         cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
 264                                           compl);
 265                         csk->snd_nxt += len;
 266
 267                 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
 268                            (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
 269                         struct cpl_close_con_req *req =
 270                                 (struct cpl_close_con_req *)skb->data;
 271                         req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
 272                         csk->wr_una_cred = 0;
 273                 }
 274
 275                 cxgbit_sock_enqueue_wr(csk, skb);
 276                 t4_set_arp_err_handler(skb, csk,
 277                                        cxgbit_arp_failure_skb_discard);
 278
 279                 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
 280                          csk, csk->tid, skb, len);
 281
 282                 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 283         }
 284 }
 285
 286 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
 287 {
 288         struct sk_buff_head backlogq;
 289         struct sk_buff *skb;
 290         void (*fn)(struct cxgbit_sock *, struct sk_buff *);
 291
 292         skb_queue_head_init(&backlogq);
 293
 294         spin_lock_bh(&csk->lock);
 295         while (skb_queue_len(&csk->backlogq)) {
 296                 skb_queue_splice_init(&csk->backlogq, &backlogq);
 297                 spin_unlock_bh(&csk->lock);
 298
 299                 while ((skb = __skb_dequeue(&backlogq))) {
 300                         fn = cxgbit_skcb_rx_backlog_fn(skb);
 301                         fn(csk, skb);
 302                 }
 303
 304                 spin_lock_bh(&csk->lock);
 305         }
 306
 307         csk->lock_owner = false;
 308         spin_unlock_bh(&csk->lock);
 309 }
 310
 311 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 312 {
 313         int ret = 0;
 314
 315         spin_lock_bh(&csk->lock);
 316         csk->lock_owner = true;
 317         spin_unlock_bh(&csk->lock);
 318
 319         if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
 320                      signal_pending(current))) {
 321                 __kfree_skb(skb);
 322                 __skb_queue_purge(&csk->ppodq);
 323                 ret = -1;
 324                 goto unlock;
 325         }
 326
 327         csk->write_seq += skb->len +
 328                           cxgbit_skcb_tx_extralen(skb);
 329
 330         skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
 331         __skb_queue_tail(&csk->txq, skb);
 332         cxgbit_push_tx_frames(csk);
 333
 334 unlock:
 335         cxgbit_unlock_sock(csk);
 336         return ret;
 337 }
 338
 339 static int
 340 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
 341                u32 data_length)
 342 {
 343         u32 i = 0, nr_frags = MAX_SKB_FRAGS;
 344         u32 padding = ((-data_length) & 3);
 345         struct scatterlist *sg;
 346         struct page *page;
 347         unsigned int page_off;
 348
 349         if (padding)
 350                 nr_frags--;
 351
 352         /*
 353          * We know each entry in t_data_sg contains a page.
 354          */
 355         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 356         page_off = (data_offset % PAGE_SIZE);
 357
 358         while (data_length && (i < nr_frags)) {
 359                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 360
 361                 page = sg_page(sg);
 362
 363                 get_page(page);
 364                 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
 365                                    cur_len);
 366                 skb->data_len += cur_len;
 367                 skb->len += cur_len;
 368                 skb->truesize += cur_len;
 369
 370                 data_length -= cur_len;
 371                 page_off = 0;
 372                 sg = sg_next(sg);
 373                 i++;
 374         }
 375
 376         if (data_length)
 377                 return -1;
 378
 379         if (padding) {
 380                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 381                 if (!page)
 382                         return -1;
 383                 skb_fill_page_desc(skb, i, page, 0, padding);
 384                 skb->data_len += padding;
 385                 skb->len += padding;
 386                 skb->truesize += padding;
 387         }
 388
 389         return 0;
 390 }
 391
 392 static int
 393 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 394                      struct iscsi_datain_req *dr)
 395 {
 396         struct iscsi_conn *conn = csk->conn;
 397         struct sk_buff *skb;
 398         struct iscsi_datain datain;
 399         struct cxgbit_iso_info iso_info;
 400         u32 data_length = cmd->se_cmd.data_length;
 401         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 402         u32 num_pdu, plen, tx_data = 0;
 403         bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
 404                 SCF_TRANSPORT_TASK_SENSE);
 405         bool set_statsn = false;
 406         int ret = -1;
 407
 408         while (data_length) {
 409                 num_pdu = (data_length + mrdsl - 1) / mrdsl;
 410                 if (num_pdu > csk->max_iso_npdu)
 411                         num_pdu = csk->max_iso_npdu;
 412
 413                 plen = num_pdu * mrdsl;
 414                 if (plen > data_length)
 415                         plen = data_length;
 416
 417                 skb = __cxgbit_alloc_skb(csk, 0, true);
 418                 if (unlikely(!skb))
 419                         return -ENOMEM;
 420
 421                 memset(skb->data, 0, ISCSI_HDR_LEN);
 422                 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
 423                 cxgbit_skcb_submode(skb) |= (csk->submode &
 424                                 CXGBIT_SUBMODE_DCRC);
 425                 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
 426                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
 427                                                 ((num_pdu - 1) * ISCSI_HDR_LEN);
 428
 429                 memset(&datain, 0, sizeof(struct iscsi_datain));
 430                 memset(&iso_info, 0, sizeof(iso_info));
 431
 432                 if (!tx_data)
 433                         iso_info.flags |= CXGBIT_ISO_FSLICE;
 434
 435                 if (!(data_length - plen)) {
 436                         iso_info.flags |= CXGBIT_ISO_LSLICE;
 437                         if (!task_sense) {
 438                                 datain.flags = ISCSI_FLAG_DATA_STATUS;
 439                                 iscsit_increment_maxcmdsn(cmd, conn->sess);
 440                                 cmd->stat_sn = conn->stat_sn++;
 441                                 set_statsn = true;
 442                         }
 443                 }
 444
 445                 iso_info.burst_len = num_pdu * mrdsl;
 446                 iso_info.mpdu = mrdsl;
 447                 iso_info.len = ISCSI_HDR_LEN + plen;
 448
 449                 cxgbit_cpl_tx_data_iso(skb, &iso_info);
 450
 451                 datain.offset = tx_data;
 452                 datain.data_sn = cmd->data_sn - 1;
 453
 454                 iscsit_build_datain_pdu(cmd, conn, &datain,
 455                                         (struct iscsi_data_rsp *)skb->data,
 456                                         set_statsn);
 457
 458                 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
 459                 if (unlikely(ret)) {
 460                         __kfree_skb(skb);
 461                         goto out;
 462                 }
 463
 464                 ret = cxgbit_queue_skb(csk, skb);
 465                 if (unlikely(ret))
 466                         goto out;
 467
 468                 tx_data += plen;
 469                 data_length -= plen;
 470
 471                 cmd->read_data_done += plen;
 472                 cmd->data_sn += num_pdu;
 473         }
 474
 475         dr->dr_complete = DATAIN_COMPLETE_NORMAL;
 476
 477         return 0;
 478
 479 out:
 480         return ret;
 481 }
 482
 483 static int
 484 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 485                  const struct iscsi_datain *datain)
 486 {
 487         struct sk_buff *skb;
 488         int ret = 0;
 489
 490         skb = cxgbit_alloc_skb(csk, 0);
 491         if (unlikely(!skb))
 492                 return -ENOMEM;
 493
 494         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 495
 496         if (datain->length) {
 497                 cxgbit_skcb_submode(skb) |= (csk->submode &
 498                                 CXGBIT_SUBMODE_DCRC);
 499                 cxgbit_skcb_tx_extralen(skb) =
 500                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
 501         }
 502
 503         ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
 504         if (ret < 0) {
 505                 __kfree_skb(skb);
 506                 return ret;
 507         }
 508
 509         return cxgbit_queue_skb(csk, skb);
 510 }
 511
 512 static int
 513 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 514                        struct iscsi_datain_req *dr,
 515                        const struct iscsi_datain *datain)
 516 {
 517         struct cxgbit_sock *csk = conn->context;
 518         u32 data_length = cmd->se_cmd.data_length;
 519         u32 padding = ((-data_length) & 3);
 520         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 521
 522         if ((data_length > mrdsl) && (!dr->recovery) &&
 523             (!padding) && (!datain->offset) && csk->max_iso_npdu) {
 524                 atomic_long_add(data_length - datain->length,
 525                                 &conn->sess->tx_data_octets);
 526                 return cxgbit_tx_datain_iso(csk, cmd, dr);
 527         }
 528
 529         return cxgbit_tx_datain(csk, cmd, datain);
 530 }
 531
 532 static int
 533 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 534                           const void *data_buf, u32 data_buf_len)
 535 {
 536         struct cxgbit_sock *csk = conn->context;
 537         struct sk_buff *skb;
 538         u32 padding = ((-data_buf_len) & 3);
 539
 540         skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
 541         if (unlikely(!skb))
 542                 return -ENOMEM;
 543
 544         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 545
 546         if (data_buf_len) {
 547                 u32 pad_bytes = 0;
 548
 549                 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
 550
 551                 if (padding)
 552                         skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
 553                                        &pad_bytes, padding);
 554         }
 555
 556         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
 557                                        cxgbit_skcb_submode(skb)];
 558
 559         return cxgbit_queue_skb(csk, skb);
 560 }
 561
 562 int
 563 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 564                 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
 565 {
 566         if (dr)
 567                 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
 568         else
 569                 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 570 }
 571
 572 int cxgbit_validate_params(struct iscsi_conn *conn)
 573 {
 574         struct cxgbit_sock *csk = conn->context;
 575         struct cxgbit_device *cdev = csk->com.cdev;
 576         struct iscsi_param *param;
 577         u32 max_xmitdsl;
 578
 579         param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
 580                                           conn->param_list);
 581         if (!param)
 582                 return -1;
 583
 584         if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
 585                 return -1;
 586
 587         if (max_xmitdsl > cdev->mdsl) {
 588                 if (iscsi_change_param_sprintf(
 589                         conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
 590                         return -1;
 591         }
 592
 593         return 0;
 594 }
 595
 596 static int cxgbit_set_digest(struct cxgbit_sock *csk)
 597 {
 598         struct iscsi_conn *conn = csk->conn;
 599         struct iscsi_param *param;
 600
 601         param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
 602         if (!param) {
 603                 pr_err("param not found key %s\n", HEADERDIGEST);
 604                 return -1;
 605         }
 606
 607         if (!strcmp(param->value, CRC32C))
 608                 csk->submode |= CXGBIT_SUBMODE_HCRC;
 609
 610         param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
 611         if (!param) {
 612                 csk->submode = 0;
 613                 pr_err("param not found key %s\n", DATADIGEST);
 614                 return -1;
 615         }
 616
 617         if (!strcmp(param->value, CRC32C))
 618                 csk->submode |= CXGBIT_SUBMODE_DCRC;
 619
 620         if (cxgbit_setup_conn_digest(csk)) {
 621                 csk->submode = 0;
 622                 return -1;
 623         }
 624
 625         return 0;
 626 }
 627
 628 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
 629 {
 630         struct iscsi_conn *conn = csk->conn;
 631         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 632         struct iscsi_param *param;
 633         u32 mrdsl, mbl;
 634         u32 max_npdu, max_iso_npdu;
 635         u32 max_iso_payload;
 636
 637         if (conn->login->leading_connection) {
 638                 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
 639                                                   conn->param_list);
 640                 if (!param) {
 641                         pr_err("param not found key %s\n", MAXBURSTLENGTH);
 642                         return -1;
 643                 }
 644
 645                 if (kstrtou32(param->value, 0, &mbl) < 0)
 646                         return -1;
 647         } else {
 648                 mbl = conn->sess->sess_ops->MaxBurstLength;
 649         }
 650
 651         mrdsl = conn_ops->MaxRecvDataSegmentLength;
 652         max_npdu = mbl / mrdsl;
 653
 654         max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
 655
 656         max_iso_npdu = max_iso_payload /
 657                        (ISCSI_HDR_LEN + mrdsl +
 658                         cxgbit_digest_len[csk->submode]);
 659
 660         csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
 661
 662         if (csk->max_iso_npdu <= 1)
 663                 csk->max_iso_npdu = 0;
 664
 665         return 0;
 666 }
 667
 668 /*
 669  * cxgbit_seq_pdu_inorder()
 670  * @csk: pointer to cxgbit socket structure
 671  *
 672  * This function checks whether data sequence and data
 673  * pdu are in order.
 674  *
 675  * Return: returns -1 on error, 0 if data sequence and
 676  * data pdu are in order, 1 if data sequence or data pdu
 677  * is not in order.
 678  */
 679 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
 680 {
 681         struct iscsi_conn *conn = csk->conn;
 682         struct iscsi_param *param;
 683
 684         if (conn->login->leading_connection) {
 685                 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
 686                                                   conn->param_list);
 687                 if (!param) {
 688                         pr_err("param not found key %s\n", DATASEQUENCEINORDER);
 689                         return -1;
 690                 }
 691
 692                 if (strcmp(param->value, YES))
 693                         return 1;
 694
 695                 param = iscsi_find_param_from_key(DATAPDUINORDER,
 696                                                   conn->param_list);
 697                 if (!param) {
 698                         pr_err("param not found key %s\n", DATAPDUINORDER);
 699                         return -1;
 700                 }
 701
 702                 if (strcmp(param->value, YES))
 703                         return 1;
 704
 705         } else {
 706                 if (!conn->sess->sess_ops->DataSequenceInOrder)
 707                         return 1;
 708                 if (!conn->sess->sess_ops->DataPDUInOrder)
 709                         return 1;
 710         }
 711
 712         return 0;
 713 }
 714
 715 static int cxgbit_set_params(struct iscsi_conn *conn)
 716 {
 717         struct cxgbit_sock *csk = conn->context;
 718         struct cxgbit_device *cdev = csk->com.cdev;
 719         struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
 720         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 721         struct iscsi_param *param;
 722         u8 erl;
 723
 724         if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
 725                 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
 726
 727         if (cxgbit_set_digest(csk))
 728                 return -1;
 729
 730         if (conn->login->leading_connection) {
 731                 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
 732                                                   conn->param_list);
 733                 if (!param) {
 734                         pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
 735                         return -1;
 736                 }
 737                 if (kstrtou8(param->value, 0, &erl) < 0)
 738                         return -1;
 739         } else {
 740                 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
 741         }
 742
 743         if (!erl) {
 744                 int ret;
 745
 746                 ret = cxgbit_seq_pdu_inorder(csk);
 747                 if (ret < 0) {
 748                         return -1;
 749                 } else if (ret > 0) {
 750                         if (is_t5(cdev->lldi.adapter_type))
 751                                 goto enable_ddp;
 752                         else
 753                                 return 0;
 754                 }
 755
 756                 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
 757                         if (cxgbit_set_iso_npdu(csk))
 758                                 return -1;
 759                 }
 760
 761 enable_ddp:
 762                 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
 763                         if (cxgbit_setup_conn_pgidx(csk,
 764                                                     ppm->tformat.pgsz_idx_dflt))
 765                                 return -1;
 766                         set_bit(CSK_DDP_ENABLE, &csk->com.flags);
 767                 }
 768         }
 769
 770         return 0;
 771 }
 772
 773 int
 774 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 775                     u32 length)
 776 {
 777         struct cxgbit_sock *csk = conn->context;
 778         struct sk_buff *skb;
 779         u32 padding_buf = 0;
 780         u8 padding = ((-length) & 3);
 781
 782         skb = cxgbit_alloc_skb(csk, length + padding);
 783         if (!skb)
 784                 return -ENOMEM;
 785         skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
 786         skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
 787
 788         if (padding)
 789                 skb_store_bits(skb, ISCSI_HDR_LEN + length,
 790                                &padding_buf, padding);
 791
 792         if (login->login_complete) {
 793                 if (cxgbit_set_params(conn)) {
 794                         kfree_skb(skb);
 795                         return -1;
 796                 }
 797
 798                 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
 799         }
 800
 801         if (cxgbit_queue_skb(csk, skb))
 802                 return -1;
 803
 804         if ((!login->login_complete) && (!login->login_failed))
 805                 schedule_delayed_work(&conn->login_work, 0);
 806
 807         return 0;
 808 }
 809
 810 static void
 811 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
 812                       unsigned int nents, u32 skip)
 813 {
 814         struct skb_seq_state st;
 815         const u8 *buf;
 816         unsigned int consumed = 0, buf_len;
 817         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
 818
 819         skb_prepare_seq_read(skb, pdu_cb->doffset,
 820                              pdu_cb->doffset + pdu_cb->dlen,
 821                              &st);
 822
 823         while (true) {
 824                 buf_len = skb_seq_read(consumed, &buf, &st);
 825                 if (!buf_len) {
 826                         skb_abort_seq_read(&st);
 827                         break;
 828                 }
 829
 830                 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
 831                                                  buf_len, skip + consumed);
 832         }
 833 }
 834
 835 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
 836 {
 837         struct iscsi_conn *conn = csk->conn;
 838         struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
 839         struct cxgbit_cmd *ccmd;
 840         struct iscsi_cmd *cmd;
 841
 842         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 843         if (!cmd) {
 844                 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
 845                 return NULL;
 846         }
 847
 848         ccmd = iscsit_priv_cmd(cmd);
 849         ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
 850         ccmd->setup_ddp = true;
 851
 852         return cmd;
 853 }
 854
 855 static int
 856 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 857                              u32 length)
 858 {
 859         struct iscsi_conn *conn = cmd->conn;
 860         struct cxgbit_sock *csk = conn->context;
 861         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 862
 863         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 864                 pr_err("ImmediateData CRC32C DataDigest error\n");
 865                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 866                         pr_err("Unable to recover from"
 867                                " Immediate Data digest failure while"
 868                                " in ERL=0.\n");
 869                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 870                                           (unsigned char *)hdr);
 871                         return IMMEDIATE_DATA_CANNOT_RECOVER;
 872                 }
 873
 874                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 875                                   (unsigned char *)hdr);
 876                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
 877         }
 878
 879         if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
 880                 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
 881                 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
 882                 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
 883
 884                 sg_init_table(&ccmd->sg, 1);
 885                 sg_set_page(&ccmd->sg, skb_frag_page(dfrag),
 886                                 skb_frag_size(dfrag), skb_frag_off(dfrag));
 887                 get_page(skb_frag_page(dfrag));
 888
 889                 cmd->se_cmd.t_data_sg = &ccmd->sg;
 890                 cmd->se_cmd.t_data_nents = 1;
 891
 892                 ccmd->release = true;
 893         } else {
 894                 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
 895                 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
 896
 897                 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
 898         }
 899
 900         cmd->write_data_done += pdu_cb->dlen;
 901
 902         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 903                 spin_lock_bh(&cmd->istate_lock);
 904                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 905                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 906                 spin_unlock_bh(&cmd->istate_lock);
 907         }
 908
 909         return IMMEDIATE_DATA_NORMAL_OPERATION;
 910 }
 911
 912 static int
 913 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 914                           bool dump_payload)
 915 {
 916         struct iscsi_conn *conn = cmd->conn;
 917         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
 918         /*
 919          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 920          */
 921         if (dump_payload)
 922                 goto after_immediate_data;
 923
 924         immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
 925                                                  cmd->first_burst_len);
 926 after_immediate_data:
 927         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 928                 /*
 929                  * A PDU/CmdSN carrying Immediate Data passed
 930                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
 931                  * Immediate Bit is not set.
 932                  */
 933                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
 934                                                 (unsigned char *)hdr,
 935                                                 hdr->cmdsn);
 936                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
 937                         return -1;
 938
 939                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
 940                         target_put_sess_cmd(&cmd->se_cmd);
 941                         return 0;
 942                 } else if (cmd->unsolicited_data) {
 943                         iscsit_set_unsolicited_dataout(cmd);
 944                 }
 945
 946         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
 947                 /*
 948                  * Immediate Data failed DataCRC and ERL>=1,
 949                  * silently drop this PDU and let the initiator
 950                  * plug the CmdSN gap.
 951                  *
 952                  * FIXME: Send Unsolicited NOPIN with reserved
 953                  * TTT here to help the initiator figure out
 954                  * the missing CmdSN, although they should be
 955                  * intelligent enough to determine the missing
 956                  * CmdSN and issue a retry to plug the sequence.
 957                  */
 958                 cmd->i_state = ISTATE_REMOVE;
 959                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 960         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
 961                 return -1;
 962
 963         return 0;
 964 }
 965
 966 static int
 967 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
 968 {
 969         struct iscsi_conn *conn = csk->conn;
 970         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 971         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
 972         int rc;
 973         bool dump_payload = false;
 974
 975         rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
 976         if (rc < 0)
 977                 return rc;
 978
 979         if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
 980             (pdu_cb->nr_dfrags == 1))
 981                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
 982
 983         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
 984         if (rc < 0)
 985                 return 0;
 986         else if (rc > 0)
 987                 dump_payload = true;
 988
 989         if (!pdu_cb->dlen)
 990                 return 0;
 991
 992         return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
 993 }
 994
 995 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
 996 {
 997         struct scatterlist *sg_start;
 998         struct iscsi_conn *conn = csk->conn;
 999         struct iscsi_cmd *cmd = NULL;
1000         struct cxgbit_cmd *ccmd;
1001         struct cxgbi_task_tag_info *ttinfo;
1002         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1003         struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1004         u32 data_offset = be32_to_cpu(hdr->offset);
1005         u32 data_len = ntoh24(hdr->dlength);
1006         int rc, sg_nents, sg_off;
1007         bool dcrc_err = false;
1008
1009         if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1010                 u32 offset = be32_to_cpu(hdr->offset);
1011                 u32 ddp_data_len;
1012                 bool success = false;
1013
1014                 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1015                 if (!cmd)
1016                         return 0;
1017
1018                 ddp_data_len = offset - cmd->write_data_done;
1019                 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1020
1021                 cmd->write_data_done = offset;
1022                 cmd->next_burst_len = ddp_data_len;
1023                 cmd->data_sn = be32_to_cpu(hdr->datasn);
1024
1025                 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1026                                                 cmd, data_len, &success);
1027                 if (rc < 0)
1028                         return rc;
1029                 else if (!success)
1030                         return 0;
1031         } else {
1032                 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1033                 if (rc < 0)
1034                         return rc;
1035                 else if (!cmd)
1036                         return 0;
1037         }
1038
1039         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1040                 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1041                        " DataSN: 0x%08x\n",
1042                        hdr->itt, hdr->offset, data_len,
1043                        hdr->datasn);
1044
1045                 dcrc_err = true;
1046                 goto check_payload;
1047         }
1048
1049         pr_debug("DataOut data_len: %u, "
1050                 "write_data_done: %u, data_length: %u\n",
1051                   data_len,  cmd->write_data_done,
1052                   cmd->se_cmd.data_length);
1053
1054         if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1055                 u32 skip = data_offset % PAGE_SIZE;
1056
1057                 sg_off = data_offset / PAGE_SIZE;
1058                 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1059                 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1060
1061                 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1062         }
1063
1064         ccmd = iscsit_priv_cmd(cmd);
1065         ttinfo = &ccmd->ttinfo;
1066
1067         if (ccmd->release && ttinfo->sgl &&
1068             (cmd->se_cmd.data_length == (cmd->write_data_done + data_len))) {
1069                 struct cxgbit_device *cdev = csk->com.cdev;
1070                 struct cxgbi_ppm *ppm = cdev2ppm(cdev);
1071
1072                 dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
1073                              DMA_FROM_DEVICE);
1074                 ttinfo->nents = 0;
1075                 ttinfo->sgl = NULL;
1076         }
1077
1078 check_payload:
1079
1080         rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1081         if (rc < 0)
1082                 return rc;
1083
1084         return 0;
1085 }
1086
1087 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1088 {
1089         struct iscsi_conn *conn = csk->conn;
1090         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1091         struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1092         unsigned char *ping_data = NULL;
1093         u32 payload_length = pdu_cb->dlen;
1094         int ret;
1095
1096         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1097         if (ret < 0)
1098                 return 0;
1099
1100         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1101                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1102                         pr_err("Unable to recover from"
1103                                " NOPOUT Ping DataCRC failure while in"
1104                                " ERL=0.\n");
1105                         ret = -1;
1106                         goto out;
1107                 } else {
1108                         /*
1109                          * drop this PDU and let the
1110                          * initiator plug the CmdSN gap.
1111                          */
1112                         pr_info("Dropping NOPOUT"
1113                                 " Command CmdSN: 0x%08x due to"
1114                                 " DataCRC error.\n", hdr->cmdsn);
1115                         ret = 0;
1116                         goto out;
1117                 }
1118         }
1119
1120         /*
1121          * Handle NOP-OUT payload for traditional iSCSI sockets
1122          */
1123         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1124                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1125                 if (!ping_data) {
1126                         pr_err("Unable to allocate memory for"
1127                                 " NOPOUT ping data.\n");
1128                         ret = -1;
1129                         goto out;
1130                 }
1131
1132                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1133                               ping_data, payload_length);
1134
1135                 ping_data[payload_length] = '\0';
1136                 /*
1137                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1138                  */
1139                 cmd->buf_ptr = ping_data;
1140                 cmd->buf_ptr_size = payload_length;
1141
1142                 pr_debug("Got %u bytes of NOPOUT ping"
1143                         " data.\n", payload_length);
1144                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1145         }
1146
1147         return iscsit_process_nop_out(conn, cmd, hdr);
1148 out:
1149         if (cmd)
1150                 iscsit_free_cmd(cmd, false);
1151         return ret;
1152 }
1153
1154 static int
1155 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1156 {
1157         struct iscsi_conn *conn = csk->conn;
1158         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1159         struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1160         u32 payload_length = pdu_cb->dlen;
1161         int rc;
1162         unsigned char *text_in = NULL;
1163
1164         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1165         if (rc < 0)
1166                 return rc;
1167
1168         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1169                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1170                         pr_err("Unable to recover from"
1171                                " Text Data digest failure while in"
1172                                " ERL=0.\n");
1173                         goto reject;
1174                 } else {
1175                         /*
1176                          * drop this PDU and let the
1177                          * initiator plug the CmdSN gap.
1178                          */
1179                         pr_info("Dropping Text"
1180                                 " Command CmdSN: 0x%08x due to"
1181                                 " DataCRC error.\n", hdr->cmdsn);
1182                         return 0;
1183                 }
1184         }
1185
1186         if (payload_length) {
1187                 text_in = kzalloc(payload_length, GFP_KERNEL);
1188                 if (!text_in) {
1189                         pr_err("Unable to allocate text_in of payload_length: %u\n",
1190                                payload_length);
1191                         return -ENOMEM;
1192                 }
1193                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1194                               text_in, payload_length);
1195
1196                 text_in[payload_length - 1] = '\0';
1197
1198                 cmd->text_in_ptr = text_in;
1199         }
1200
1201         return iscsit_process_text_cmd(conn, cmd, hdr);
1202
1203 reject:
1204         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1205                                  pdu_cb->hdr);
1206 }
1207
1208 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1209 {
1210         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1211         struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1212         struct iscsi_conn *conn = csk->conn;
1213         struct iscsi_cmd *cmd = NULL;
1214         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1215         int ret = -EINVAL;
1216
1217         switch (opcode) {
1218         case ISCSI_OP_SCSI_CMD:
1219                 cmd = cxgbit_allocate_cmd(csk);
1220                 if (!cmd)
1221                         goto reject;
1222
1223                 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1224                 break;
1225         case ISCSI_OP_SCSI_DATA_OUT:
1226                 ret = cxgbit_handle_iscsi_dataout(csk);
1227                 break;
1228         case ISCSI_OP_NOOP_OUT:
1229                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1230                         cmd = cxgbit_allocate_cmd(csk);
1231                         if (!cmd)
1232                                 goto reject;
1233                 }
1234
1235                 ret = cxgbit_handle_nop_out(csk, cmd);
1236                 break;
1237         case ISCSI_OP_SCSI_TMFUNC:
1238                 cmd = cxgbit_allocate_cmd(csk);
1239                 if (!cmd)
1240                         goto reject;
1241
1242                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1243                                                  (unsigned char *)hdr);
1244                 break;
1245         case ISCSI_OP_TEXT:
1246                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1247                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1248                         if (!cmd)
1249                                 goto reject;
1250                 } else {
1251                         cmd = cxgbit_allocate_cmd(csk);
1252                         if (!cmd)
1253                                 goto reject;
1254                 }
1255
1256                 ret = cxgbit_handle_text_cmd(csk, cmd);
1257                 break;
1258         case ISCSI_OP_LOGOUT:
1259                 cmd = cxgbit_allocate_cmd(csk);
1260                 if (!cmd)
1261                         goto reject;
1262
1263                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1264                 if (ret > 0)
1265                         wait_for_completion_timeout(&conn->conn_logout_comp,
1266                                                     SECONDS_FOR_LOGOUT_COMP
1267                                                     * HZ);
1268                 break;
1269         case ISCSI_OP_SNACK:
1270                 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1271                 break;
1272         default:
1273                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1274                 dump_stack();
1275                 break;
1276         }
1277
1278         return ret;
1279
1280 reject:
1281         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1282                                  (unsigned char *)hdr);
1283         return ret;
1284 }
1285
1286 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1287 {
1288         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1289         struct iscsi_conn *conn = csk->conn;
1290         struct iscsi_hdr *hdr = pdu_cb->hdr;
1291         u8 opcode;
1292
1293         if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1294                 atomic_long_inc(&conn->sess->conn_digest_errors);
1295                 goto transport_err;
1296         }
1297
1298         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1299                 goto transport_err;
1300
1301         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1302
1303         if (conn->sess->sess_ops->SessionType &&
1304             ((!(opcode & ISCSI_OP_TEXT)) ||
1305              (!(opcode & ISCSI_OP_LOGOUT)))) {
1306                 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1307                         " while in Discovery Session, rejecting.\n", opcode);
1308                 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1309                                   (unsigned char *)hdr);
1310                 goto transport_err;
1311         }
1312
1313         if (cxgbit_target_rx_opcode(csk) < 0)
1314                 goto transport_err;
1315
1316         return 0;
1317
1318 transport_err:
1319         return -1;
1320 }
1321
1322 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1323 {
1324         struct iscsi_conn *conn = csk->conn;
1325         struct iscsi_login *login = conn->login;
1326         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1327         struct iscsi_login_req *login_req;
1328
1329         login_req = (struct iscsi_login_req *)login->req;
1330         memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1331
1332         pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1333                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1334                 login_req->flags, login_req->itt, login_req->cmdsn,
1335                 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1336         /*
1337          * Setup the initial iscsi_login values from the leading
1338          * login request PDU.
1339          */
1340         if (login->first_request) {
1341                 login_req = (struct iscsi_login_req *)login->req;
1342                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1343                 login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(
1344                                 login_req->flags);
1345                 login->version_min      = login_req->min_version;
1346                 login->version_max      = login_req->max_version;
1347                 memcpy(login->isid, login_req->isid, 6);
1348                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1349                 login->init_task_tag    = login_req->itt;
1350                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1351                 login->cid              = be16_to_cpu(login_req->cid);
1352                 login->tsih             = be16_to_cpu(login_req->tsih);
1353         }
1354
1355         if (iscsi_target_check_login_request(conn, login) < 0)
1356                 return -1;
1357
1358         memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1359         skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1360
1361         return 0;
1362 }
1363
1364 static int
1365 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1366 {
1367         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1368         int ret;
1369
1370         cxgbit_rx_pdu_cb(skb) = pdu_cb;
1371
1372         csk->skb = skb;
1373
1374         if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1375                 ret = cxgbit_rx_login_pdu(csk);
1376                 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1377         } else {
1378                 ret = cxgbit_rx_opcode(csk);
1379         }
1380
1381         return ret;
1382 }
1383
1384 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1385 {
1386         struct skb_shared_info *ssi = skb_shinfo(skb);
1387         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1388         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1389         u8 i;
1390
1391         pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1392                 skb, skb->head, skb->data, skb->len, skb->data_len,
1393                 ssi->nr_frags);
1394         pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1395                 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1396
1397         for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1398                 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1399                         "frags %u.\n",
1400                         skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1401                         pdu_cb->ddigest, pdu_cb->frags);
1402         for (i = 0; i < ssi->nr_frags; i++)
1403                 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1404                         skb, i, skb_frag_off(&ssi->frags[i]),
1405                         skb_frag_size(&ssi->frags[i]));
1406 }
1407
1408 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1409 {
1410         struct sk_buff *skb = csk->lro_hskb;
1411         struct skb_shared_info *ssi = skb_shinfo(skb);
1412         u8 i;
1413
1414         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1415         for (i = 0; i < ssi->nr_frags; i++)
1416                 put_page(skb_frag_page(&ssi->frags[i]));
1417         ssi->nr_frags = 0;
1418         skb->data_len = 0;
1419         skb->truesize -= skb->len;
1420         skb->len = 0;
1421 }
1422
1423 static void
1424 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1425 {
1426         struct sk_buff *hskb = csk->lro_hskb;
1427         struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1428         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1429         struct skb_shared_info *hssi = skb_shinfo(hskb);
1430         struct skb_shared_info *ssi = skb_shinfo(skb);
1431         unsigned int len = 0;
1432
1433         if (pdu_cb->flags & PDUCBF_RX_HDR) {
1434                 u8 hfrag_idx = hssi->nr_frags;
1435
1436                 hpdu_cb->flags |= pdu_cb->flags;
1437                 hpdu_cb->seq = pdu_cb->seq;
1438                 hpdu_cb->hdr = pdu_cb->hdr;
1439                 hpdu_cb->hlen = pdu_cb->hlen;
1440
1441                 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1442                        sizeof(skb_frag_t));
1443
1444                 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1445                 hssi->nr_frags++;
1446                 hpdu_cb->frags++;
1447                 hpdu_cb->hfrag_idx = hfrag_idx;
1448
1449                 len = skb_frag_size(&hssi->frags[hfrag_idx]);
1450                 hskb->len += len;
1451                 hskb->data_len += len;
1452                 hskb->truesize += len;
1453         }
1454
1455         if (pdu_cb->flags & PDUCBF_RX_DATA) {
1456                 u8 dfrag_idx = hssi->nr_frags, i;
1457
1458                 hpdu_cb->flags |= pdu_cb->flags;
1459                 hpdu_cb->dfrag_idx = dfrag_idx;
1460
1461                 len = 0;
1462                 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1463                         memcpy(&hssi->frags[dfrag_idx],
1464                                &ssi->frags[pdu_cb->dfrag_idx + i],
1465                                sizeof(skb_frag_t));
1466
1467                         get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1468
1469                         len += skb_frag_size(&hssi->frags[dfrag_idx]);
1470
1471                         hssi->nr_frags++;
1472                         hpdu_cb->frags++;
1473                 }
1474
1475                 hpdu_cb->dlen = pdu_cb->dlen;
1476                 hpdu_cb->doffset = hpdu_cb->hlen;
1477                 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1478                 hskb->len += len;
1479                 hskb->data_len += len;
1480                 hskb->truesize += len;
1481         }
1482
1483         if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1484                 hpdu_cb->flags |= pdu_cb->flags;
1485
1486                 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1487                         hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1488
1489                 hpdu_cb->ddigest = pdu_cb->ddigest;
1490                 hpdu_cb->pdulen = pdu_cb->pdulen;
1491         }
1492 }
1493
1494 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1495 {
1496         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1497         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1498         u8 pdu_idx = 0, last_idx = 0;
1499         int ret = 0;
1500
1501         if (!pdu_cb->complete) {
1502                 cxgbit_lro_skb_merge(csk, skb, 0);
1503
1504                 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1505                         struct sk_buff *hskb = csk->lro_hskb;
1506
1507                         ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1508
1509                         cxgbit_lro_hskb_reset(csk);
1510
1511                         if (ret < 0)
1512                                 goto out;
1513                 }
1514
1515                 pdu_idx = 1;
1516         }
1517
1518         if (lro_cb->pdu_idx)
1519                 last_idx = lro_cb->pdu_idx - 1;
1520
1521         for (; pdu_idx <= last_idx; pdu_idx++) {
1522                 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1523                 if (ret < 0)
1524                         goto out;
1525         }
1526
1527         if ((!lro_cb->complete) && lro_cb->pdu_idx)
1528                 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1529
1530 out:
1531         return ret;
1532 }
1533
1534 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1535 {
1536         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1537         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1538         int ret = -1;
1539
1540         if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1541             (pdu_cb->seq != csk->rcv_nxt)) {
1542                 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1543                         csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1544                 cxgbit_lro_skb_dump(skb);
1545                 return ret;
1546         }
1547
1548         csk->rcv_nxt += lro_cb->pdu_totallen;
1549
1550         ret = cxgbit_process_lro_skb(csk, skb);
1551
1552         csk->rx_credits += lro_cb->pdu_totallen;
1553
1554         if (csk->rx_credits >= (csk->rcv_win / 4))
1555                 cxgbit_rx_data_ack(csk);
1556
1557         return ret;
1558 }
1559
1560 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1561 {
1562         struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1563         int ret = -1;
1564
1565         if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1566                 if (is_t5(lldi->adapter_type))
1567                         ret = cxgbit_rx_lro_skb(csk, skb);
1568                 else
1569                         ret = cxgbit_process_lro_skb(csk, skb);
1570         }
1571
1572         __kfree_skb(skb);
1573         return ret;
1574 }
1575
1576 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1577 {
1578         spin_lock_bh(&csk->rxq.lock);
1579         if (skb_queue_len(&csk->rxq)) {
1580                 skb_queue_splice_init(&csk->rxq, rxq);
1581                 spin_unlock_bh(&csk->rxq.lock);
1582                 return true;
1583         }
1584         spin_unlock_bh(&csk->rxq.lock);
1585         return false;
1586 }
1587
1588 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1589 {
1590         struct sk_buff *skb;
1591         struct sk_buff_head rxq;
1592
1593         skb_queue_head_init(&rxq);
1594
1595         wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1596
1597         if (signal_pending(current))
1598                 goto out;
1599
1600         while ((skb = __skb_dequeue(&rxq))) {
1601                 if (cxgbit_rx_skb(csk, skb))
1602                         goto out;
1603         }
1604
1605         return 0;
1606 out:
1607         __skb_queue_purge(&rxq);
1608         return -1;
1609 }
1610
1611 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1612 {
1613         struct cxgbit_sock *csk = conn->context;
1614         int ret = -1;
1615
1616         while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1617                 ret = cxgbit_wait_rxq(csk);
1618                 if (ret) {
1619                         clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1620                         break;
1621                 }
1622         }
1623
1624         return ret;
1625 }
1626
1627 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1628 {
1629         struct cxgbit_sock *csk = conn->context;
1630
1631         while (!kthread_should_stop()) {
1632                 iscsit_thread_check_cpumask(conn, current, 0);
1633                 if (cxgbit_wait_rxq(csk))
1634                         return;
1635         }
1636 }