drivers/target/iscsi/cxgbit/cxgbit_target.c

   1 /*
   2  * Copyright (c) 2016 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #include <linux/workqueue.h>
  10 #include <linux/kthread.h>
  11 #include <linux/sched/signal.h>
  12
  13 #include <asm/unaligned.h>
  14 #include <net/tcp.h>
  15 #include <target/target_core_base.h>
  16 #include <target/target_core_fabric.h>
  17 #include "cxgbit.h"
  18
  19 struct sge_opaque_hdr {
  20         void *dev;
  21         dma_addr_t addr[MAX_SKB_FRAGS + 1];
  22 };
  23
  24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
  25
  26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
  27                     sizeof(struct fw_ofld_tx_data_wr))
  28
  29 static struct sk_buff *
  30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
  31 {
  32         struct sk_buff *skb = NULL;
  33         u8 submode = 0;
  34         int errcode;
  35         static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
  36
  37         if (len) {
  38                 skb = alloc_skb_with_frags(hdr_len, len,
  39                                            0, &errcode,
  40                                            GFP_KERNEL);
  41                 if (!skb)
  42                         return NULL;
  43
  44                 skb_reserve(skb, TX_HDR_LEN);
  45                 skb_reset_transport_header(skb);
  46                 __skb_put(skb, ISCSI_HDR_LEN);
  47                 skb->data_len = len;
  48                 skb->len += len;
  49                 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
  50
  51         } else {
  52                 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
  53
  54                 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
  55                 if (!skb)
  56                         return NULL;
  57
  58                 skb_reserve(skb, TX_HDR_LEN + iso_len);
  59                 skb_reset_transport_header(skb);
  60                 __skb_put(skb, ISCSI_HDR_LEN);
  61         }
  62
  63         submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
  64         cxgbit_skcb_submode(skb) = submode;
  65         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
  66         cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
  67         return skb;
  68 }
  69
  70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
  71 {
  72         return __cxgbit_alloc_skb(csk, len, false);
  73 }
  74
  75 /*
  76  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
  77  * @skb: the packet
  78  *
  79  * Returns true if a packet can be sent as an offload WR with immediate
  80  * data.  We currently use the same limit as for Ethernet packets.
  81  */
  82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
  83 {
  84         int length = skb->len;
  85
  86         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
  87                 length += sizeof(struct fw_ofld_tx_data_wr);
  88
  89         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
  90                 length += sizeof(struct cpl_tx_data_iso);
  91
  92         return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
  93 }
  94
  95 /*
  96  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
  97  * @n: the number of SGL entries
  98  * Calculates the number of flits needed for a scatter/gather list that
  99  * can hold the given number of entries.
 100  */
 101 static inline unsigned int cxgbit_sgl_len(unsigned int n)
 102 {
 103         n--;
 104         return (3 * n) / 2 + (n & 1) + 2;
 105 }
 106
 107 /*
 108  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
 109  * @skb: the packet
 110  *
 111  * Returns the number of flits needed for the given offload packet.
 112  * These packets are already fully constructed and no additional headers
 113  * will be added.
 114  */
 115 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
 116 {
 117         unsigned int flits, cnt;
 118
 119         if (cxgbit_is_ofld_imm(skb))
 120                 return DIV_ROUND_UP(skb->len, 8);
 121         flits = skb_transport_offset(skb) / 8;
 122         cnt = skb_shinfo(skb)->nr_frags;
 123         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 124                 cnt++;
 125         return flits + cxgbit_sgl_len(cnt);
 126 }
 127
 128 #define CXGBIT_ISO_FSLICE 0x1
 129 #define CXGBIT_ISO_LSLICE 0x2
 130 static void
 131 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
 132 {
 133         struct cpl_tx_data_iso *cpl;
 134         unsigned int submode = cxgbit_skcb_submode(skb);
 135         unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
 136         unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
 137
 138         cpl = __skb_push(skb, sizeof(*cpl));
 139
 140         cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
 141                         CPL_TX_DATA_ISO_FIRST_V(fslice) |
 142                         CPL_TX_DATA_ISO_LAST_V(lslice) |
 143                         CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
 144                         CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
 145                         CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
 146                         CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
 147                         CPL_TX_DATA_ISO_SCSI_V(2));
 148
 149         cpl->ahs_len = 0;
 150         cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
 151         cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
 152         cpl->len = htonl(iso_info->len);
 153         cpl->reserved2_seglen_offset = htonl(0);
 154         cpl->datasn_offset = htonl(0);
 155         cpl->buffer_offset = htonl(0);
 156         cpl->reserved3 = 0;
 157
 158         __skb_pull(skb, sizeof(*cpl));
 159 }
 160
 161 static void
 162 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
 163                   u32 len, u32 credits, u32 compl)
 164 {
 165         struct fw_ofld_tx_data_wr *req;
 166         const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
 167         u32 submode = cxgbit_skcb_submode(skb);
 168         u32 wr_ulp_mode = 0;
 169         u32 hdr_size = sizeof(*req);
 170         u32 opcode = FW_OFLD_TX_DATA_WR;
 171         u32 immlen = 0;
 172         u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
 173                     T6_TX_FORCE_F;
 174
 175         if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
 176                 opcode = FW_ISCSI_TX_DATA_WR;
 177                 immlen += sizeof(struct cpl_tx_data_iso);
 178                 hdr_size += sizeof(struct cpl_tx_data_iso);
 179                 submode |= 8;
 180         }
 181
 182         if (cxgbit_is_ofld_imm(skb))
 183                 immlen += dlen;
 184
 185         req = __skb_push(skb, hdr_size);
 186         req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
 187                                         FW_WR_COMPL_V(compl) |
 188                                         FW_WR_IMMDLEN_V(immlen));
 189         req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 190                                         FW_WR_LEN16_V(credits));
 191         req->plen = htonl(len);
 192         wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
 193                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 194
 195         req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
 196                  FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
 197 }
 198
 199 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 200 {
 201         kfree_skb(skb);
 202 }
 203
 204 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
 205 {
 206         struct sk_buff *skb;
 207
 208         while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
 209                 u32 dlen = skb->len;
 210                 u32 len = skb->len;
 211                 u32 credits_needed;
 212                 u32 compl = 0;
 213                 u32 flowclen16 = 0;
 214                 u32 iso_cpl_len = 0;
 215
 216                 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
 217                         iso_cpl_len = sizeof(struct cpl_tx_data_iso);
 218
 219                 if (cxgbit_is_ofld_imm(skb))
 220                         credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
 221                 else
 222                         credits_needed = DIV_ROUND_UP((8 *
 223                                         cxgbit_calc_tx_flits_ofld(skb)) +
 224                                         iso_cpl_len, 16);
 225
 226                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
 227                         credits_needed += DIV_ROUND_UP(
 228                                 sizeof(struct fw_ofld_tx_data_wr), 16);
 229                 /*
 230                  * Assumes the initial credits is large enough to support
 231                  * fw_flowc_wr plus largest possible first payload
 232                  */
 233
 234                 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
 235                         flowclen16 = cxgbit_send_tx_flowc_wr(csk);
 236                         csk->wr_cred -= flowclen16;
 237                         csk->wr_una_cred += flowclen16;
 238                 }
 239
 240                 if (csk->wr_cred < credits_needed) {
 241                         pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
 242                                  csk, skb->len, skb->data_len,
 243                                  credits_needed, csk->wr_cred);
 244                         break;
 245                 }
 246                 __skb_unlink(skb, &csk->txq);
 247                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
 248                 skb->csum = (__force __wsum)(credits_needed + flowclen16);
 249                 csk->wr_cred -= credits_needed;
 250                 csk->wr_una_cred += credits_needed;
 251
 252                 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 253                          csk, skb->len, skb->data_len, credits_needed,
 254                          csk->wr_cred, csk->wr_una_cred);
 255
 256                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
 257                         len += cxgbit_skcb_tx_extralen(skb);
 258
 259                         if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
 260                             (!before(csk->write_seq,
 261                                      csk->snd_una + csk->snd_win))) {
 262                                 compl = 1;
 263                                 csk->wr_una_cred = 0;
 264                         }
 265
 266                         cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
 267                                           compl);
 268                         csk->snd_nxt += len;
 269
 270                 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
 271                            (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
 272                         struct cpl_close_con_req *req =
 273                                 (struct cpl_close_con_req *)skb->data;
 274                         req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
 275                         csk->wr_una_cred = 0;
 276                 }
 277
 278                 cxgbit_sock_enqueue_wr(csk, skb);
 279                 t4_set_arp_err_handler(skb, csk,
 280                                        cxgbit_arp_failure_skb_discard);
 281
 282                 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
 283                          csk, csk->tid, skb, len);
 284
 285                 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 286         }
 287 }
 288
 289 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
 290 {
 291         spin_lock_bh(&csk->lock);
 292
 293         if (before(csk->write_seq, csk->snd_una + csk->snd_win))
 294                 csk->lock_owner = true;
 295
 296         spin_unlock_bh(&csk->lock);
 297
 298         return csk->lock_owner;
 299 }
 300
 301 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
 302 {
 303         struct sk_buff_head backlogq;
 304         struct sk_buff *skb;
 305         void (*fn)(struct cxgbit_sock *, struct sk_buff *);
 306
 307         skb_queue_head_init(&backlogq);
 308
 309         spin_lock_bh(&csk->lock);
 310         while (skb_queue_len(&csk->backlogq)) {
 311                 skb_queue_splice_init(&csk->backlogq, &backlogq);
 312                 spin_unlock_bh(&csk->lock);
 313
 314                 while ((skb = __skb_dequeue(&backlogq))) {
 315                         fn = cxgbit_skcb_rx_backlog_fn(skb);
 316                         fn(csk, skb);
 317                 }
 318
 319                 spin_lock_bh(&csk->lock);
 320         }
 321
 322         csk->lock_owner = false;
 323         spin_unlock_bh(&csk->lock);
 324 }
 325
 326 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 327 {
 328         int ret = 0;
 329
 330         wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
 331
 332         if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
 333                      signal_pending(current))) {
 334                 __kfree_skb(skb);
 335                 __skb_queue_purge(&csk->ppodq);
 336                 ret = -1;
 337                 spin_lock_bh(&csk->lock);
 338                 if (csk->lock_owner) {
 339                         spin_unlock_bh(&csk->lock);
 340                         goto unlock;
 341                 }
 342                 spin_unlock_bh(&csk->lock);
 343                 return ret;
 344         }
 345
 346         csk->write_seq += skb->len +
 347                           cxgbit_skcb_tx_extralen(skb);
 348
 349         skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
 350         __skb_queue_tail(&csk->txq, skb);
 351         cxgbit_push_tx_frames(csk);
 352
 353 unlock:
 354         cxgbit_unlock_sock(csk);
 355         return ret;
 356 }
 357
 358 static int
 359 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
 360                u32 data_length)
 361 {
 362         u32 i = 0, nr_frags = MAX_SKB_FRAGS;
 363         u32 padding = ((-data_length) & 3);
 364         struct scatterlist *sg;
 365         struct page *page;
 366         unsigned int page_off;
 367
 368         if (padding)
 369                 nr_frags--;
 370
 371         /*
 372          * We know each entry in t_data_sg contains a page.
 373          */
 374         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 375         page_off = (data_offset % PAGE_SIZE);
 376
 377         while (data_length && (i < nr_frags)) {
 378                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 379
 380                 page = sg_page(sg);
 381
 382                 get_page(page);
 383                 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
 384                                    cur_len);
 385                 skb->data_len += cur_len;
 386                 skb->len += cur_len;
 387                 skb->truesize += cur_len;
 388
 389                 data_length -= cur_len;
 390                 page_off = 0;
 391                 sg = sg_next(sg);
 392                 i++;
 393         }
 394
 395         if (data_length)
 396                 return -1;
 397
 398         if (padding) {
 399                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 400                 if (!page)
 401                         return -1;
 402                 skb_fill_page_desc(skb, i, page, 0, padding);
 403                 skb->data_len += padding;
 404                 skb->len += padding;
 405                 skb->truesize += padding;
 406         }
 407
 408         return 0;
 409 }
 410
 411 static int
 412 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 413                      struct iscsi_datain_req *dr)
 414 {
 415         struct iscsi_conn *conn = csk->conn;
 416         struct sk_buff *skb;
 417         struct iscsi_datain datain;
 418         struct cxgbit_iso_info iso_info;
 419         u32 data_length = cmd->se_cmd.data_length;
 420         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 421         u32 num_pdu, plen, tx_data = 0;
 422         bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
 423                 SCF_TRANSPORT_TASK_SENSE);
 424         bool set_statsn = false;
 425         int ret = -1;
 426
 427         while (data_length) {
 428                 num_pdu = (data_length + mrdsl - 1) / mrdsl;
 429                 if (num_pdu > csk->max_iso_npdu)
 430                         num_pdu = csk->max_iso_npdu;
 431
 432                 plen = num_pdu * mrdsl;
 433                 if (plen > data_length)
 434                         plen = data_length;
 435
 436                 skb = __cxgbit_alloc_skb(csk, 0, true);
 437                 if (unlikely(!skb))
 438                         return -ENOMEM;
 439
 440                 memset(skb->data, 0, ISCSI_HDR_LEN);
 441                 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
 442                 cxgbit_skcb_submode(skb) |= (csk->submode &
 443                                 CXGBIT_SUBMODE_DCRC);
 444                 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
 445                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
 446                                                 ((num_pdu - 1) * ISCSI_HDR_LEN);
 447
 448                 memset(&datain, 0, sizeof(struct iscsi_datain));
 449                 memset(&iso_info, 0, sizeof(iso_info));
 450
 451                 if (!tx_data)
 452                         iso_info.flags |= CXGBIT_ISO_FSLICE;
 453
 454                 if (!(data_length - plen)) {
 455                         iso_info.flags |= CXGBIT_ISO_LSLICE;
 456                         if (!task_sense) {
 457                                 datain.flags = ISCSI_FLAG_DATA_STATUS;
 458                                 iscsit_increment_maxcmdsn(cmd, conn->sess);
 459                                 cmd->stat_sn = conn->stat_sn++;
 460                                 set_statsn = true;
 461                         }
 462                 }
 463
 464                 iso_info.burst_len = num_pdu * mrdsl;
 465                 iso_info.mpdu = mrdsl;
 466                 iso_info.len = ISCSI_HDR_LEN + plen;
 467
 468                 cxgbit_cpl_tx_data_iso(skb, &iso_info);
 469
 470                 datain.offset = tx_data;
 471                 datain.data_sn = cmd->data_sn - 1;
 472
 473                 iscsit_build_datain_pdu(cmd, conn, &datain,
 474                                         (struct iscsi_data_rsp *)skb->data,
 475                                         set_statsn);
 476
 477                 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
 478                 if (unlikely(ret)) {
 479                         __kfree_skb(skb);
 480                         goto out;
 481                 }
 482
 483                 ret = cxgbit_queue_skb(csk, skb);
 484                 if (unlikely(ret))
 485                         goto out;
 486
 487                 tx_data += plen;
 488                 data_length -= plen;
 489
 490                 cmd->read_data_done += plen;
 491                 cmd->data_sn += num_pdu;
 492         }
 493
 494         dr->dr_complete = DATAIN_COMPLETE_NORMAL;
 495
 496         return 0;
 497
 498 out:
 499         return ret;
 500 }
 501
 502 static int
 503 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 504                  const struct iscsi_datain *datain)
 505 {
 506         struct sk_buff *skb;
 507         int ret = 0;
 508
 509         skb = cxgbit_alloc_skb(csk, 0);
 510         if (unlikely(!skb))
 511                 return -ENOMEM;
 512
 513         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 514
 515         if (datain->length) {
 516                 cxgbit_skcb_submode(skb) |= (csk->submode &
 517                                 CXGBIT_SUBMODE_DCRC);
 518                 cxgbit_skcb_tx_extralen(skb) =
 519                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
 520         }
 521
 522         ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
 523         if (ret < 0) {
 524                 __kfree_skb(skb);
 525                 return ret;
 526         }
 527
 528         return cxgbit_queue_skb(csk, skb);
 529 }
 530
 531 static int
 532 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 533                        struct iscsi_datain_req *dr,
 534                        const struct iscsi_datain *datain)
 535 {
 536         struct cxgbit_sock *csk = conn->context;
 537         u32 data_length = cmd->se_cmd.data_length;
 538         u32 padding = ((-data_length) & 3);
 539         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 540
 541         if ((data_length > mrdsl) && (!dr->recovery) &&
 542             (!padding) && (!datain->offset) && csk->max_iso_npdu) {
 543                 atomic_long_add(data_length - datain->length,
 544                                 &conn->sess->tx_data_octets);
 545                 return cxgbit_tx_datain_iso(csk, cmd, dr);
 546         }
 547
 548         return cxgbit_tx_datain(csk, cmd, datain);
 549 }
 550
 551 static int
 552 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 553                           const void *data_buf, u32 data_buf_len)
 554 {
 555         struct cxgbit_sock *csk = conn->context;
 556         struct sk_buff *skb;
 557         u32 padding = ((-data_buf_len) & 3);
 558
 559         skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
 560         if (unlikely(!skb))
 561                 return -ENOMEM;
 562
 563         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 564
 565         if (data_buf_len) {
 566                 u32 pad_bytes = 0;
 567
 568                 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
 569
 570                 if (padding)
 571                         skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
 572                                        &pad_bytes, padding);
 573         }
 574
 575         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
 576                                        cxgbit_skcb_submode(skb)];
 577
 578         return cxgbit_queue_skb(csk, skb);
 579 }
 580
 581 int
 582 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 583                 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
 584 {
 585         if (dr)
 586                 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
 587         else
 588                 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 589 }
 590
 591 int cxgbit_validate_params(struct iscsi_conn *conn)
 592 {
 593         struct cxgbit_sock *csk = conn->context;
 594         struct cxgbit_device *cdev = csk->com.cdev;
 595         struct iscsi_param *param;
 596         u32 max_xmitdsl;
 597
 598         param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
 599                                           conn->param_list);
 600         if (!param)
 601                 return -1;
 602
 603         if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
 604                 return -1;
 605
 606         if (max_xmitdsl > cdev->mdsl) {
 607                 if (iscsi_change_param_sprintf(
 608                         conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
 609                         return -1;
 610         }
 611
 612         return 0;
 613 }
 614
 615 static int cxgbit_set_digest(struct cxgbit_sock *csk)
 616 {
 617         struct iscsi_conn *conn = csk->conn;
 618         struct iscsi_param *param;
 619
 620         param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
 621         if (!param) {
 622                 pr_err("param not found key %s\n", HEADERDIGEST);
 623                 return -1;
 624         }
 625
 626         if (!strcmp(param->value, CRC32C))
 627                 csk->submode |= CXGBIT_SUBMODE_HCRC;
 628
 629         param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
 630         if (!param) {
 631                 csk->submode = 0;
 632                 pr_err("param not found key %s\n", DATADIGEST);
 633                 return -1;
 634         }
 635
 636         if (!strcmp(param->value, CRC32C))
 637                 csk->submode |= CXGBIT_SUBMODE_DCRC;
 638
 639         if (cxgbit_setup_conn_digest(csk)) {
 640                 csk->submode = 0;
 641                 return -1;
 642         }
 643
 644         return 0;
 645 }
 646
 647 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
 648 {
 649         struct iscsi_conn *conn = csk->conn;
 650         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 651         struct iscsi_param *param;
 652         u32 mrdsl, mbl;
 653         u32 max_npdu, max_iso_npdu;
 654         u32 max_iso_payload;
 655
 656         if (conn->login->leading_connection) {
 657                 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
 658                                                   conn->param_list);
 659                 if (!param) {
 660                         pr_err("param not found key %s\n", MAXBURSTLENGTH);
 661                         return -1;
 662                 }
 663
 664                 if (kstrtou32(param->value, 0, &mbl) < 0)
 665                         return -1;
 666         } else {
 667                 mbl = conn->sess->sess_ops->MaxBurstLength;
 668         }
 669
 670         mrdsl = conn_ops->MaxRecvDataSegmentLength;
 671         max_npdu = mbl / mrdsl;
 672
 673         max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
 674
 675         max_iso_npdu = max_iso_payload /
 676                        (ISCSI_HDR_LEN + mrdsl +
 677                         cxgbit_digest_len[csk->submode]);
 678
 679         csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
 680
 681         if (csk->max_iso_npdu <= 1)
 682                 csk->max_iso_npdu = 0;
 683
 684         return 0;
 685 }
 686
 687 /*
 688  * cxgbit_seq_pdu_inorder()
 689  * @csk: pointer to cxgbit socket structure
 690  *
 691  * This function checks whether data sequence and data
 692  * pdu are in order.
 693  *
 694  * Return: returns -1 on error, 0 if data sequence and
 695  * data pdu are in order, 1 if data sequence or data pdu
 696  * is not in order.
 697  */
 698 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
 699 {
 700         struct iscsi_conn *conn = csk->conn;
 701         struct iscsi_param *param;
 702
 703         if (conn->login->leading_connection) {
 704                 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
 705                                                   conn->param_list);
 706                 if (!param) {
 707                         pr_err("param not found key %s\n", DATASEQUENCEINORDER);
 708                         return -1;
 709                 }
 710
 711                 if (strcmp(param->value, YES))
 712                         return 1;
 713
 714                 param = iscsi_find_param_from_key(DATAPDUINORDER,
 715                                                   conn->param_list);
 716                 if (!param) {
 717                         pr_err("param not found key %s\n", DATAPDUINORDER);
 718                         return -1;
 719                 }
 720
 721                 if (strcmp(param->value, YES))
 722                         return 1;
 723
 724         } else {
 725                 if (!conn->sess->sess_ops->DataSequenceInOrder)
 726                         return 1;
 727                 if (!conn->sess->sess_ops->DataPDUInOrder)
 728                         return 1;
 729         }
 730
 731         return 0;
 732 }
 733
 734 static int cxgbit_set_params(struct iscsi_conn *conn)
 735 {
 736         struct cxgbit_sock *csk = conn->context;
 737         struct cxgbit_device *cdev = csk->com.cdev;
 738         struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
 739         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 740         struct iscsi_param *param;
 741         u8 erl;
 742
 743         if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
 744                 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
 745
 746         if (cxgbit_set_digest(csk))
 747                 return -1;
 748
 749         if (conn->login->leading_connection) {
 750                 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
 751                                                   conn->param_list);
 752                 if (!param) {
 753                         pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
 754                         return -1;
 755                 }
 756                 if (kstrtou8(param->value, 0, &erl) < 0)
 757                         return -1;
 758         } else {
 759                 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
 760         }
 761
 762         if (!erl) {
 763                 int ret;
 764
 765                 ret = cxgbit_seq_pdu_inorder(csk);
 766                 if (ret < 0) {
 767                         return -1;
 768                 } else if (ret > 0) {
 769                         if (is_t5(cdev->lldi.adapter_type))
 770                                 goto enable_ddp;
 771                         else
 772                                 return 0;
 773                 }
 774
 775                 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
 776                         if (cxgbit_set_iso_npdu(csk))
 777                                 return -1;
 778                 }
 779
 780 enable_ddp:
 781                 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
 782                         if (cxgbit_setup_conn_pgidx(csk,
 783                                                     ppm->tformat.pgsz_idx_dflt))
 784                                 return -1;
 785                         set_bit(CSK_DDP_ENABLE, &csk->com.flags);
 786                 }
 787         }
 788
 789         return 0;
 790 }
 791
 792 int
 793 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 794                     u32 length)
 795 {
 796         struct cxgbit_sock *csk = conn->context;
 797         struct sk_buff *skb;
 798         u32 padding_buf = 0;
 799         u8 padding = ((-length) & 3);
 800
 801         skb = cxgbit_alloc_skb(csk, length + padding);
 802         if (!skb)
 803                 return -ENOMEM;
 804         skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
 805         skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
 806
 807         if (padding)
 808                 skb_store_bits(skb, ISCSI_HDR_LEN + length,
 809                                &padding_buf, padding);
 810
 811         if (login->login_complete) {
 812                 if (cxgbit_set_params(conn)) {
 813                         kfree_skb(skb);
 814                         return -1;
 815                 }
 816
 817                 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
 818         }
 819
 820         if (cxgbit_queue_skb(csk, skb))
 821                 return -1;
 822
 823         if ((!login->login_complete) && (!login->login_failed))
 824                 schedule_delayed_work(&conn->login_work, 0);
 825
 826         return 0;
 827 }
 828
 829 static void
 830 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
 831                       unsigned int nents, u32 skip)
 832 {
 833         struct skb_seq_state st;
 834         const u8 *buf;
 835         unsigned int consumed = 0, buf_len;
 836         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
 837
 838         skb_prepare_seq_read(skb, pdu_cb->doffset,
 839                              pdu_cb->doffset + pdu_cb->dlen,
 840                              &st);
 841
 842         while (true) {
 843                 buf_len = skb_seq_read(consumed, &buf, &st);
 844                 if (!buf_len) {
 845                         skb_abort_seq_read(&st);
 846                         break;
 847                 }
 848
 849                 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
 850                                                  buf_len, skip + consumed);
 851         }
 852 }
 853
 854 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
 855 {
 856         struct iscsi_conn *conn = csk->conn;
 857         struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
 858         struct cxgbit_cmd *ccmd;
 859         struct iscsi_cmd *cmd;
 860
 861         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 862         if (!cmd) {
 863                 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
 864                 return NULL;
 865         }
 866
 867         ccmd = iscsit_priv_cmd(cmd);
 868         ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
 869         ccmd->setup_ddp = true;
 870
 871         return cmd;
 872 }
 873
 874 static int
 875 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 876                              u32 length)
 877 {
 878         struct iscsi_conn *conn = cmd->conn;
 879         struct cxgbit_sock *csk = conn->context;
 880         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 881
 882         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 883                 pr_err("ImmediateData CRC32C DataDigest error\n");
 884                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 885                         pr_err("Unable to recover from"
 886                                " Immediate Data digest failure while"
 887                                " in ERL=0.\n");
 888                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 889                                           (unsigned char *)hdr);
 890                         return IMMEDIATE_DATA_CANNOT_RECOVER;
 891                 }
 892
 893                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 894                                   (unsigned char *)hdr);
 895                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
 896         }
 897
 898         if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
 899                 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
 900                 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
 901                 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
 902
 903                 sg_init_table(&ccmd->sg, 1);
 904                 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
 905                             dfrag->page_offset);
 906                 get_page(dfrag->page.p);
 907
 908                 cmd->se_cmd.t_data_sg = &ccmd->sg;
 909                 cmd->se_cmd.t_data_nents = 1;
 910
 911                 ccmd->release = true;
 912         } else {
 913                 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
 914                 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
 915
 916                 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
 917         }
 918
 919         cmd->write_data_done += pdu_cb->dlen;
 920
 921         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 922                 spin_lock_bh(&cmd->istate_lock);
 923                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 924                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 925                 spin_unlock_bh(&cmd->istate_lock);
 926         }
 927
 928         return IMMEDIATE_DATA_NORMAL_OPERATION;
 929 }
 930
 931 static int
 932 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 933                           bool dump_payload)
 934 {
 935         struct iscsi_conn *conn = cmd->conn;
 936         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
 937         /*
 938          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 939          */
 940         if (dump_payload)
 941                 goto after_immediate_data;
 942
 943         immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
 944                                                  cmd->first_burst_len);
 945 after_immediate_data:
 946         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 947                 /*
 948                  * A PDU/CmdSN carrying Immediate Data passed
 949                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
 950                  * Immediate Bit is not set.
 951                  */
 952                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
 953                                                 (unsigned char *)hdr,
 954                                                 hdr->cmdsn);
 955                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
 956                         return -1;
 957
 958                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
 959                         target_put_sess_cmd(&cmd->se_cmd);
 960                         return 0;
 961                 } else if (cmd->unsolicited_data) {
 962                         iscsit_set_unsoliticed_dataout(cmd);
 963                 }
 964
 965         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
 966                 /*
 967                  * Immediate Data failed DataCRC and ERL>=1,
 968                  * silently drop this PDU and let the initiator
 969                  * plug the CmdSN gap.
 970                  *
 971                  * FIXME: Send Unsolicited NOPIN with reserved
 972                  * TTT here to help the initiator figure out
 973                  * the missing CmdSN, although they should be
 974                  * intelligent enough to determine the missing
 975                  * CmdSN and issue a retry to plug the sequence.
 976                  */
 977                 cmd->i_state = ISTATE_REMOVE;
 978                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 979         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
 980                 return -1;
 981
 982         return 0;
 983 }
 984
 985 static int
 986 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
 987 {
 988         struct iscsi_conn *conn = csk->conn;
 989         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 990         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
 991         int rc;
 992         bool dump_payload = false;
 993
 994         rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
 995         if (rc < 0)
 996                 return rc;
 997
 998         if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
 999             (pdu_cb->nr_dfrags == 1))
1000                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1001
1002         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1003         if (rc < 0)
1004                 return 0;
1005         else if (rc > 0)
1006                 dump_payload = true;
1007
1008         if (!pdu_cb->dlen)
1009                 return 0;
1010
1011         return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
1012 }
1013
1014 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
1015 {
1016         struct scatterlist *sg_start;
1017         struct iscsi_conn *conn = csk->conn;
1018         struct iscsi_cmd *cmd = NULL;
1019         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1020         struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1021         u32 data_offset = be32_to_cpu(hdr->offset);
1022         u32 data_len = pdu_cb->dlen;
1023         int rc, sg_nents, sg_off;
1024         bool dcrc_err = false;
1025
1026         if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1027                 u32 offset = be32_to_cpu(hdr->offset);
1028                 u32 ddp_data_len;
1029                 u32 payload_length = ntoh24(hdr->dlength);
1030                 bool success = false;
1031
1032                 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1033                 if (!cmd)
1034                         return 0;
1035
1036                 ddp_data_len = offset - cmd->write_data_done;
1037                 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1038
1039                 cmd->write_data_done = offset;
1040                 cmd->next_burst_len = ddp_data_len;
1041                 cmd->data_sn = be32_to_cpu(hdr->datasn);
1042
1043                 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1044                                                 cmd, payload_length, &success);
1045                 if (rc < 0)
1046                         return rc;
1047                 else if (!success)
1048                         return 0;
1049         } else {
1050                 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1051                 if (rc < 0)
1052                         return rc;
1053                 else if (!cmd)
1054                         return 0;
1055         }
1056
1057         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1058                 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1059                        " DataSN: 0x%08x\n",
1060                        hdr->itt, hdr->offset, data_len,
1061                        hdr->datasn);
1062
1063                 dcrc_err = true;
1064                 goto check_payload;
1065         }
1066
1067         pr_debug("DataOut data_len: %u, "
1068                 "write_data_done: %u, data_length: %u\n",
1069                   data_len,  cmd->write_data_done,
1070                   cmd->se_cmd.data_length);
1071
1072         if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1073                 u32 skip = data_offset % PAGE_SIZE;
1074
1075                 sg_off = data_offset / PAGE_SIZE;
1076                 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1077                 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1078
1079                 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1080         }
1081
1082 check_payload:
1083
1084         rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1085         if (rc < 0)
1086                 return rc;
1087
1088         return 0;
1089 }
1090
1091 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1092 {
1093         struct iscsi_conn *conn = csk->conn;
1094         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1095         struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1096         unsigned char *ping_data = NULL;
1097         u32 payload_length = pdu_cb->dlen;
1098         int ret;
1099
1100         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1101         if (ret < 0)
1102                 return 0;
1103
1104         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1105                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1106                         pr_err("Unable to recover from"
1107                                " NOPOUT Ping DataCRC failure while in"
1108                                " ERL=0.\n");
1109                         ret = -1;
1110                         goto out;
1111                 } else {
1112                         /*
1113                          * drop this PDU and let the
1114                          * initiator plug the CmdSN gap.
1115                          */
1116                         pr_info("Dropping NOPOUT"
1117                                 " Command CmdSN: 0x%08x due to"
1118                                 " DataCRC error.\n", hdr->cmdsn);
1119                         ret = 0;
1120                         goto out;
1121                 }
1122         }
1123
1124         /*
1125          * Handle NOP-OUT payload for traditional iSCSI sockets
1126          */
1127         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1128                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1129                 if (!ping_data) {
1130                         pr_err("Unable to allocate memory for"
1131                                 " NOPOUT ping data.\n");
1132                         ret = -1;
1133                         goto out;
1134                 }
1135
1136                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1137                               ping_data, payload_length);
1138
1139                 ping_data[payload_length] = '\0';
1140                 /*
1141                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1142                  */
1143                 cmd->buf_ptr = ping_data;
1144                 cmd->buf_ptr_size = payload_length;
1145
1146                 pr_debug("Got %u bytes of NOPOUT ping"
1147                         " data.\n", payload_length);
1148                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1149         }
1150
1151         return iscsit_process_nop_out(conn, cmd, hdr);
1152 out:
1153         if (cmd)
1154                 iscsit_free_cmd(cmd, false);
1155         return ret;
1156 }
1157
1158 static int
1159 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1160 {
1161         struct iscsi_conn *conn = csk->conn;
1162         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1163         struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1164         u32 payload_length = pdu_cb->dlen;
1165         int rc;
1166         unsigned char *text_in = NULL;
1167
1168         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1169         if (rc < 0)
1170                 return rc;
1171
1172         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1173                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1174                         pr_err("Unable to recover from"
1175                                " Text Data digest failure while in"
1176                                " ERL=0.\n");
1177                         goto reject;
1178                 } else {
1179                         /*
1180                          * drop this PDU and let the
1181                          * initiator plug the CmdSN gap.
1182                          */
1183                         pr_info("Dropping Text"
1184                                 " Command CmdSN: 0x%08x due to"
1185                                 " DataCRC error.\n", hdr->cmdsn);
1186                         return 0;
1187                 }
1188         }
1189
1190         if (payload_length) {
1191                 text_in = kzalloc(payload_length, GFP_KERNEL);
1192                 if (!text_in) {
1193                         pr_err("Unable to allocate text_in of payload_length: %u\n",
1194                                payload_length);
1195                         return -ENOMEM;
1196                 }
1197                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1198                               text_in, payload_length);
1199
1200                 text_in[payload_length - 1] = '\0';
1201
1202                 cmd->text_in_ptr = text_in;
1203         }
1204
1205         return iscsit_process_text_cmd(conn, cmd, hdr);
1206
1207 reject:
1208         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1209                                  pdu_cb->hdr);
1210 }
1211
1212 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1213 {
1214         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1215         struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1216         struct iscsi_conn *conn = csk->conn;
1217         struct iscsi_cmd *cmd = NULL;
1218         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1219         int ret = -EINVAL;
1220
1221         switch (opcode) {
1222         case ISCSI_OP_SCSI_CMD:
1223                 cmd = cxgbit_allocate_cmd(csk);
1224                 if (!cmd)
1225                         goto reject;
1226
1227                 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1228                 break;
1229         case ISCSI_OP_SCSI_DATA_OUT:
1230                 ret = cxgbit_handle_iscsi_dataout(csk);
1231                 break;
1232         case ISCSI_OP_NOOP_OUT:
1233                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1234                         cmd = cxgbit_allocate_cmd(csk);
1235                         if (!cmd)
1236                                 goto reject;
1237                 }
1238
1239                 ret = cxgbit_handle_nop_out(csk, cmd);
1240                 break;
1241         case ISCSI_OP_SCSI_TMFUNC:
1242                 cmd = cxgbit_allocate_cmd(csk);
1243                 if (!cmd)
1244                         goto reject;
1245
1246                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1247                                                  (unsigned char *)hdr);
1248                 break;
1249         case ISCSI_OP_TEXT:
1250                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1251                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1252                         if (!cmd)
1253                                 goto reject;
1254                 } else {
1255                         cmd = cxgbit_allocate_cmd(csk);
1256                         if (!cmd)
1257                                 goto reject;
1258                 }
1259
1260                 ret = cxgbit_handle_text_cmd(csk, cmd);
1261                 break;
1262         case ISCSI_OP_LOGOUT:
1263                 cmd = cxgbit_allocate_cmd(csk);
1264                 if (!cmd)
1265                         goto reject;
1266
1267                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1268                 if (ret > 0)
1269                         wait_for_completion_timeout(&conn->conn_logout_comp,
1270                                                     SECONDS_FOR_LOGOUT_COMP
1271                                                     * HZ);
1272                 break;
1273         case ISCSI_OP_SNACK:
1274                 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1275                 break;
1276         default:
1277                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1278                 dump_stack();
1279                 break;
1280         }
1281
1282         return ret;
1283
1284 reject:
1285         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1286                                  (unsigned char *)hdr);
1287         return ret;
1288 }
1289
1290 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1291 {
1292         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1293         struct iscsi_conn *conn = csk->conn;
1294         struct iscsi_hdr *hdr = pdu_cb->hdr;
1295         u8 opcode;
1296
1297         if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1298                 atomic_long_inc(&conn->sess->conn_digest_errors);
1299                 goto transport_err;
1300         }
1301
1302         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1303                 goto transport_err;
1304
1305         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1306
1307         if (conn->sess->sess_ops->SessionType &&
1308             ((!(opcode & ISCSI_OP_TEXT)) ||
1309              (!(opcode & ISCSI_OP_LOGOUT)))) {
1310                 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1311                         " while in Discovery Session, rejecting.\n", opcode);
1312                 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1313                                   (unsigned char *)hdr);
1314                 goto transport_err;
1315         }
1316
1317         if (cxgbit_target_rx_opcode(csk) < 0)
1318                 goto transport_err;
1319
1320         return 0;
1321
1322 transport_err:
1323         return -1;
1324 }
1325
1326 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1327 {
1328         struct iscsi_conn *conn = csk->conn;
1329         struct iscsi_login *login = conn->login;
1330         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1331         struct iscsi_login_req *login_req;
1332
1333         login_req = (struct iscsi_login_req *)login->req;
1334         memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1335
1336         pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1337                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1338                 login_req->flags, login_req->itt, login_req->cmdsn,
1339                 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1340         /*
1341          * Setup the initial iscsi_login values from the leading
1342          * login request PDU.
1343          */
1344         if (login->first_request) {
1345                 login_req = (struct iscsi_login_req *)login->req;
1346                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1347                 login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(
1348                                 login_req->flags);
1349                 login->version_min      = login_req->min_version;
1350                 login->version_max      = login_req->max_version;
1351                 memcpy(login->isid, login_req->isid, 6);
1352                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1353                 login->init_task_tag    = login_req->itt;
1354                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1355                 login->cid              = be16_to_cpu(login_req->cid);
1356                 login->tsih             = be16_to_cpu(login_req->tsih);
1357         }
1358
1359         if (iscsi_target_check_login_request(conn, login) < 0)
1360                 return -1;
1361
1362         memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1363         skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1364
1365         return 0;
1366 }
1367
1368 static int
1369 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1370 {
1371         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1372         int ret;
1373
1374         cxgbit_rx_pdu_cb(skb) = pdu_cb;
1375
1376         csk->skb = skb;
1377
1378         if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1379                 ret = cxgbit_rx_login_pdu(csk);
1380                 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1381         } else {
1382                 ret = cxgbit_rx_opcode(csk);
1383         }
1384
1385         return ret;
1386 }
1387
1388 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1389 {
1390         struct skb_shared_info *ssi = skb_shinfo(skb);
1391         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1392         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1393         u8 i;
1394
1395         pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1396                 skb, skb->head, skb->data, skb->len, skb->data_len,
1397                 ssi->nr_frags);
1398         pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1399                 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1400
1401         for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1402                 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1403                         "frags %u.\n",
1404                         skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1405                         pdu_cb->ddigest, pdu_cb->frags);
1406         for (i = 0; i < ssi->nr_frags; i++)
1407                 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1408                         skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1409 }
1410
1411 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1412 {
1413         struct sk_buff *skb = csk->lro_hskb;
1414         struct skb_shared_info *ssi = skb_shinfo(skb);
1415         u8 i;
1416
1417         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1418         for (i = 0; i < ssi->nr_frags; i++)
1419                 put_page(skb_frag_page(&ssi->frags[i]));
1420         ssi->nr_frags = 0;
1421         skb->data_len = 0;
1422         skb->truesize -= skb->len;
1423         skb->len = 0;
1424 }
1425
1426 static void
1427 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1428 {
1429         struct sk_buff *hskb = csk->lro_hskb;
1430         struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1431         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1432         struct skb_shared_info *hssi = skb_shinfo(hskb);
1433         struct skb_shared_info *ssi = skb_shinfo(skb);
1434         unsigned int len = 0;
1435
1436         if (pdu_cb->flags & PDUCBF_RX_HDR) {
1437                 u8 hfrag_idx = hssi->nr_frags;
1438
1439                 hpdu_cb->flags |= pdu_cb->flags;
1440                 hpdu_cb->seq = pdu_cb->seq;
1441                 hpdu_cb->hdr = pdu_cb->hdr;
1442                 hpdu_cb->hlen = pdu_cb->hlen;
1443
1444                 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1445                        sizeof(skb_frag_t));
1446
1447                 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1448                 hssi->nr_frags++;
1449                 hpdu_cb->frags++;
1450                 hpdu_cb->hfrag_idx = hfrag_idx;
1451
1452                 len = hssi->frags[hfrag_idx].size;
1453                 hskb->len += len;
1454                 hskb->data_len += len;
1455                 hskb->truesize += len;
1456         }
1457
1458         if (pdu_cb->flags & PDUCBF_RX_DATA) {
1459                 u8 dfrag_idx = hssi->nr_frags, i;
1460
1461                 hpdu_cb->flags |= pdu_cb->flags;
1462                 hpdu_cb->dfrag_idx = dfrag_idx;
1463
1464                 len = 0;
1465                 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1466                         memcpy(&hssi->frags[dfrag_idx],
1467                                &ssi->frags[pdu_cb->dfrag_idx + i],
1468                                sizeof(skb_frag_t));
1469
1470                         get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1471
1472                         len += hssi->frags[dfrag_idx].size;
1473
1474                         hssi->nr_frags++;
1475                         hpdu_cb->frags++;
1476                 }
1477
1478                 hpdu_cb->dlen = pdu_cb->dlen;
1479                 hpdu_cb->doffset = hpdu_cb->hlen;
1480                 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1481                 hskb->len += len;
1482                 hskb->data_len += len;
1483                 hskb->truesize += len;
1484         }
1485
1486         if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1487                 hpdu_cb->flags |= pdu_cb->flags;
1488
1489                 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1490                         hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1491
1492                 hpdu_cb->ddigest = pdu_cb->ddigest;
1493                 hpdu_cb->pdulen = pdu_cb->pdulen;
1494         }
1495 }
1496
1497 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1498 {
1499         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1500         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1501         u8 pdu_idx = 0, last_idx = 0;
1502         int ret = 0;
1503
1504         if (!pdu_cb->complete) {
1505                 cxgbit_lro_skb_merge(csk, skb, 0);
1506
1507                 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1508                         struct sk_buff *hskb = csk->lro_hskb;
1509
1510                         ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1511
1512                         cxgbit_lro_hskb_reset(csk);
1513
1514                         if (ret < 0)
1515                                 goto out;
1516                 }
1517
1518                 pdu_idx = 1;
1519         }
1520
1521         if (lro_cb->pdu_idx)
1522                 last_idx = lro_cb->pdu_idx - 1;
1523
1524         for (; pdu_idx <= last_idx; pdu_idx++) {
1525                 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1526                 if (ret < 0)
1527                         goto out;
1528         }
1529
1530         if ((!lro_cb->complete) && lro_cb->pdu_idx)
1531                 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1532
1533 out:
1534         return ret;
1535 }
1536
1537 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1538 {
1539         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1540         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1541         int ret = -1;
1542
1543         if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1544             (pdu_cb->seq != csk->rcv_nxt)) {
1545                 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1546                         csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1547                 cxgbit_lro_skb_dump(skb);
1548                 return ret;
1549         }
1550
1551         csk->rcv_nxt += lro_cb->pdu_totallen;
1552
1553         ret = cxgbit_process_lro_skb(csk, skb);
1554
1555         csk->rx_credits += lro_cb->pdu_totallen;
1556
1557         if (csk->rx_credits >= (csk->rcv_win / 4))
1558                 cxgbit_rx_data_ack(csk);
1559
1560         return ret;
1561 }
1562
1563 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1564 {
1565         struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1566         int ret = -1;
1567
1568         if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1569                 if (is_t5(lldi->adapter_type))
1570                         ret = cxgbit_rx_lro_skb(csk, skb);
1571                 else
1572                         ret = cxgbit_process_lro_skb(csk, skb);
1573         }
1574
1575         __kfree_skb(skb);
1576         return ret;
1577 }
1578
1579 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1580 {
1581         spin_lock_bh(&csk->rxq.lock);
1582         if (skb_queue_len(&csk->rxq)) {
1583                 skb_queue_splice_init(&csk->rxq, rxq);
1584                 spin_unlock_bh(&csk->rxq.lock);
1585                 return true;
1586         }
1587         spin_unlock_bh(&csk->rxq.lock);
1588         return false;
1589 }
1590
1591 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1592 {
1593         struct sk_buff *skb;
1594         struct sk_buff_head rxq;
1595
1596         skb_queue_head_init(&rxq);
1597
1598         wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1599
1600         if (signal_pending(current))
1601                 goto out;
1602
1603         while ((skb = __skb_dequeue(&rxq))) {
1604                 if (cxgbit_rx_skb(csk, skb))
1605                         goto out;
1606         }
1607
1608         return 0;
1609 out:
1610         __skb_queue_purge(&rxq);
1611         return -1;
1612 }
1613
1614 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1615 {
1616         struct cxgbit_sock *csk = conn->context;
1617         int ret = -1;
1618
1619         while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1620                 ret = cxgbit_wait_rxq(csk);
1621                 if (ret) {
1622                         clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1623                         break;
1624                 }
1625         }
1626
1627         return ret;
1628 }
1629
1630 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1631 {
1632         struct cxgbit_sock *csk = conn->context;
1633
1634         while (!kthread_should_stop()) {
1635                 iscsit_thread_check_cpumask(conn, current, 0);
1636                 if (cxgbit_wait_rxq(csk))
1637                         return;
1638         }
1639 }