drivers/target/iscsi/cxgbit/cxgbit_target.c

   1 /*
   2  * Copyright (c) 2016 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #include <linux/workqueue.h>
  10 #include <linux/kthread.h>
  11 #include <linux/sched/signal.h>
  12
  13 #include <asm/unaligned.h>
  14 #include <net/tcp.h>
  15 #include <target/target_core_base.h>
  16 #include <target/target_core_fabric.h>
  17 #include "cxgbit.h"
  18
  19 struct sge_opaque_hdr {
  20         void *dev;
  21         dma_addr_t addr[MAX_SKB_FRAGS + 1];
  22 };
  23
  24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
  25
  26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
  27                     sizeof(struct fw_ofld_tx_data_wr))
  28
  29 static struct sk_buff *
  30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
  31 {
  32         struct sk_buff *skb = NULL;
  33         u8 submode = 0;
  34         int errcode;
  35         static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
  36
  37         if (len) {
  38                 skb = alloc_skb_with_frags(hdr_len, len,
  39                                            0, &errcode,
  40                                            GFP_KERNEL);
  41                 if (!skb)
  42                         return NULL;
  43
  44                 skb_reserve(skb, TX_HDR_LEN);
  45                 skb_reset_transport_header(skb);
  46                 __skb_put(skb, ISCSI_HDR_LEN);
  47                 skb->data_len = len;
  48                 skb->len += len;
  49                 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
  50
  51         } else {
  52                 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
  53
  54                 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
  55                 if (!skb)
  56                         return NULL;
  57
  58                 skb_reserve(skb, TX_HDR_LEN + iso_len);
  59                 skb_reset_transport_header(skb);
  60                 __skb_put(skb, ISCSI_HDR_LEN);
  61         }
  62
  63         submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
  64         cxgbit_skcb_submode(skb) = submode;
  65         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
  66         cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
  67         return skb;
  68 }
  69
  70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
  71 {
  72         return __cxgbit_alloc_skb(csk, len, false);
  73 }
  74
  75 /*
  76  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
  77  * @skb: the packet
  78  *
  79  * Returns true if a packet can be sent as an offload WR with immediate
  80  * data.  We currently use the same limit as for Ethernet packets.
  81  */
  82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
  83 {
  84         int length = skb->len;
  85
  86         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
  87                 length += sizeof(struct fw_ofld_tx_data_wr);
  88
  89         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
  90                 length += sizeof(struct cpl_tx_data_iso);
  91
  92 #define MAX_IMM_TX_PKT_LEN      256
  93         return length <= MAX_IMM_TX_PKT_LEN;
  94 }
  95
  96 /*
  97  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
  98  * @n: the number of SGL entries
  99  * Calculates the number of flits needed for a scatter/gather list that
 100  * can hold the given number of entries.
 101  */
 102 static inline unsigned int cxgbit_sgl_len(unsigned int n)
 103 {
 104         n--;
 105         return (3 * n) / 2 + (n & 1) + 2;
 106 }
 107
 108 /*
 109  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
 110  * @skb: the packet
 111  *
 112  * Returns the number of flits needed for the given offload packet.
 113  * These packets are already fully constructed and no additional headers
 114  * will be added.
 115  */
 116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
 117 {
 118         unsigned int flits, cnt;
 119
 120         if (cxgbit_is_ofld_imm(skb))
 121                 return DIV_ROUND_UP(skb->len, 8);
 122         flits = skb_transport_offset(skb) / 8;
 123         cnt = skb_shinfo(skb)->nr_frags;
 124         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 125                 cnt++;
 126         return flits + cxgbit_sgl_len(cnt);
 127 }
 128
 129 #define CXGBIT_ISO_FSLICE 0x1
 130 #define CXGBIT_ISO_LSLICE 0x2
 131 static void
 132 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
 133 {
 134         struct cpl_tx_data_iso *cpl;
 135         unsigned int submode = cxgbit_skcb_submode(skb);
 136         unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
 137         unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
 138
 139         cpl = __skb_push(skb, sizeof(*cpl));
 140
 141         cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
 142                         CPL_TX_DATA_ISO_FIRST_V(fslice) |
 143                         CPL_TX_DATA_ISO_LAST_V(lslice) |
 144                         CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
 145                         CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
 146                         CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
 147                         CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
 148                         CPL_TX_DATA_ISO_SCSI_V(2));
 149
 150         cpl->ahs_len = 0;
 151         cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
 152         cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
 153         cpl->len = htonl(iso_info->len);
 154         cpl->reserved2_seglen_offset = htonl(0);
 155         cpl->datasn_offset = htonl(0);
 156         cpl->buffer_offset = htonl(0);
 157         cpl->reserved3 = 0;
 158
 159         __skb_pull(skb, sizeof(*cpl));
 160 }
 161
 162 static void
 163 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
 164                   u32 len, u32 credits, u32 compl)
 165 {
 166         struct fw_ofld_tx_data_wr *req;
 167         const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
 168         u32 submode = cxgbit_skcb_submode(skb);
 169         u32 wr_ulp_mode = 0;
 170         u32 hdr_size = sizeof(*req);
 171         u32 opcode = FW_OFLD_TX_DATA_WR;
 172         u32 immlen = 0;
 173         u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
 174                     T6_TX_FORCE_F;
 175
 176         if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
 177                 opcode = FW_ISCSI_TX_DATA_WR;
 178                 immlen += sizeof(struct cpl_tx_data_iso);
 179                 hdr_size += sizeof(struct cpl_tx_data_iso);
 180                 submode |= 8;
 181         }
 182
 183         if (cxgbit_is_ofld_imm(skb))
 184                 immlen += dlen;
 185
 186         req = __skb_push(skb, hdr_size);
 187         req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
 188                                         FW_WR_COMPL_V(compl) |
 189                                         FW_WR_IMMDLEN_V(immlen));
 190         req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 191                                         FW_WR_LEN16_V(credits));
 192         req->plen = htonl(len);
 193         wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
 194                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 195
 196         req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
 197                  FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
 198 }
 199
 200 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 201 {
 202         kfree_skb(skb);
 203 }
 204
 205 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
 206 {
 207         struct sk_buff *skb;
 208
 209         while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
 210                 u32 dlen = skb->len;
 211                 u32 len = skb->len;
 212                 u32 credits_needed;
 213                 u32 compl = 0;
 214                 u32 flowclen16 = 0;
 215                 u32 iso_cpl_len = 0;
 216
 217                 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
 218                         iso_cpl_len = sizeof(struct cpl_tx_data_iso);
 219
 220                 if (cxgbit_is_ofld_imm(skb))
 221                         credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
 222                 else
 223                         credits_needed = DIV_ROUND_UP((8 *
 224                                         cxgbit_calc_tx_flits_ofld(skb)) +
 225                                         iso_cpl_len, 16);
 226
 227                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
 228                         credits_needed += DIV_ROUND_UP(
 229                                 sizeof(struct fw_ofld_tx_data_wr), 16);
 230                 /*
 231                  * Assumes the initial credits is large enough to support
 232                  * fw_flowc_wr plus largest possible first payload
 233                  */
 234
 235                 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
 236                         flowclen16 = cxgbit_send_tx_flowc_wr(csk);
 237                         csk->wr_cred -= flowclen16;
 238                         csk->wr_una_cred += flowclen16;
 239                 }
 240
 241                 if (csk->wr_cred < credits_needed) {
 242                         pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
 243                                  csk, skb->len, skb->data_len,
 244                                  credits_needed, csk->wr_cred);
 245                         break;
 246                 }
 247                 __skb_unlink(skb, &csk->txq);
 248                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
 249                 skb->csum = (__force __wsum)(credits_needed + flowclen16);
 250                 csk->wr_cred -= credits_needed;
 251                 csk->wr_una_cred += credits_needed;
 252
 253                 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 254                          csk, skb->len, skb->data_len, credits_needed,
 255                          csk->wr_cred, csk->wr_una_cred);
 256
 257                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
 258                         len += cxgbit_skcb_tx_extralen(skb);
 259
 260                         if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
 261                             (!before(csk->write_seq,
 262                                      csk->snd_una + csk->snd_win))) {
 263                                 compl = 1;
 264                                 csk->wr_una_cred = 0;
 265                         }
 266
 267                         cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
 268                                           compl);
 269                         csk->snd_nxt += len;
 270
 271                 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
 272                            (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
 273                         struct cpl_close_con_req *req =
 274                                 (struct cpl_close_con_req *)skb->data;
 275                         req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
 276                         csk->wr_una_cred = 0;
 277                 }
 278
 279                 cxgbit_sock_enqueue_wr(csk, skb);
 280                 t4_set_arp_err_handler(skb, csk,
 281                                        cxgbit_arp_failure_skb_discard);
 282
 283                 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
 284                          csk, csk->tid, skb, len);
 285
 286                 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 287         }
 288 }
 289
 290 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
 291 {
 292         spin_lock_bh(&csk->lock);
 293
 294         if (before(csk->write_seq, csk->snd_una + csk->snd_win))
 295                 csk->lock_owner = true;
 296
 297         spin_unlock_bh(&csk->lock);
 298
 299         return csk->lock_owner;
 300 }
 301
 302 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
 303 {
 304         struct sk_buff_head backlogq;
 305         struct sk_buff *skb;
 306         void (*fn)(struct cxgbit_sock *, struct sk_buff *);
 307
 308         skb_queue_head_init(&backlogq);
 309
 310         spin_lock_bh(&csk->lock);
 311         while (skb_queue_len(&csk->backlogq)) {
 312                 skb_queue_splice_init(&csk->backlogq, &backlogq);
 313                 spin_unlock_bh(&csk->lock);
 314
 315                 while ((skb = __skb_dequeue(&backlogq))) {
 316                         fn = cxgbit_skcb_rx_backlog_fn(skb);
 317                         fn(csk, skb);
 318                 }
 319
 320                 spin_lock_bh(&csk->lock);
 321         }
 322
 323         csk->lock_owner = false;
 324         spin_unlock_bh(&csk->lock);
 325 }
 326
 327 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 328 {
 329         int ret = 0;
 330
 331         wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
 332
 333         if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
 334                      signal_pending(current))) {
 335                 __kfree_skb(skb);
 336                 __skb_queue_purge(&csk->ppodq);
 337                 ret = -1;
 338                 spin_lock_bh(&csk->lock);
 339                 if (csk->lock_owner) {
 340                         spin_unlock_bh(&csk->lock);
 341                         goto unlock;
 342                 }
 343                 spin_unlock_bh(&csk->lock);
 344                 return ret;
 345         }
 346
 347         csk->write_seq += skb->len +
 348                           cxgbit_skcb_tx_extralen(skb);
 349
 350         skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
 351         __skb_queue_tail(&csk->txq, skb);
 352         cxgbit_push_tx_frames(csk);
 353
 354 unlock:
 355         cxgbit_unlock_sock(csk);
 356         return ret;
 357 }
 358
 359 static int
 360 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
 361                u32 data_length)
 362 {
 363         u32 i = 0, nr_frags = MAX_SKB_FRAGS;
 364         u32 padding = ((-data_length) & 3);
 365         struct scatterlist *sg;
 366         struct page *page;
 367         unsigned int page_off;
 368
 369         if (padding)
 370                 nr_frags--;
 371
 372         /*
 373          * We know each entry in t_data_sg contains a page.
 374          */
 375         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 376         page_off = (data_offset % PAGE_SIZE);
 377
 378         while (data_length && (i < nr_frags)) {
 379                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 380
 381                 page = sg_page(sg);
 382
 383                 get_page(page);
 384                 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
 385                                    cur_len);
 386                 skb->data_len += cur_len;
 387                 skb->len += cur_len;
 388                 skb->truesize += cur_len;
 389
 390                 data_length -= cur_len;
 391                 page_off = 0;
 392                 sg = sg_next(sg);
 393                 i++;
 394         }
 395
 396         if (data_length)
 397                 return -1;
 398
 399         if (padding) {
 400                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 401                 if (!page)
 402                         return -1;
 403                 skb_fill_page_desc(skb, i, page, 0, padding);
 404                 skb->data_len += padding;
 405                 skb->len += padding;
 406                 skb->truesize += padding;
 407         }
 408
 409         return 0;
 410 }
 411
 412 static int
 413 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 414                      struct iscsi_datain_req *dr)
 415 {
 416         struct iscsi_conn *conn = csk->conn;
 417         struct sk_buff *skb;
 418         struct iscsi_datain datain;
 419         struct cxgbit_iso_info iso_info;
 420         u32 data_length = cmd->se_cmd.data_length;
 421         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 422         u32 num_pdu, plen, tx_data = 0;
 423         bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
 424                 SCF_TRANSPORT_TASK_SENSE);
 425         bool set_statsn = false;
 426         int ret = -1;
 427
 428         while (data_length) {
 429                 num_pdu = (data_length + mrdsl - 1) / mrdsl;
 430                 if (num_pdu > csk->max_iso_npdu)
 431                         num_pdu = csk->max_iso_npdu;
 432
 433                 plen = num_pdu * mrdsl;
 434                 if (plen > data_length)
 435                         plen = data_length;
 436
 437                 skb = __cxgbit_alloc_skb(csk, 0, true);
 438                 if (unlikely(!skb))
 439                         return -ENOMEM;
 440
 441                 memset(skb->data, 0, ISCSI_HDR_LEN);
 442                 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
 443                 cxgbit_skcb_submode(skb) |= (csk->submode &
 444                                 CXGBIT_SUBMODE_DCRC);
 445                 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
 446                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
 447                                                 ((num_pdu - 1) * ISCSI_HDR_LEN);
 448
 449                 memset(&datain, 0, sizeof(struct iscsi_datain));
 450                 memset(&iso_info, 0, sizeof(iso_info));
 451
 452                 if (!tx_data)
 453                         iso_info.flags |= CXGBIT_ISO_FSLICE;
 454
 455                 if (!(data_length - plen)) {
 456                         iso_info.flags |= CXGBIT_ISO_LSLICE;
 457                         if (!task_sense) {
 458                                 datain.flags = ISCSI_FLAG_DATA_STATUS;
 459                                 iscsit_increment_maxcmdsn(cmd, conn->sess);
 460                                 cmd->stat_sn = conn->stat_sn++;
 461                                 set_statsn = true;
 462                         }
 463                 }
 464
 465                 iso_info.burst_len = num_pdu * mrdsl;
 466                 iso_info.mpdu = mrdsl;
 467                 iso_info.len = ISCSI_HDR_LEN + plen;
 468
 469                 cxgbit_cpl_tx_data_iso(skb, &iso_info);
 470
 471                 datain.offset = tx_data;
 472                 datain.data_sn = cmd->data_sn - 1;
 473
 474                 iscsit_build_datain_pdu(cmd, conn, &datain,
 475                                         (struct iscsi_data_rsp *)skb->data,
 476                                         set_statsn);
 477
 478                 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
 479                 if (unlikely(ret)) {
 480                         __kfree_skb(skb);
 481                         goto out;
 482                 }
 483
 484                 ret = cxgbit_queue_skb(csk, skb);
 485                 if (unlikely(ret))
 486                         goto out;
 487
 488                 tx_data += plen;
 489                 data_length -= plen;
 490
 491                 cmd->read_data_done += plen;
 492                 cmd->data_sn += num_pdu;
 493         }
 494
 495         dr->dr_complete = DATAIN_COMPLETE_NORMAL;
 496
 497         return 0;
 498
 499 out:
 500         return ret;
 501 }
 502
 503 static int
 504 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 505                  const struct iscsi_datain *datain)
 506 {
 507         struct sk_buff *skb;
 508         int ret = 0;
 509
 510         skb = cxgbit_alloc_skb(csk, 0);
 511         if (unlikely(!skb))
 512                 return -ENOMEM;
 513
 514         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 515
 516         if (datain->length) {
 517                 cxgbit_skcb_submode(skb) |= (csk->submode &
 518                                 CXGBIT_SUBMODE_DCRC);
 519                 cxgbit_skcb_tx_extralen(skb) =
 520                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
 521         }
 522
 523         ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
 524         if (ret < 0) {
 525                 __kfree_skb(skb);
 526                 return ret;
 527         }
 528
 529         return cxgbit_queue_skb(csk, skb);
 530 }
 531
 532 static int
 533 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 534                        struct iscsi_datain_req *dr,
 535                        const struct iscsi_datain *datain)
 536 {
 537         struct cxgbit_sock *csk = conn->context;
 538         u32 data_length = cmd->se_cmd.data_length;
 539         u32 padding = ((-data_length) & 3);
 540         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 541
 542         if ((data_length > mrdsl) && (!dr->recovery) &&
 543             (!padding) && (!datain->offset) && csk->max_iso_npdu) {
 544                 atomic_long_add(data_length - datain->length,
 545                                 &conn->sess->tx_data_octets);
 546                 return cxgbit_tx_datain_iso(csk, cmd, dr);
 547         }
 548
 549         return cxgbit_tx_datain(csk, cmd, datain);
 550 }
 551
 552 static int
 553 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 554                           const void *data_buf, u32 data_buf_len)
 555 {
 556         struct cxgbit_sock *csk = conn->context;
 557         struct sk_buff *skb;
 558         u32 padding = ((-data_buf_len) & 3);
 559
 560         skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
 561         if (unlikely(!skb))
 562                 return -ENOMEM;
 563
 564         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 565
 566         if (data_buf_len) {
 567                 u32 pad_bytes = 0;
 568
 569                 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
 570
 571                 if (padding)
 572                         skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
 573                                        &pad_bytes, padding);
 574         }
 575
 576         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
 577                                        cxgbit_skcb_submode(skb)];
 578
 579         return cxgbit_queue_skb(csk, skb);
 580 }
 581
 582 int
 583 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 584                 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
 585 {
 586         if (dr)
 587                 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
 588         else
 589                 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 590 }
 591
 592 int cxgbit_validate_params(struct iscsi_conn *conn)
 593 {
 594         struct cxgbit_sock *csk = conn->context;
 595         struct cxgbit_device *cdev = csk->com.cdev;
 596         struct iscsi_param *param;
 597         u32 max_xmitdsl;
 598
 599         param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
 600                                           conn->param_list);
 601         if (!param)
 602                 return -1;
 603
 604         if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
 605                 return -1;
 606
 607         if (max_xmitdsl > cdev->mdsl) {
 608                 if (iscsi_change_param_sprintf(
 609                         conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
 610                         return -1;
 611         }
 612
 613         return 0;
 614 }
 615
 616 static int cxgbit_set_digest(struct cxgbit_sock *csk)
 617 {
 618         struct iscsi_conn *conn = csk->conn;
 619         struct iscsi_param *param;
 620
 621         param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
 622         if (!param) {
 623                 pr_err("param not found key %s\n", HEADERDIGEST);
 624                 return -1;
 625         }
 626
 627         if (!strcmp(param->value, CRC32C))
 628                 csk->submode |= CXGBIT_SUBMODE_HCRC;
 629
 630         param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
 631         if (!param) {
 632                 csk->submode = 0;
 633                 pr_err("param not found key %s\n", DATADIGEST);
 634                 return -1;
 635         }
 636
 637         if (!strcmp(param->value, CRC32C))
 638                 csk->submode |= CXGBIT_SUBMODE_DCRC;
 639
 640         if (cxgbit_setup_conn_digest(csk)) {
 641                 csk->submode = 0;
 642                 return -1;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
 649 {
 650         struct iscsi_conn *conn = csk->conn;
 651         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 652         struct iscsi_param *param;
 653         u32 mrdsl, mbl;
 654         u32 max_npdu, max_iso_npdu;
 655         u32 max_iso_payload;
 656
 657         if (conn->login->leading_connection) {
 658                 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
 659                                                   conn->param_list);
 660                 if (!param) {
 661                         pr_err("param not found key %s\n", MAXBURSTLENGTH);
 662                         return -1;
 663                 }
 664
 665                 if (kstrtou32(param->value, 0, &mbl) < 0)
 666                         return -1;
 667         } else {
 668                 mbl = conn->sess->sess_ops->MaxBurstLength;
 669         }
 670
 671         mrdsl = conn_ops->MaxRecvDataSegmentLength;
 672         max_npdu = mbl / mrdsl;
 673
 674         max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
 675
 676         max_iso_npdu = max_iso_payload /
 677                        (ISCSI_HDR_LEN + mrdsl +
 678                         cxgbit_digest_len[csk->submode]);
 679
 680         csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
 681
 682         if (csk->max_iso_npdu <= 1)
 683                 csk->max_iso_npdu = 0;
 684
 685         return 0;
 686 }
 687
 688 /*
 689  * cxgbit_seq_pdu_inorder()
 690  * @csk: pointer to cxgbit socket structure
 691  *
 692  * This function checks whether data sequence and data
 693  * pdu are in order.
 694  *
 695  * Return: returns -1 on error, 0 if data sequence and
 696  * data pdu are in order, 1 if data sequence or data pdu
 697  * is not in order.
 698  */
 699 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
 700 {
 701         struct iscsi_conn *conn = csk->conn;
 702         struct iscsi_param *param;
 703
 704         if (conn->login->leading_connection) {
 705                 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
 706                                                   conn->param_list);
 707                 if (!param) {
 708                         pr_err("param not found key %s\n", DATASEQUENCEINORDER);
 709                         return -1;
 710                 }
 711
 712                 if (strcmp(param->value, YES))
 713                         return 1;
 714
 715                 param = iscsi_find_param_from_key(DATAPDUINORDER,
 716                                                   conn->param_list);
 717                 if (!param) {
 718                         pr_err("param not found key %s\n", DATAPDUINORDER);
 719                         return -1;
 720                 }
 721
 722                 if (strcmp(param->value, YES))
 723                         return 1;
 724
 725         } else {
 726                 if (!conn->sess->sess_ops->DataSequenceInOrder)
 727                         return 1;
 728                 if (!conn->sess->sess_ops->DataPDUInOrder)
 729                         return 1;
 730         }
 731
 732         return 0;
 733 }
 734
 735 static int cxgbit_set_params(struct iscsi_conn *conn)
 736 {
 737         struct cxgbit_sock *csk = conn->context;
 738         struct cxgbit_device *cdev = csk->com.cdev;
 739         struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
 740         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 741         struct iscsi_param *param;
 742         u8 erl;
 743
 744         if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
 745                 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
 746
 747         if (cxgbit_set_digest(csk))
 748                 return -1;
 749
 750         if (conn->login->leading_connection) {
 751                 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
 752                                                   conn->param_list);
 753                 if (!param) {
 754                         pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
 755                         return -1;
 756                 }
 757                 if (kstrtou8(param->value, 0, &erl) < 0)
 758                         return -1;
 759         } else {
 760                 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
 761         }
 762
 763         if (!erl) {
 764                 int ret;
 765
 766                 ret = cxgbit_seq_pdu_inorder(csk);
 767                 if (ret < 0) {
 768                         return -1;
 769                 } else if (ret > 0) {
 770                         if (is_t5(cdev->lldi.adapter_type))
 771                                 goto enable_ddp;
 772                         else
 773                                 return 0;
 774                 }
 775
 776                 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
 777                         if (cxgbit_set_iso_npdu(csk))
 778                                 return -1;
 779                 }
 780
 781 enable_ddp:
 782                 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
 783                         if (cxgbit_setup_conn_pgidx(csk,
 784                                                     ppm->tformat.pgsz_idx_dflt))
 785                                 return -1;
 786                         set_bit(CSK_DDP_ENABLE, &csk->com.flags);
 787                 }
 788         }
 789
 790         return 0;
 791 }
 792
 793 int
 794 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 795                     u32 length)
 796 {
 797         struct cxgbit_sock *csk = conn->context;
 798         struct sk_buff *skb;
 799         u32 padding_buf = 0;
 800         u8 padding = ((-length) & 3);
 801
 802         skb = cxgbit_alloc_skb(csk, length + padding);
 803         if (!skb)
 804                 return -ENOMEM;
 805         skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
 806         skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
 807
 808         if (padding)
 809                 skb_store_bits(skb, ISCSI_HDR_LEN + length,
 810                                &padding_buf, padding);
 811
 812         if (login->login_complete) {
 813                 if (cxgbit_set_params(conn)) {
 814                         kfree_skb(skb);
 815                         return -1;
 816                 }
 817
 818                 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
 819         }
 820
 821         if (cxgbit_queue_skb(csk, skb))
 822                 return -1;
 823
 824         if ((!login->login_complete) && (!login->login_failed))
 825                 schedule_delayed_work(&conn->login_work, 0);
 826
 827         return 0;
 828 }
 829
 830 static void
 831 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
 832                       unsigned int nents, u32 skip)
 833 {
 834         struct skb_seq_state st;
 835         const u8 *buf;
 836         unsigned int consumed = 0, buf_len;
 837         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
 838
 839         skb_prepare_seq_read(skb, pdu_cb->doffset,
 840                              pdu_cb->doffset + pdu_cb->dlen,
 841                              &st);
 842
 843         while (true) {
 844                 buf_len = skb_seq_read(consumed, &buf, &st);
 845                 if (!buf_len) {
 846                         skb_abort_seq_read(&st);
 847                         break;
 848                 }
 849
 850                 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
 851                                                  buf_len, skip + consumed);
 852         }
 853 }
 854
 855 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
 856 {
 857         struct iscsi_conn *conn = csk->conn;
 858         struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
 859         struct cxgbit_cmd *ccmd;
 860         struct iscsi_cmd *cmd;
 861
 862         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 863         if (!cmd) {
 864                 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
 865                 return NULL;
 866         }
 867
 868         ccmd = iscsit_priv_cmd(cmd);
 869         ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
 870         ccmd->setup_ddp = true;
 871
 872         return cmd;
 873 }
 874
 875 static int
 876 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 877                              u32 length)
 878 {
 879         struct iscsi_conn *conn = cmd->conn;
 880         struct cxgbit_sock *csk = conn->context;
 881         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 882
 883         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 884                 pr_err("ImmediateData CRC32C DataDigest error\n");
 885                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 886                         pr_err("Unable to recover from"
 887                                " Immediate Data digest failure while"
 888                                " in ERL=0.\n");
 889                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 890                                           (unsigned char *)hdr);
 891                         return IMMEDIATE_DATA_CANNOT_RECOVER;
 892                 }
 893
 894                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 895                                   (unsigned char *)hdr);
 896                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
 897         }
 898
 899         if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
 900                 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
 901                 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
 902                 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
 903
 904                 sg_init_table(&ccmd->sg, 1);
 905                 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
 906                             dfrag->page_offset);
 907                 get_page(dfrag->page.p);
 908
 909                 cmd->se_cmd.t_data_sg = &ccmd->sg;
 910                 cmd->se_cmd.t_data_nents = 1;
 911
 912                 ccmd->release = true;
 913         } else {
 914                 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
 915                 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
 916
 917                 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
 918         }
 919
 920         cmd->write_data_done += pdu_cb->dlen;
 921
 922         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 923                 spin_lock_bh(&cmd->istate_lock);
 924                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 925                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 926                 spin_unlock_bh(&cmd->istate_lock);
 927         }
 928
 929         return IMMEDIATE_DATA_NORMAL_OPERATION;
 930 }
 931
 932 static int
 933 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 934                           bool dump_payload)
 935 {
 936         struct iscsi_conn *conn = cmd->conn;
 937         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
 938         /*
 939          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 940          */
 941         if (dump_payload)
 942                 goto after_immediate_data;
 943
 944         immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
 945                                                  cmd->first_burst_len);
 946 after_immediate_data:
 947         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 948                 /*
 949                  * A PDU/CmdSN carrying Immediate Data passed
 950                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
 951                  * Immediate Bit is not set.
 952                  */
 953                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
 954                                                 (unsigned char *)hdr,
 955                                                 hdr->cmdsn);
 956                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
 957                         return -1;
 958
 959                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
 960                         target_put_sess_cmd(&cmd->se_cmd);
 961                         return 0;
 962                 } else if (cmd->unsolicited_data) {
 963                         iscsit_set_unsoliticed_dataout(cmd);
 964                 }
 965
 966         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
 967                 /*
 968                  * Immediate Data failed DataCRC and ERL>=1,
 969                  * silently drop this PDU and let the initiator
 970                  * plug the CmdSN gap.
 971                  *
 972                  * FIXME: Send Unsolicited NOPIN with reserved
 973                  * TTT here to help the initiator figure out
 974                  * the missing CmdSN, although they should be
 975                  * intelligent enough to determine the missing
 976                  * CmdSN and issue a retry to plug the sequence.
 977                  */
 978                 cmd->i_state = ISTATE_REMOVE;
 979                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 980         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
 981                 return -1;
 982
 983         return 0;
 984 }
 985
 986 static int
 987 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
 988 {
 989         struct iscsi_conn *conn = csk->conn;
 990         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 991         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
 992         int rc;
 993         bool dump_payload = false;
 994
 995         rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
 996         if (rc < 0)
 997                 return rc;
 998
 999         if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
1000             (pdu_cb->nr_dfrags == 1))
1001                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1002
1003         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1004         if (rc < 0)
1005                 return 0;
1006         else if (rc > 0)
1007                 dump_payload = true;
1008
1009         if (!pdu_cb->dlen)
1010                 return 0;
1011
1012         return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
1013 }
1014
1015 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
1016 {
1017         struct scatterlist *sg_start;
1018         struct iscsi_conn *conn = csk->conn;
1019         struct iscsi_cmd *cmd = NULL;
1020         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1021         struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1022         u32 data_offset = be32_to_cpu(hdr->offset);
1023         u32 data_len = pdu_cb->dlen;
1024         int rc, sg_nents, sg_off;
1025         bool dcrc_err = false;
1026
1027         if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1028                 u32 offset = be32_to_cpu(hdr->offset);
1029                 u32 ddp_data_len;
1030                 u32 payload_length = ntoh24(hdr->dlength);
1031                 bool success = false;
1032
1033                 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1034                 if (!cmd)
1035                         return 0;
1036
1037                 ddp_data_len = offset - cmd->write_data_done;
1038                 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1039
1040                 cmd->write_data_done = offset;
1041                 cmd->next_burst_len = ddp_data_len;
1042                 cmd->data_sn = be32_to_cpu(hdr->datasn);
1043
1044                 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1045                                                 cmd, payload_length, &success);
1046                 if (rc < 0)
1047                         return rc;
1048                 else if (!success)
1049                         return 0;
1050         } else {
1051                 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1052                 if (rc < 0)
1053                         return rc;
1054                 else if (!cmd)
1055                         return 0;
1056         }
1057
1058         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1059                 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1060                        " DataSN: 0x%08x\n",
1061                        hdr->itt, hdr->offset, data_len,
1062                        hdr->datasn);
1063
1064                 dcrc_err = true;
1065                 goto check_payload;
1066         }
1067
1068         pr_debug("DataOut data_len: %u, "
1069                 "write_data_done: %u, data_length: %u\n",
1070                   data_len,  cmd->write_data_done,
1071                   cmd->se_cmd.data_length);
1072
1073         if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1074                 u32 skip = data_offset % PAGE_SIZE;
1075
1076                 sg_off = data_offset / PAGE_SIZE;
1077                 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1078                 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1079
1080                 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1081         }
1082
1083 check_payload:
1084
1085         rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1086         if (rc < 0)
1087                 return rc;
1088
1089         return 0;
1090 }
1091
1092 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1093 {
1094         struct iscsi_conn *conn = csk->conn;
1095         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1096         struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1097         unsigned char *ping_data = NULL;
1098         u32 payload_length = pdu_cb->dlen;
1099         int ret;
1100
1101         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1102         if (ret < 0)
1103                 return 0;
1104
1105         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1106                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1107                         pr_err("Unable to recover from"
1108                                " NOPOUT Ping DataCRC failure while in"
1109                                " ERL=0.\n");
1110                         ret = -1;
1111                         goto out;
1112                 } else {
1113                         /*
1114                          * drop this PDU and let the
1115                          * initiator plug the CmdSN gap.
1116                          */
1117                         pr_info("Dropping NOPOUT"
1118                                 " Command CmdSN: 0x%08x due to"
1119                                 " DataCRC error.\n", hdr->cmdsn);
1120                         ret = 0;
1121                         goto out;
1122                 }
1123         }
1124
1125         /*
1126          * Handle NOP-OUT payload for traditional iSCSI sockets
1127          */
1128         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1129                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1130                 if (!ping_data) {
1131                         pr_err("Unable to allocate memory for"
1132                                 " NOPOUT ping data.\n");
1133                         ret = -1;
1134                         goto out;
1135                 }
1136
1137                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1138                               ping_data, payload_length);
1139
1140                 ping_data[payload_length] = '\0';
1141                 /*
1142                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1143                  */
1144                 cmd->buf_ptr = ping_data;
1145                 cmd->buf_ptr_size = payload_length;
1146
1147                 pr_debug("Got %u bytes of NOPOUT ping"
1148                         " data.\n", payload_length);
1149                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1150         }
1151
1152         return iscsit_process_nop_out(conn, cmd, hdr);
1153 out:
1154         if (cmd)
1155                 iscsit_free_cmd(cmd, false);
1156         return ret;
1157 }
1158
1159 static int
1160 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1161 {
1162         struct iscsi_conn *conn = csk->conn;
1163         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1164         struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1165         u32 payload_length = pdu_cb->dlen;
1166         int rc;
1167         unsigned char *text_in = NULL;
1168
1169         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1170         if (rc < 0)
1171                 return rc;
1172
1173         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1174                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1175                         pr_err("Unable to recover from"
1176                                " Text Data digest failure while in"
1177                                " ERL=0.\n");
1178                         goto reject;
1179                 } else {
1180                         /*
1181                          * drop this PDU and let the
1182                          * initiator plug the CmdSN gap.
1183                          */
1184                         pr_info("Dropping Text"
1185                                 " Command CmdSN: 0x%08x due to"
1186                                 " DataCRC error.\n", hdr->cmdsn);
1187                         return 0;
1188                 }
1189         }
1190
1191         if (payload_length) {
1192                 text_in = kzalloc(payload_length, GFP_KERNEL);
1193                 if (!text_in) {
1194                         pr_err("Unable to allocate text_in of payload_length: %u\n",
1195                                payload_length);
1196                         return -ENOMEM;
1197                 }
1198                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1199                               text_in, payload_length);
1200
1201                 text_in[payload_length - 1] = '\0';
1202
1203                 cmd->text_in_ptr = text_in;
1204         }
1205
1206         return iscsit_process_text_cmd(conn, cmd, hdr);
1207
1208 reject:
1209         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1210                                  pdu_cb->hdr);
1211 }
1212
1213 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1214 {
1215         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1216         struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1217         struct iscsi_conn *conn = csk->conn;
1218         struct iscsi_cmd *cmd = NULL;
1219         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1220         int ret = -EINVAL;
1221
1222         switch (opcode) {
1223         case ISCSI_OP_SCSI_CMD:
1224                 cmd = cxgbit_allocate_cmd(csk);
1225                 if (!cmd)
1226                         goto reject;
1227
1228                 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1229                 break;
1230         case ISCSI_OP_SCSI_DATA_OUT:
1231                 ret = cxgbit_handle_iscsi_dataout(csk);
1232                 break;
1233         case ISCSI_OP_NOOP_OUT:
1234                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1235                         cmd = cxgbit_allocate_cmd(csk);
1236                         if (!cmd)
1237                                 goto reject;
1238                 }
1239
1240                 ret = cxgbit_handle_nop_out(csk, cmd);
1241                 break;
1242         case ISCSI_OP_SCSI_TMFUNC:
1243                 cmd = cxgbit_allocate_cmd(csk);
1244                 if (!cmd)
1245                         goto reject;
1246
1247                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1248                                                  (unsigned char *)hdr);
1249                 break;
1250         case ISCSI_OP_TEXT:
1251                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1252                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1253                         if (!cmd)
1254                                 goto reject;
1255                 } else {
1256                         cmd = cxgbit_allocate_cmd(csk);
1257                         if (!cmd)
1258                                 goto reject;
1259                 }
1260
1261                 ret = cxgbit_handle_text_cmd(csk, cmd);
1262                 break;
1263         case ISCSI_OP_LOGOUT:
1264                 cmd = cxgbit_allocate_cmd(csk);
1265                 if (!cmd)
1266                         goto reject;
1267
1268                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1269                 if (ret > 0)
1270                         wait_for_completion_timeout(&conn->conn_logout_comp,
1271                                                     SECONDS_FOR_LOGOUT_COMP
1272                                                     * HZ);
1273                 break;
1274         case ISCSI_OP_SNACK:
1275                 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1276                 break;
1277         default:
1278                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1279                 dump_stack();
1280                 break;
1281         }
1282
1283         return ret;
1284
1285 reject:
1286         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1287                                  (unsigned char *)hdr);
1288         return ret;
1289 }
1290
1291 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1292 {
1293         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1294         struct iscsi_conn *conn = csk->conn;
1295         struct iscsi_hdr *hdr = pdu_cb->hdr;
1296         u8 opcode;
1297
1298         if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1299                 atomic_long_inc(&conn->sess->conn_digest_errors);
1300                 goto transport_err;
1301         }
1302
1303         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1304                 goto transport_err;
1305
1306         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1307
1308         if (conn->sess->sess_ops->SessionType &&
1309             ((!(opcode & ISCSI_OP_TEXT)) ||
1310              (!(opcode & ISCSI_OP_LOGOUT)))) {
1311                 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1312                         " while in Discovery Session, rejecting.\n", opcode);
1313                 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1314                                   (unsigned char *)hdr);
1315                 goto transport_err;
1316         }
1317
1318         if (cxgbit_target_rx_opcode(csk) < 0)
1319                 goto transport_err;
1320
1321         return 0;
1322
1323 transport_err:
1324         return -1;
1325 }
1326
1327 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1328 {
1329         struct iscsi_conn *conn = csk->conn;
1330         struct iscsi_login *login = conn->login;
1331         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1332         struct iscsi_login_req *login_req;
1333
1334         login_req = (struct iscsi_login_req *)login->req;
1335         memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1336
1337         pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1338                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1339                 login_req->flags, login_req->itt, login_req->cmdsn,
1340                 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1341         /*
1342          * Setup the initial iscsi_login values from the leading
1343          * login request PDU.
1344          */
1345         if (login->first_request) {
1346                 login_req = (struct iscsi_login_req *)login->req;
1347                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1348                 login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(
1349                                 login_req->flags);
1350                 login->version_min      = login_req->min_version;
1351                 login->version_max      = login_req->max_version;
1352                 memcpy(login->isid, login_req->isid, 6);
1353                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1354                 login->init_task_tag    = login_req->itt;
1355                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1356                 login->cid              = be16_to_cpu(login_req->cid);
1357                 login->tsih             = be16_to_cpu(login_req->tsih);
1358         }
1359
1360         if (iscsi_target_check_login_request(conn, login) < 0)
1361                 return -1;
1362
1363         memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1364         skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1365
1366         return 0;
1367 }
1368
1369 static int
1370 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1371 {
1372         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1373         int ret;
1374
1375         cxgbit_rx_pdu_cb(skb) = pdu_cb;
1376
1377         csk->skb = skb;
1378
1379         if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1380                 ret = cxgbit_rx_login_pdu(csk);
1381                 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1382         } else {
1383                 ret = cxgbit_rx_opcode(csk);
1384         }
1385
1386         return ret;
1387 }
1388
1389 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1390 {
1391         struct skb_shared_info *ssi = skb_shinfo(skb);
1392         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1393         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1394         u8 i;
1395
1396         pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1397                 skb, skb->head, skb->data, skb->len, skb->data_len,
1398                 ssi->nr_frags);
1399         pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1400                 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1401
1402         for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1403                 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1404                         "frags %u.\n",
1405                         skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1406                         pdu_cb->ddigest, pdu_cb->frags);
1407         for (i = 0; i < ssi->nr_frags; i++)
1408                 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1409                         skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1410 }
1411
1412 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1413 {
1414         struct sk_buff *skb = csk->lro_hskb;
1415         struct skb_shared_info *ssi = skb_shinfo(skb);
1416         u8 i;
1417
1418         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1419         for (i = 0; i < ssi->nr_frags; i++)
1420                 put_page(skb_frag_page(&ssi->frags[i]));
1421         ssi->nr_frags = 0;
1422         skb->data_len = 0;
1423         skb->truesize -= skb->len;
1424         skb->len = 0;
1425 }
1426
1427 static void
1428 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1429 {
1430         struct sk_buff *hskb = csk->lro_hskb;
1431         struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1432         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1433         struct skb_shared_info *hssi = skb_shinfo(hskb);
1434         struct skb_shared_info *ssi = skb_shinfo(skb);
1435         unsigned int len = 0;
1436
1437         if (pdu_cb->flags & PDUCBF_RX_HDR) {
1438                 u8 hfrag_idx = hssi->nr_frags;
1439
1440                 hpdu_cb->flags |= pdu_cb->flags;
1441                 hpdu_cb->seq = pdu_cb->seq;
1442                 hpdu_cb->hdr = pdu_cb->hdr;
1443                 hpdu_cb->hlen = pdu_cb->hlen;
1444
1445                 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1446                        sizeof(skb_frag_t));
1447
1448                 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1449                 hssi->nr_frags++;
1450                 hpdu_cb->frags++;
1451                 hpdu_cb->hfrag_idx = hfrag_idx;
1452
1453                 len = hssi->frags[hfrag_idx].size;
1454                 hskb->len += len;
1455                 hskb->data_len += len;
1456                 hskb->truesize += len;
1457         }
1458
1459         if (pdu_cb->flags & PDUCBF_RX_DATA) {
1460                 u8 dfrag_idx = hssi->nr_frags, i;
1461
1462                 hpdu_cb->flags |= pdu_cb->flags;
1463                 hpdu_cb->dfrag_idx = dfrag_idx;
1464
1465                 len = 0;
1466                 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1467                         memcpy(&hssi->frags[dfrag_idx],
1468                                &ssi->frags[pdu_cb->dfrag_idx + i],
1469                                sizeof(skb_frag_t));
1470
1471                         get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1472
1473                         len += hssi->frags[dfrag_idx].size;
1474
1475                         hssi->nr_frags++;
1476                         hpdu_cb->frags++;
1477                 }
1478
1479                 hpdu_cb->dlen = pdu_cb->dlen;
1480                 hpdu_cb->doffset = hpdu_cb->hlen;
1481                 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1482                 hskb->len += len;
1483                 hskb->data_len += len;
1484                 hskb->truesize += len;
1485         }
1486
1487         if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1488                 hpdu_cb->flags |= pdu_cb->flags;
1489
1490                 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1491                         hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1492
1493                 hpdu_cb->ddigest = pdu_cb->ddigest;
1494                 hpdu_cb->pdulen = pdu_cb->pdulen;
1495         }
1496 }
1497
1498 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1499 {
1500         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1501         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1502         u8 pdu_idx = 0, last_idx = 0;
1503         int ret = 0;
1504
1505         if (!pdu_cb->complete) {
1506                 cxgbit_lro_skb_merge(csk, skb, 0);
1507
1508                 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1509                         struct sk_buff *hskb = csk->lro_hskb;
1510
1511                         ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1512
1513                         cxgbit_lro_hskb_reset(csk);
1514
1515                         if (ret < 0)
1516                                 goto out;
1517                 }
1518
1519                 pdu_idx = 1;
1520         }
1521
1522         if (lro_cb->pdu_idx)
1523                 last_idx = lro_cb->pdu_idx - 1;
1524
1525         for (; pdu_idx <= last_idx; pdu_idx++) {
1526                 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1527                 if (ret < 0)
1528                         goto out;
1529         }
1530
1531         if ((!lro_cb->complete) && lro_cb->pdu_idx)
1532                 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1533
1534 out:
1535         return ret;
1536 }
1537
1538 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1539 {
1540         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1541         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1542         int ret = -1;
1543
1544         if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1545             (pdu_cb->seq != csk->rcv_nxt)) {
1546                 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1547                         csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1548                 cxgbit_lro_skb_dump(skb);
1549                 return ret;
1550         }
1551
1552         csk->rcv_nxt += lro_cb->pdu_totallen;
1553
1554         ret = cxgbit_process_lro_skb(csk, skb);
1555
1556         csk->rx_credits += lro_cb->pdu_totallen;
1557
1558         if (csk->rx_credits >= (csk->rcv_win / 4))
1559                 cxgbit_rx_data_ack(csk);
1560
1561         return ret;
1562 }
1563
1564 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1565 {
1566         struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1567         int ret = -1;
1568
1569         if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1570                 if (is_t5(lldi->adapter_type))
1571                         ret = cxgbit_rx_lro_skb(csk, skb);
1572                 else
1573                         ret = cxgbit_process_lro_skb(csk, skb);
1574         }
1575
1576         __kfree_skb(skb);
1577         return ret;
1578 }
1579
1580 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1581 {
1582         spin_lock_bh(&csk->rxq.lock);
1583         if (skb_queue_len(&csk->rxq)) {
1584                 skb_queue_splice_init(&csk->rxq, rxq);
1585                 spin_unlock_bh(&csk->rxq.lock);
1586                 return true;
1587         }
1588         spin_unlock_bh(&csk->rxq.lock);
1589         return false;
1590 }
1591
1592 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1593 {
1594         struct sk_buff *skb;
1595         struct sk_buff_head rxq;
1596
1597         skb_queue_head_init(&rxq);
1598
1599         wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1600
1601         if (signal_pending(current))
1602                 goto out;
1603
1604         while ((skb = __skb_dequeue(&rxq))) {
1605                 if (cxgbit_rx_skb(csk, skb))
1606                         goto out;
1607         }
1608
1609         return 0;
1610 out:
1611         __skb_queue_purge(&rxq);
1612         return -1;
1613 }
1614
1615 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1616 {
1617         struct cxgbit_sock *csk = conn->context;
1618         int ret = -1;
1619
1620         while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1621                 ret = cxgbit_wait_rxq(csk);
1622                 if (ret) {
1623                         clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1624                         break;
1625                 }
1626         }
1627
1628         return ret;
1629 }
1630
1631 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1632 {
1633         struct cxgbit_sock *csk = conn->context;
1634
1635         while (!kthread_should_stop()) {
1636                 iscsit_thread_check_cpumask(conn, current, 0);
1637                 if (cxgbit_wait_rxq(csk))
1638                         return;
1639         }
1640 }