drivers/target/iscsi/iscsi_target_erl0.c

   1 /******************************************************************************
   2  * This file contains error recovery level zero functions used by
   3  * the iSCSI Target driver.
   4  *
   5  * (c) Copyright 2007-2013 Datera, Inc.
   6  *
   7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  ******************************************************************************/
  19
  20 #include <scsi/iscsi_proto.h>
  21 #include <target/target_core_base.h>
  22 #include <target/target_core_fabric.h>
  23
  24 #include <target/iscsi/iscsi_target_core.h>
  25 #include "iscsi_target_seq_pdu_list.h"
  26 #include "iscsi_target_erl0.h"
  27 #include "iscsi_target_erl1.h"
  28 #include "iscsi_target_erl2.h"
  29 #include "iscsi_target_util.h"
  30 #include "iscsi_target.h"
  31
  32 /*
  33  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  34  *      checks against to determine a PDU's Offset+Length is within the current
  35  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  36  */
  37 void iscsit_set_dataout_sequence_values(
  38         struct iscsi_cmd *cmd)
  39 {
  40         struct iscsi_conn *conn = cmd->conn;
  41         /*
  42          * Still set seq_start_offset and seq_end_offset for Unsolicited
  43          * DataOUT, even if DataSequenceInOrder=No.
  44          */
  45         if (cmd->unsolicited_data) {
  46                 cmd->seq_start_offset = cmd->write_data_done;
  47                 cmd->seq_end_offset = min(cmd->se_cmd.data_length,
  48                                         conn->sess->sess_ops->FirstBurstLength);
  49                 return;
  50         }
  51
  52         if (!conn->sess->sess_ops->DataSequenceInOrder)
  53                 return;
  54
  55         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  56                 cmd->seq_start_offset = cmd->write_data_done;
  57                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  58                         conn->sess->sess_ops->MaxBurstLength) ?
  59                         (cmd->write_data_done +
  60                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  61         } else {
  62                 cmd->seq_start_offset = cmd->seq_end_offset;
  63                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  64                         conn->sess->sess_ops->MaxBurstLength) >=
  65                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  66                         (cmd->seq_end_offset +
  67                          conn->sess->sess_ops->MaxBurstLength);
  68         }
  69 }
  70
  71 static int iscsit_dataout_within_command_recovery_check(
  72         struct iscsi_cmd *cmd,
  73         unsigned char *buf)
  74 {
  75         struct iscsi_conn *conn = cmd->conn;
  76         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  77         u32 payload_length = ntoh24(hdr->dlength);
  78
  79         /*
  80          * We do the within-command recovery checks here as it is
  81          * the first function called in iscsi_check_pre_dataout().
  82          * Basically, if we are in within-command recovery and
  83          * the PDU does not contain the offset the sequence needs,
  84          * dump the payload.
  85          *
  86          * This only applies to DataPDUInOrder=Yes, for
  87          * DataPDUInOrder=No we only re-request the failed PDU
  88          * and check that all PDUs in a sequence are received
  89          * upon end of sequence.
  90          */
  91         if (conn->sess->sess_ops->DataSequenceInOrder) {
  92                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  93                     cmd->write_data_done != be32_to_cpu(hdr->offset))
  94                         goto dump;
  95
  96                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
  97         } else {
  98                 struct iscsi_seq *seq;
  99
 100                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 101                                             payload_length);
 102                 if (!seq)
 103                         return DATAOUT_CANNOT_RECOVER;
 104                 /*
 105                  * Set the struct iscsi_seq pointer to reuse later.
 106                  */
 107                 cmd->seq_ptr = seq;
 108
 109                 if (conn->sess->sess_ops->DataPDUInOrder) {
 110                         if (seq->status ==
 111                             DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 112                            (seq->offset != be32_to_cpu(hdr->offset) ||
 113                             seq->data_sn != be32_to_cpu(hdr->datasn)))
 114                                 goto dump;
 115                 } else {
 116                         if (seq->status ==
 117                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 118                             seq->data_sn != be32_to_cpu(hdr->datasn))
 119                                 goto dump;
 120                 }
 121
 122                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 123                         goto dump;
 124
 125                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 126                         seq->status = 0;
 127         }
 128
 129         return DATAOUT_NORMAL;
 130
 131 dump:
 132         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 133                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 134         return iscsit_dump_data_payload(conn, payload_length, 1);
 135 }
 136
 137 static int iscsit_dataout_check_unsolicited_sequence(
 138         struct iscsi_cmd *cmd,
 139         unsigned char *buf)
 140 {
 141         u32 first_burst_len;
 142         struct iscsi_conn *conn = cmd->conn;
 143         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 144         u32 payload_length = ntoh24(hdr->dlength);
 145
 146
 147         if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 148            ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 149                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 150                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 151                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 152                 be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 153                         cmd->seq_end_offset);
 154                 return DATAOUT_CANNOT_RECOVER;
 155         }
 156
 157         first_burst_len = (cmd->first_burst_len + payload_length);
 158
 159         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 160                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 161                         " for this Unsolicited DataOut Burst.\n",
 162                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 163                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 164                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 165                 return DATAOUT_CANNOT_RECOVER;
 166         }
 167
 168         /*
 169          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 170          * checks for the current Unsolicited DataOUT Sequence.
 171          */
 172         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 173                 /*
 174                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 175                  * sequence checks are handled in
 176                  * iscsit_dataout_datapduinorder_no_fbit().
 177                  */
 178                 if (!conn->sess->sess_ops->DataPDUInOrder)
 179                         goto out;
 180
 181                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 182                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 183                         pr_err("Unsolicited non-immediate data"
 184                         " received %u does not equal FirstBurstLength: %u, and"
 185                         " does not equal ExpXferLen %u.\n", first_burst_len,
 186                                 conn->sess->sess_ops->FirstBurstLength,
 187                                 cmd->se_cmd.data_length);
 188                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 189                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 190                         return DATAOUT_CANNOT_RECOVER;
 191                 }
 192         } else {
 193                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 194                         pr_err("Command ITT: 0x%08x reached"
 195                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 196                                 " error.\n", cmd->init_task_tag,
 197                                 conn->sess->sess_ops->FirstBurstLength);
 198                         return DATAOUT_CANNOT_RECOVER;
 199                 }
 200                 if (first_burst_len == cmd->se_cmd.data_length) {
 201                         pr_err("Command ITT: 0x%08x reached"
 202                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 203                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 204                         return DATAOUT_CANNOT_RECOVER;
 205                 }
 206         }
 207
 208 out:
 209         return DATAOUT_NORMAL;
 210 }
 211
 212 static int iscsit_dataout_check_sequence(
 213         struct iscsi_cmd *cmd,
 214         unsigned char *buf)
 215 {
 216         u32 next_burst_len;
 217         struct iscsi_conn *conn = cmd->conn;
 218         struct iscsi_seq *seq = NULL;
 219         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 220         u32 payload_length = ntoh24(hdr->dlength);
 221
 222         /*
 223          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 224          * is within range as defined by iscsi_set_dataout_sequence_values().
 225          *
 226          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 227          * offset+length tuple.
 228          */
 229         if (conn->sess->sess_ops->DataSequenceInOrder) {
 230                 /*
 231                  * Due to possibility of recovery DataOUT sent by the initiator
 232                  * fullfilling an Recovery R2T, it's best to just dump the
 233                  * payload here, instead of erroring out.
 234                  */
 235                 if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 236                    ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 237                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 238                         " Length: %u outside of Sequence %u:%u while"
 239                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 240                         be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 241                                 cmd->seq_end_offset);
 242
 243                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 244                                 return DATAOUT_CANNOT_RECOVER;
 245                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 246                 }
 247
 248                 next_burst_len = (cmd->next_burst_len + payload_length);
 249         } else {
 250                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 251                                             payload_length);
 252                 if (!seq)
 253                         return DATAOUT_CANNOT_RECOVER;
 254                 /*
 255                  * Set the struct iscsi_seq pointer to reuse later.
 256                  */
 257                 cmd->seq_ptr = seq;
 258
 259                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 260                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 261                                 return DATAOUT_CANNOT_RECOVER;
 262                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 263                 }
 264
 265                 next_burst_len = (seq->next_burst_len + payload_length);
 266         }
 267
 268         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 269                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 270                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 271                         " error.\n", cmd->init_task_tag,
 272                         (next_burst_len - payload_length),
 273                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 274                 return DATAOUT_CANNOT_RECOVER;
 275         }
 276
 277         /*
 278          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 279          * checks for the current DataOUT Sequence.
 280          */
 281         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 282                 /*
 283                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 284                  * sequence checks are handled in
 285                  * iscsit_dataout_datapduinorder_no_fbit().
 286                  */
 287                 if (!conn->sess->sess_ops->DataPDUInOrder)
 288                         goto out;
 289
 290                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 291                         if ((next_burst_len <
 292                              conn->sess->sess_ops->MaxBurstLength) &&
 293                            ((cmd->write_data_done + payload_length) <
 294                              cmd->se_cmd.data_length)) {
 295                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 296                                 " before end of DataOUT sequence, protocol"
 297                                 " error.\n", cmd->init_task_tag);
 298                                 return DATAOUT_CANNOT_RECOVER;
 299                         }
 300                 } else {
 301                         if (next_burst_len < seq->xfer_len) {
 302                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 303                                 " before end of DataOUT sequence, protocol"
 304                                 " error.\n", cmd->init_task_tag);
 305                                 return DATAOUT_CANNOT_RECOVER;
 306                         }
 307                 }
 308         } else {
 309                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 310                         if (next_burst_len ==
 311                                         conn->sess->sess_ops->MaxBurstLength) {
 312                                 pr_err("Command ITT: 0x%08x reached"
 313                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 314                                 " not set, protocol error.", cmd->init_task_tag,
 315                                         conn->sess->sess_ops->MaxBurstLength);
 316                                 return DATAOUT_CANNOT_RECOVER;
 317                         }
 318                         if ((cmd->write_data_done + payload_length) ==
 319                                         cmd->se_cmd.data_length) {
 320                                 pr_err("Command ITT: 0x%08x reached"
 321                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 322                                 "CMD_FINAL is not set, protocol error.\n",
 323                                         cmd->init_task_tag);
 324                                 return DATAOUT_CANNOT_RECOVER;
 325                         }
 326                 } else {
 327                         if (next_burst_len == seq->xfer_len) {
 328                                 pr_err("Command ITT: 0x%08x reached"
 329                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 330                                 "CMD_FINAL is not set, protocol error.\n",
 331                                         cmd->init_task_tag);
 332                                 return DATAOUT_CANNOT_RECOVER;
 333                         }
 334                 }
 335         }
 336
 337 out:
 338         return DATAOUT_NORMAL;
 339 }
 340
 341 static int iscsit_dataout_check_datasn(
 342         struct iscsi_cmd *cmd,
 343         unsigned char *buf)
 344 {
 345         u32 data_sn = 0;
 346         struct iscsi_conn *conn = cmd->conn;
 347         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 348         u32 payload_length = ntoh24(hdr->dlength);
 349
 350         /*
 351          * Considering the target has no method of re-requesting DataOUT
 352          * by DataSN, if we receieve a greater DataSN than expected we
 353          * assume the functions for DataPDUInOrder=[Yes,No] below will
 354          * handle it.
 355          *
 356          * If the DataSN is less than expected, dump the payload.
 357          */
 358         if (conn->sess->sess_ops->DataSequenceInOrder)
 359                 data_sn = cmd->data_sn;
 360         else {
 361                 struct iscsi_seq *seq = cmd->seq_ptr;
 362                 data_sn = seq->data_sn;
 363         }
 364
 365         if (be32_to_cpu(hdr->datasn) > data_sn) {
 366                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 367                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 368                                 be32_to_cpu(hdr->datasn), data_sn);
 369                 goto recover;
 370         } else if (be32_to_cpu(hdr->datasn) < data_sn) {
 371                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 372                         " lower than expected 0x%08x, discarding payload.\n",
 373                         cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
 374                 goto dump;
 375         }
 376
 377         return DATAOUT_NORMAL;
 378
 379 recover:
 380         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 381                 pr_err("Unable to perform within-command recovery"
 382                                 " while ERL=0.\n");
 383                 return DATAOUT_CANNOT_RECOVER;
 384         }
 385 dump:
 386         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 387                 return DATAOUT_CANNOT_RECOVER;
 388
 389         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 390 }
 391
 392 static int iscsit_dataout_pre_datapduinorder_yes(
 393         struct iscsi_cmd *cmd,
 394         unsigned char *buf)
 395 {
 396         int dump = 0, recovery = 0;
 397         struct iscsi_conn *conn = cmd->conn;
 398         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 399         u32 payload_length = ntoh24(hdr->dlength);
 400
 401         /*
 402          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 403          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 404          * occurred and fail the connection.
 405          *
 406          * For DataSequenceInOrder=No: If the offset is greater than the per
 407          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 408          * error has occurred and fail the connection.
 409          */
 410         if (conn->sess->sess_ops->DataSequenceInOrder) {
 411                 if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
 412                         pr_err("Command ITT: 0x%08x, received offset"
 413                         " %u different than expected %u.\n", cmd->init_task_tag,
 414                                 be32_to_cpu(hdr->offset), cmd->write_data_done);
 415                         recovery = 1;
 416                         goto recover;
 417                 }
 418         } else {
 419                 struct iscsi_seq *seq = cmd->seq_ptr;
 420
 421                 if (be32_to_cpu(hdr->offset) > seq->offset) {
 422                         pr_err("Command ITT: 0x%08x, received offset"
 423                         " %u greater than expected %u.\n", cmd->init_task_tag,
 424                                 be32_to_cpu(hdr->offset), seq->offset);
 425                         recovery = 1;
 426                         goto recover;
 427                 } else if (be32_to_cpu(hdr->offset) < seq->offset) {
 428                         pr_err("Command ITT: 0x%08x, received offset"
 429                         " %u less than expected %u, discarding payload.\n",
 430                                 cmd->init_task_tag, be32_to_cpu(hdr->offset),
 431                                 seq->offset);
 432                         dump = 1;
 433                         goto dump;
 434                 }
 435         }
 436
 437         return DATAOUT_NORMAL;
 438
 439 recover:
 440         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 441                 pr_err("Unable to perform within-command recovery"
 442                                 " while ERL=0.\n");
 443                 return DATAOUT_CANNOT_RECOVER;
 444         }
 445 dump:
 446         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 447                 return DATAOUT_CANNOT_RECOVER;
 448
 449         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 450                 be32_to_cpu(hdr->offset), payload_length) :
 451                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 452 }
 453
 454 static int iscsit_dataout_pre_datapduinorder_no(
 455         struct iscsi_cmd *cmd,
 456         unsigned char *buf)
 457 {
 458         struct iscsi_pdu *pdu;
 459         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 460         u32 payload_length = ntoh24(hdr->dlength);
 461
 462         pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
 463                                     payload_length);
 464         if (!pdu)
 465                 return DATAOUT_CANNOT_RECOVER;
 466
 467         cmd->pdu_ptr = pdu;
 468
 469         switch (pdu->status) {
 470         case ISCSI_PDU_NOT_RECEIVED:
 471         case ISCSI_PDU_CRC_FAILED:
 472         case ISCSI_PDU_TIMED_OUT:
 473                 break;
 474         case ISCSI_PDU_RECEIVED_OK:
 475                 pr_err("Command ITT: 0x%08x received already gotten"
 476                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 477                                 be32_to_cpu(hdr->offset), payload_length);
 478                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 479         default:
 480                 return DATAOUT_CANNOT_RECOVER;
 481         }
 482
 483         return DATAOUT_NORMAL;
 484 }
 485
 486 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 487 {
 488         struct iscsi_r2t *r2t;
 489
 490         if (cmd->unsolicited_data)
 491                 return 0;
 492
 493         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 494         if (!r2t)
 495                 return -1;
 496
 497         spin_lock_bh(&cmd->r2t_lock);
 498         r2t->seq_complete = 1;
 499         cmd->outstanding_r2ts--;
 500         spin_unlock_bh(&cmd->r2t_lock);
 501
 502         return 0;
 503 }
 504
 505 static int iscsit_dataout_update_datapduinorder_no(
 506         struct iscsi_cmd *cmd,
 507         u32 data_sn,
 508         int f_bit)
 509 {
 510         int ret = 0;
 511         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 512
 513         pdu->data_sn = data_sn;
 514
 515         switch (pdu->status) {
 516         case ISCSI_PDU_NOT_RECEIVED:
 517                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 518                 break;
 519         case ISCSI_PDU_CRC_FAILED:
 520                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 521                 break;
 522         case ISCSI_PDU_TIMED_OUT:
 523                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 524                 break;
 525         default:
 526                 return DATAOUT_CANNOT_RECOVER;
 527         }
 528
 529         if (f_bit) {
 530                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 531                 if (ret == DATAOUT_CANNOT_RECOVER)
 532                         return ret;
 533         }
 534
 535         return DATAOUT_NORMAL;
 536 }
 537
 538 static int iscsit_dataout_post_crc_passed(
 539         struct iscsi_cmd *cmd,
 540         unsigned char *buf)
 541 {
 542         int ret, send_r2t = 0;
 543         struct iscsi_conn *conn = cmd->conn;
 544         struct iscsi_seq *seq = NULL;
 545         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 546         u32 payload_length = ntoh24(hdr->dlength);
 547
 548         if (cmd->unsolicited_data) {
 549                 if ((cmd->first_burst_len + payload_length) ==
 550                      conn->sess->sess_ops->FirstBurstLength) {
 551                         if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
 552                                         payload_length) < 0)
 553                                 return DATAOUT_CANNOT_RECOVER;
 554                         send_r2t = 1;
 555                 }
 556
 557                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 558                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 559                                 be32_to_cpu(hdr->datasn),
 560                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 561                         if (ret == DATAOUT_CANNOT_RECOVER)
 562                                 return ret;
 563                 }
 564
 565                 cmd->first_burst_len += payload_length;
 566
 567                 if (conn->sess->sess_ops->DataSequenceInOrder)
 568                         cmd->data_sn++;
 569                 else {
 570                         seq = cmd->seq_ptr;
 571                         seq->data_sn++;
 572                         seq->offset += payload_length;
 573                 }
 574
 575                 if (send_r2t) {
 576                         if (seq)
 577                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 578                         cmd->first_burst_len = 0;
 579                         cmd->unsolicited_data = 0;
 580                 }
 581         } else {
 582                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 583                         if ((cmd->next_burst_len + payload_length) ==
 584                              conn->sess->sess_ops->MaxBurstLength) {
 585                                 if (iscsit_dataout_update_r2t(cmd,
 586                                                 be32_to_cpu(hdr->offset),
 587                                                 payload_length) < 0)
 588                                         return DATAOUT_CANNOT_RECOVER;
 589                                 send_r2t = 1;
 590                         }
 591
 592                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 593                                 ret = iscsit_dataout_update_datapduinorder_no(
 594                                                 cmd, be32_to_cpu(hdr->datasn),
 595                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 596                                 if (ret == DATAOUT_CANNOT_RECOVER)
 597                                         return ret;
 598                         }
 599
 600                         cmd->next_burst_len += payload_length;
 601                         cmd->data_sn++;
 602
 603                         if (send_r2t)
 604                                 cmd->next_burst_len = 0;
 605                 } else {
 606                         seq = cmd->seq_ptr;
 607
 608                         if ((seq->next_burst_len + payload_length) ==
 609                              seq->xfer_len) {
 610                                 if (iscsit_dataout_update_r2t(cmd,
 611                                                 be32_to_cpu(hdr->offset),
 612                                                 payload_length) < 0)
 613                                         return DATAOUT_CANNOT_RECOVER;
 614                                 send_r2t = 1;
 615                         }
 616
 617                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 618                                 ret = iscsit_dataout_update_datapduinorder_no(
 619                                                 cmd, be32_to_cpu(hdr->datasn),
 620                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 621                                 if (ret == DATAOUT_CANNOT_RECOVER)
 622                                         return ret;
 623                         }
 624
 625                         seq->data_sn++;
 626                         seq->offset += payload_length;
 627                         seq->next_burst_len += payload_length;
 628
 629                         if (send_r2t) {
 630                                 seq->next_burst_len = 0;
 631                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 632                         }
 633                 }
 634         }
 635
 636         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 637                 cmd->data_sn = 0;
 638
 639         cmd->write_data_done += payload_length;
 640
 641         if (cmd->write_data_done == cmd->se_cmd.data_length)
 642                 return DATAOUT_SEND_TO_TRANSPORT;
 643         else if (send_r2t)
 644                 return DATAOUT_SEND_R2T;
 645         else
 646                 return DATAOUT_NORMAL;
 647 }
 648
 649 static int iscsit_dataout_post_crc_failed(
 650         struct iscsi_cmd *cmd,
 651         unsigned char *buf)
 652 {
 653         struct iscsi_conn *conn = cmd->conn;
 654         struct iscsi_pdu *pdu;
 655         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 656         u32 payload_length = ntoh24(hdr->dlength);
 657
 658         if (conn->sess->sess_ops->DataPDUInOrder)
 659                 goto recover;
 660         /*
 661          * The rest of this function is only called when DataPDUInOrder=No.
 662          */
 663         pdu = cmd->pdu_ptr;
 664
 665         switch (pdu->status) {
 666         case ISCSI_PDU_NOT_RECEIVED:
 667                 pdu->status = ISCSI_PDU_CRC_FAILED;
 668                 break;
 669         case ISCSI_PDU_CRC_FAILED:
 670                 break;
 671         case ISCSI_PDU_TIMED_OUT:
 672                 pdu->status = ISCSI_PDU_CRC_FAILED;
 673                 break;
 674         default:
 675                 return DATAOUT_CANNOT_RECOVER;
 676         }
 677
 678 recover:
 679         return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
 680                                                 payload_length);
 681 }
 682
 683 /*
 684  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 685  *      and CRC computed.
 686  */
 687 int iscsit_check_pre_dataout(
 688         struct iscsi_cmd *cmd,
 689         unsigned char *buf)
 690 {
 691         int ret;
 692         struct iscsi_conn *conn = cmd->conn;
 693
 694         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 695         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 696             (ret == DATAOUT_CANNOT_RECOVER))
 697                 return ret;
 698
 699         ret = iscsit_dataout_check_datasn(cmd, buf);
 700         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 701             (ret == DATAOUT_CANNOT_RECOVER))
 702                 return ret;
 703
 704         if (cmd->unsolicited_data) {
 705                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 706                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 707                     (ret == DATAOUT_CANNOT_RECOVER))
 708                         return ret;
 709         } else {
 710                 ret = iscsit_dataout_check_sequence(cmd, buf);
 711                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 712                     (ret == DATAOUT_CANNOT_RECOVER))
 713                         return ret;
 714         }
 715
 716         return (conn->sess->sess_ops->DataPDUInOrder) ?
 717                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 718                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 719 }
 720
 721 /*
 722  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 723  *      and CRC computed.
 724  */
 725 int iscsit_check_post_dataout(
 726         struct iscsi_cmd *cmd,
 727         unsigned char *buf,
 728         u8 data_crc_failed)
 729 {
 730         struct iscsi_conn *conn = cmd->conn;
 731
 732         cmd->dataout_timeout_retries = 0;
 733
 734         if (!data_crc_failed)
 735                 return iscsit_dataout_post_crc_passed(cmd, buf);
 736         else {
 737                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 738                         pr_err("Unable to recover from DataOUT CRC"
 739                                 " failure while ERL=0, closing session.\n");
 740                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 741                                           buf);
 742                         return DATAOUT_CANNOT_RECOVER;
 743                 }
 744
 745                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
 746                 return iscsit_dataout_post_crc_failed(cmd, buf);
 747         }
 748 }
 749
 750 static void iscsit_handle_time2retain_timeout(unsigned long data)
 751 {
 752         struct iscsi_session *sess = (struct iscsi_session *) data;
 753         struct iscsi_portal_group *tpg = sess->tpg;
 754         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 755
 756         spin_lock_bh(&se_tpg->session_lock);
 757         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 758                 spin_unlock_bh(&se_tpg->session_lock);
 759                 return;
 760         }
 761         if (atomic_read(&sess->session_reinstatement)) {
 762                 pr_err("Exiting Time2Retain handler because"
 763                                 " session_reinstatement=1\n");
 764                 spin_unlock_bh(&se_tpg->session_lock);
 765                 return;
 766         }
 767         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 768
 769         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 770                         " iSCSI session.\n", sess->sid);
 771         {
 772         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 773
 774         if (tiqn) {
 775                 spin_lock(&tiqn->sess_err_stats.lock);
 776                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 777                         (void *)sess->sess_ops->InitiatorName);
 778                 tiqn->sess_err_stats.last_sess_failure_type =
 779                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 780                 tiqn->sess_err_stats.cxn_timeout_errors++;
 781                 atomic_long_inc(&sess->conn_timeout_errors);
 782                 spin_unlock(&tiqn->sess_err_stats.lock);
 783         }
 784         }
 785
 786         spin_unlock_bh(&se_tpg->session_lock);
 787         target_put_session(sess->se_sess);
 788 }
 789
 790 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 791 {
 792         int tpg_active;
 793         /*
 794          * Only start Time2Retain timer when the associated TPG is still in
 795          * an ACTIVE (eg: not disabled or shutdown) state.
 796          */
 797         spin_lock(&sess->tpg->tpg_state_lock);
 798         tpg_active = (sess->tpg->tpg_state == TPG_STATE_ACTIVE);
 799         spin_unlock(&sess->tpg->tpg_state_lock);
 800
 801         if (!tpg_active)
 802                 return;
 803
 804         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 805                 return;
 806
 807         pr_debug("Starting Time2Retain timer for %u seconds on"
 808                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 809
 810         init_timer(&sess->time2retain_timer);
 811         sess->time2retain_timer.expires =
 812                 (get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 813         sess->time2retain_timer.data = (unsigned long)sess;
 814         sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 815         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 816         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 817         add_timer(&sess->time2retain_timer);
 818 }
 819
 820 /*
 821  *      Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 822  */
 823 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 824 {
 825         struct iscsi_portal_group *tpg = sess->tpg;
 826         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 827
 828         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 829                 return -1;
 830
 831         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 832                 return 0;
 833
 834         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 835         spin_unlock(&se_tpg->session_lock);
 836
 837         del_timer_sync(&sess->time2retain_timer);
 838
 839         spin_lock(&se_tpg->session_lock);
 840         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 841         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 842                         sess->sid);
 843         return 0;
 844 }
 845
 846 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 847 {
 848         spin_lock_bh(&conn->state_lock);
 849         if (atomic_read(&conn->connection_exit)) {
 850                 spin_unlock_bh(&conn->state_lock);
 851                 goto sleep;
 852         }
 853
 854         if (atomic_read(&conn->transport_failed)) {
 855                 spin_unlock_bh(&conn->state_lock);
 856                 goto sleep;
 857         }
 858         spin_unlock_bh(&conn->state_lock);
 859
 860         if (conn->tx_thread && conn->tx_thread_active)
 861                 send_sig(SIGINT, conn->tx_thread, 1);
 862         if (conn->rx_thread && conn->rx_thread_active)
 863                 send_sig(SIGINT, conn->rx_thread, 1);
 864
 865 sleep:
 866         wait_for_completion(&conn->conn_wait_rcfr_comp);
 867         complete(&conn->conn_post_wait_comp);
 868 }
 869
 870 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 871 {
 872         spin_lock_bh(&conn->state_lock);
 873         if (atomic_read(&conn->connection_exit)) {
 874                 spin_unlock_bh(&conn->state_lock);
 875                 return;
 876         }
 877
 878         if (atomic_read(&conn->transport_failed)) {
 879                 spin_unlock_bh(&conn->state_lock);
 880                 return;
 881         }
 882
 883         if (atomic_read(&conn->connection_reinstatement)) {
 884                 spin_unlock_bh(&conn->state_lock);
 885                 return;
 886         }
 887
 888         if (conn->tx_thread && conn->tx_thread_active)
 889                 send_sig(SIGINT, conn->tx_thread, 1);
 890         if (conn->rx_thread && conn->rx_thread_active)
 891                 send_sig(SIGINT, conn->rx_thread, 1);
 892
 893         atomic_set(&conn->connection_reinstatement, 1);
 894         if (!sleep) {
 895                 spin_unlock_bh(&conn->state_lock);
 896                 return;
 897         }
 898
 899         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 900         spin_unlock_bh(&conn->state_lock);
 901
 902         wait_for_completion(&conn->conn_wait_comp);
 903         complete(&conn->conn_post_wait_comp);
 904 }
 905 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
 906
 907 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 908 {
 909         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 910                         " %u\n", sess->sid);
 911
 912         atomic_set(&sess->session_fall_back_to_erl0, 1);
 913 }
 914
 915 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 916 {
 917         struct iscsi_session *sess = conn->sess;
 918
 919         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 920             !atomic_read(&sess->session_reinstatement) &&
 921             !atomic_read(&sess->session_fall_back_to_erl0))
 922                 iscsit_connection_recovery_transport_reset(conn);
 923         else {
 924                 pr_debug("Performing cleanup for failed iSCSI"
 925                         " Connection ID: %hu from %s\n", conn->cid,
 926                         sess->sess_ops->InitiatorName);
 927                 iscsit_close_connection(conn);
 928         }
 929 }
 930
 931 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn, bool *conn_freed)
 932 {
 933         *conn_freed = false;
 934
 935         spin_lock_bh(&conn->state_lock);
 936         if (atomic_read(&conn->connection_exit)) {
 937                 spin_unlock_bh(&conn->state_lock);
 938                 return;
 939         }
 940         atomic_set(&conn->connection_exit, 1);
 941
 942         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 943                 spin_unlock_bh(&conn->state_lock);
 944                 iscsit_close_connection(conn);
 945                 *conn_freed = true;
 946                 return;
 947         }
 948
 949         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 950                 spin_unlock_bh(&conn->state_lock);
 951                 return;
 952         }
 953
 954         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 955         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 956         spin_unlock_bh(&conn->state_lock);
 957
 958         iscsit_handle_connection_cleanup(conn);
 959         *conn_freed = true;
 960 }