drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   5  *
   6  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   7  *
   8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  ******************************************************************************/
  20
  21 #include <linux/string.h>
  22 #include <linux/kthread.h>
  23 #include <linux/crypto.h>
  24 #include <linux/completion.h>
  25 #include <linux/module.h>
  26 #include <linux/idr.h>
  27 #include <asm/unaligned.h>
  28 #include <scsi/scsi_device.h>
  29 #include <scsi/iscsi_proto.h>
  30 #include <scsi/scsi_tcq.h>
  31 #include <target/target_core_base.h>
  32 #include <target/target_core_fabric.h>
  33 #include <target/target_core_configfs.h>
  34
  35 #include "iscsi_target_core.h"
  36 #include "iscsi_target_parameters.h"
  37 #include "iscsi_target_seq_pdu_list.h"
  38 #include "iscsi_target_tq.h"
  39 #include "iscsi_target_configfs.h"
  40 #include "iscsi_target_datain_values.h"
  41 #include "iscsi_target_erl0.h"
  42 #include "iscsi_target_erl1.h"
  43 #include "iscsi_target_erl2.h"
  44 #include "iscsi_target_login.h"
  45 #include "iscsi_target_tmr.h"
  46 #include "iscsi_target_tpg.h"
  47 #include "iscsi_target_util.h"
  48 #include "iscsi_target.h"
  49 #include "iscsi_target_device.h"
  50 #include "iscsi_target_stat.h"
  51
  52 #include <target/iscsi/iscsi_transport.h>
  53
  54 static LIST_HEAD(g_tiqn_list);
  55 static LIST_HEAD(g_np_list);
  56 static DEFINE_SPINLOCK(tiqn_lock);
  57 static DEFINE_MUTEX(np_lock);
  58
  59 static struct idr tiqn_idr;
  60 struct idr sess_idr;
  61 struct mutex auth_id_lock;
  62 spinlock_t sess_idr_lock;
  63
  64 struct iscsit_global *iscsit_global;
  65
  66 struct kmem_cache *lio_cmd_cache;
  67 struct kmem_cache *lio_qr_cache;
  68 struct kmem_cache *lio_dr_cache;
  69 struct kmem_cache *lio_ooo_cache;
  70 struct kmem_cache *lio_r2t_cache;
  71
  72 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  73                         struct iscsi_scsi_req *, u32);
  74
  75 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  76 {
  77         struct iscsi_tiqn *tiqn = NULL;
  78
  79         spin_lock(&tiqn_lock);
  80         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  81                 if (!strcmp(tiqn->tiqn, buf)) {
  82
  83                         spin_lock(&tiqn->tiqn_state_lock);
  84                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  85                                 tiqn->tiqn_access_count++;
  86                                 spin_unlock(&tiqn->tiqn_state_lock);
  87                                 spin_unlock(&tiqn_lock);
  88                                 return tiqn;
  89                         }
  90                         spin_unlock(&tiqn->tiqn_state_lock);
  91                 }
  92         }
  93         spin_unlock(&tiqn_lock);
  94
  95         return NULL;
  96 }
  97
  98 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  99 {
 100         spin_lock(&tiqn->tiqn_state_lock);
 101         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
 102                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 103                 spin_unlock(&tiqn->tiqn_state_lock);
 104                 return 0;
 105         }
 106         spin_unlock(&tiqn->tiqn_state_lock);
 107
 108         return -1;
 109 }
 110
 111 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 112 {
 113         spin_lock(&tiqn->tiqn_state_lock);
 114         tiqn->tiqn_access_count--;
 115         spin_unlock(&tiqn->tiqn_state_lock);
 116 }
 117
 118 /*
 119  * Note that IQN formatting is expected to be done in userspace, and
 120  * no explict IQN format checks are done here.
 121  */
 122 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 123 {
 124         struct iscsi_tiqn *tiqn = NULL;
 125         int ret;
 126
 127         if (strlen(buf) >= ISCSI_IQN_LEN) {
 128                 pr_err("Target IQN exceeds %d bytes\n",
 129                                 ISCSI_IQN_LEN);
 130                 return ERR_PTR(-EINVAL);
 131         }
 132
 133         tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
 134         if (!tiqn) {
 135                 pr_err("Unable to allocate struct iscsi_tiqn\n");
 136                 return ERR_PTR(-ENOMEM);
 137         }
 138
 139         sprintf(tiqn->tiqn, "%s", buf);
 140         INIT_LIST_HEAD(&tiqn->tiqn_list);
 141         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 142         spin_lock_init(&tiqn->tiqn_state_lock);
 143         spin_lock_init(&tiqn->tiqn_tpg_lock);
 144         spin_lock_init(&tiqn->sess_err_stats.lock);
 145         spin_lock_init(&tiqn->login_stats.lock);
 146         spin_lock_init(&tiqn->logout_stats.lock);
 147
 148         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 149
 150         idr_preload(GFP_KERNEL);
 151         spin_lock(&tiqn_lock);
 152
 153         ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
 154         if (ret < 0) {
 155                 pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
 156                 spin_unlock(&tiqn_lock);
 157                 idr_preload_end();
 158                 kfree(tiqn);
 159                 return ERR_PTR(ret);
 160         }
 161         tiqn->tiqn_index = ret;
 162         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 163
 164         spin_unlock(&tiqn_lock);
 165         idr_preload_end();
 166
 167         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 168
 169         return tiqn;
 170
 171 }
 172
 173 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 174 {
 175         /*
 176          * Wait for accesses to said struct iscsi_tiqn to end.
 177          */
 178         spin_lock(&tiqn->tiqn_state_lock);
 179         while (tiqn->tiqn_access_count != 0) {
 180                 spin_unlock(&tiqn->tiqn_state_lock);
 181                 msleep(10);
 182                 spin_lock(&tiqn->tiqn_state_lock);
 183         }
 184         spin_unlock(&tiqn->tiqn_state_lock);
 185 }
 186
 187 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 188 {
 189         /*
 190          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 191          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 192          * attempts to access this struct iscsi_tiqn will fail from both transport
 193          * fabric and control code paths.
 194          */
 195         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 196                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 197                 return;
 198         }
 199
 200         iscsit_wait_for_tiqn(tiqn);
 201
 202         spin_lock(&tiqn_lock);
 203         list_del(&tiqn->tiqn_list);
 204         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 205         spin_unlock(&tiqn_lock);
 206
 207         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 208                         tiqn->tiqn);
 209         kfree(tiqn);
 210 }
 211
 212 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 213 {
 214         int ret;
 215         /*
 216          * Determine if the network portal is accepting storage traffic.
 217          */
 218         spin_lock_bh(&np->np_thread_lock);
 219         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 220                 spin_unlock_bh(&np->np_thread_lock);
 221                 return -1;
 222         }
 223         if (np->np_login_tpg) {
 224                 pr_err("np->np_login_tpg() is not NULL!\n");
 225                 spin_unlock_bh(&np->np_thread_lock);
 226                 return -1;
 227         }
 228         spin_unlock_bh(&np->np_thread_lock);
 229         /*
 230          * Determine if the portal group is accepting storage traffic.
 231          */
 232         spin_lock_bh(&tpg->tpg_state_lock);
 233         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 234                 spin_unlock_bh(&tpg->tpg_state_lock);
 235                 return -1;
 236         }
 237         spin_unlock_bh(&tpg->tpg_state_lock);
 238
 239         /*
 240          * Here we serialize access across the TIQN+TPG Tuple.
 241          */
 242         ret = mutex_lock_interruptible(&tpg->np_login_lock);
 243         if ((ret != 0) || signal_pending(current))
 244                 return -1;
 245
 246         spin_lock_bh(&np->np_thread_lock);
 247         np->np_login_tpg = tpg;
 248         spin_unlock_bh(&np->np_thread_lock);
 249
 250         return 0;
 251 }
 252
 253 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 254 {
 255         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 256
 257         spin_lock_bh(&np->np_thread_lock);
 258         np->np_login_tpg = NULL;
 259         spin_unlock_bh(&np->np_thread_lock);
 260
 261         mutex_unlock(&tpg->np_login_lock);
 262
 263         if (tiqn)
 264                 iscsit_put_tiqn_for_login(tiqn);
 265
 266         return 0;
 267 }
 268
 269 bool iscsit_check_np_match(
 270         struct __kernel_sockaddr_storage *sockaddr,
 271         struct iscsi_np *np,
 272         int network_transport)
 273 {
 274         struct sockaddr_in *sock_in, *sock_in_e;
 275         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 276         bool ip_match = false;
 277         u16 port;
 278
 279         if (sockaddr->ss_family == AF_INET6) {
 280                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 281                 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 282
 283                 if (!memcmp(&sock_in6->sin6_addr.in6_u,
 284                             &sock_in6_e->sin6_addr.in6_u,
 285                             sizeof(struct in6_addr)))
 286                         ip_match = true;
 287
 288                 port = ntohs(sock_in6->sin6_port);
 289         } else {
 290                 sock_in = (struct sockaddr_in *)sockaddr;
 291                 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 292
 293                 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
 294                         ip_match = true;
 295
 296                 port = ntohs(sock_in->sin_port);
 297         }
 298
 299         if ((ip_match == true) && (np->np_port == port) &&
 300             (np->np_network_transport == network_transport))
 301                 return true;
 302
 303         return false;
 304 }
 305
 306 /*
 307  * Called with mutex np_lock held
 308  */
 309 static struct iscsi_np *iscsit_get_np(
 310         struct __kernel_sockaddr_storage *sockaddr,
 311         int network_transport)
 312 {
 313         struct iscsi_np *np;
 314         bool match;
 315
 316         list_for_each_entry(np, &g_np_list, np_list) {
 317                 spin_lock_bh(&np->np_thread_lock);
 318                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 319                         spin_unlock_bh(&np->np_thread_lock);
 320                         continue;
 321                 }
 322
 323                 match = iscsit_check_np_match(sockaddr, np, network_transport);
 324                 if (match == true) {
 325                         /*
 326                          * Increment the np_exports reference count now to
 327                          * prevent iscsit_del_np() below from being called
 328                          * while iscsi_tpg_add_network_portal() is called.
 329                          */
 330                         np->np_exports++;
 331                         spin_unlock_bh(&np->np_thread_lock);
 332                         return np;
 333                 }
 334                 spin_unlock_bh(&np->np_thread_lock);
 335         }
 336
 337         return NULL;
 338 }
 339
 340 struct iscsi_np *iscsit_add_np(
 341         struct __kernel_sockaddr_storage *sockaddr,
 342         char *ip_str,
 343         int network_transport)
 344 {
 345         struct sockaddr_in *sock_in;
 346         struct sockaddr_in6 *sock_in6;
 347         struct iscsi_np *np;
 348         int ret;
 349
 350         mutex_lock(&np_lock);
 351
 352         /*
 353          * Locate the existing struct iscsi_np if already active..
 354          */
 355         np = iscsit_get_np(sockaddr, network_transport);
 356         if (np) {
 357                 mutex_unlock(&np_lock);
 358                 return np;
 359         }
 360
 361         np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
 362         if (!np) {
 363                 pr_err("Unable to allocate memory for struct iscsi_np\n");
 364                 mutex_unlock(&np_lock);
 365                 return ERR_PTR(-ENOMEM);
 366         }
 367
 368         np->np_flags |= NPF_IP_NETWORK;
 369         if (sockaddr->ss_family == AF_INET6) {
 370                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 371                 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str);
 372                 np->np_port = ntohs(sock_in6->sin6_port);
 373         } else {
 374                 sock_in = (struct sockaddr_in *)sockaddr;
 375                 sprintf(np->np_ip, "%s", ip_str);
 376                 np->np_port = ntohs(sock_in->sin_port);
 377         }
 378
 379         np->np_network_transport = network_transport;
 380         spin_lock_init(&np->np_thread_lock);
 381         init_completion(&np->np_restart_comp);
 382         INIT_LIST_HEAD(&np->np_list);
 383
 384         ret = iscsi_target_setup_login_socket(np, sockaddr);
 385         if (ret != 0) {
 386                 kfree(np);
 387                 mutex_unlock(&np_lock);
 388                 return ERR_PTR(ret);
 389         }
 390
 391         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 392         if (IS_ERR(np->np_thread)) {
 393                 pr_err("Unable to create kthread: iscsi_np\n");
 394                 ret = PTR_ERR(np->np_thread);
 395                 kfree(np);
 396                 mutex_unlock(&np_lock);
 397                 return ERR_PTR(ret);
 398         }
 399         /*
 400          * Increment the np_exports reference count now to prevent
 401          * iscsit_del_np() below from being run while a new call to
 402          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 403          * active.  We don't need to hold np->np_thread_lock at this
 404          * point because iscsi_np has not been added to g_np_list yet.
 405          */
 406         np->np_exports = 1;
 407         np->np_thread_state = ISCSI_NP_THREAD_ACTIVE;
 408
 409         list_add_tail(&np->np_list, &g_np_list);
 410         mutex_unlock(&np_lock);
 411
 412         pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
 413                 np->np_ip, np->np_port, np->np_transport->name);
 414
 415         return np;
 416 }
 417
 418 int iscsit_reset_np_thread(
 419         struct iscsi_np *np,
 420         struct iscsi_tpg_np *tpg_np,
 421         struct iscsi_portal_group *tpg)
 422 {
 423         spin_lock_bh(&np->np_thread_lock);
 424         if (tpg && tpg_np) {
 425                 /*
 426                  * The reset operation need only be performed when the
 427                  * passed struct iscsi_portal_group has a login in progress
 428                  * to one of the network portals.
 429                  */
 430                 if (tpg_np->tpg_np->np_login_tpg != tpg) {
 431                         spin_unlock_bh(&np->np_thread_lock);
 432                         return 0;
 433                 }
 434         }
 435         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 436                 spin_unlock_bh(&np->np_thread_lock);
 437                 return 0;
 438         }
 439         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 440
 441         if (np->np_thread) {
 442                 spin_unlock_bh(&np->np_thread_lock);
 443                 send_sig(SIGINT, np->np_thread, 1);
 444                 wait_for_completion(&np->np_restart_comp);
 445                 spin_lock_bh(&np->np_thread_lock);
 446         }
 447         spin_unlock_bh(&np->np_thread_lock);
 448
 449         return 0;
 450 }
 451
 452 static void iscsit_free_np(struct iscsi_np *np)
 453 {
 454         if (np->np_socket)
 455                 sock_release(np->np_socket);
 456 }
 457
 458 int iscsit_del_np(struct iscsi_np *np)
 459 {
 460         spin_lock_bh(&np->np_thread_lock);
 461         np->np_exports--;
 462         if (np->np_exports) {
 463                 spin_unlock_bh(&np->np_thread_lock);
 464                 return 0;
 465         }
 466         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 467         spin_unlock_bh(&np->np_thread_lock);
 468
 469         if (np->np_thread) {
 470                 /*
 471                  * We need to send the signal to wakeup Linux/Net
 472                  * which may be sleeping in sock_accept()..
 473                  */
 474                 send_sig(SIGINT, np->np_thread, 1);
 475                 kthread_stop(np->np_thread);
 476         }
 477
 478         np->np_transport->iscsit_free_np(np);
 479
 480         mutex_lock(&np_lock);
 481         list_del(&np->np_list);
 482         mutex_unlock(&np_lock);
 483
 484         pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
 485                 np->np_ip, np->np_port, np->np_transport->name);
 486
 487         iscsit_put_transport(np->np_transport);
 488         kfree(np);
 489         return 0;
 490 }
 491
 492 static int iscsit_immediate_queue(struct iscsi_conn *, struct iscsi_cmd *, int);
 493 static int iscsit_response_queue(struct iscsi_conn *, struct iscsi_cmd *, int);
 494
 495 static int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 496 {
 497         iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
 498         return 0;
 499 }
 500
 501 static struct iscsit_transport iscsi_target_transport = {
 502         .name                   = "iSCSI/TCP",
 503         .transport_type         = ISCSI_TCP,
 504         .owner                  = NULL,
 505         .iscsit_setup_np        = iscsit_setup_np,
 506         .iscsit_accept_np       = iscsit_accept_np,
 507         .iscsit_free_np         = iscsit_free_np,
 508         .iscsit_alloc_cmd       = iscsit_alloc_cmd,
 509         .iscsit_get_login_rx    = iscsit_get_login_rx,
 510         .iscsit_put_login_tx    = iscsit_put_login_tx,
 511         .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
 512         .iscsit_immediate_queue = iscsit_immediate_queue,
 513         .iscsit_response_queue  = iscsit_response_queue,
 514         .iscsit_queue_data_in   = iscsit_queue_rsp,
 515         .iscsit_queue_status    = iscsit_queue_rsp,
 516 };
 517
 518 static int __init iscsi_target_init_module(void)
 519 {
 520         int ret = 0;
 521
 522         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 523
 524         iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
 525         if (!iscsit_global) {
 526                 pr_err("Unable to allocate memory for iscsit_global\n");
 527                 return -1;
 528         }
 529         mutex_init(&auth_id_lock);
 530         spin_lock_init(&sess_idr_lock);
 531         idr_init(&tiqn_idr);
 532         idr_init(&sess_idr);
 533
 534         ret = iscsi_target_register_configfs();
 535         if (ret < 0)
 536                 goto out;
 537
 538         ret = iscsi_thread_set_init();
 539         if (ret < 0)
 540                 goto configfs_out;
 541
 542         if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
 543                         TARGET_THREAD_SET_COUNT) {
 544                 pr_err("iscsi_allocate_thread_sets() returned"
 545                         " unexpected value!\n");
 546                 goto ts_out1;
 547         }
 548
 549         lio_cmd_cache = kmem_cache_create("lio_cmd_cache",
 550                         sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd),
 551                         0, NULL);
 552         if (!lio_cmd_cache) {
 553                 pr_err("Unable to kmem_cache_create() for"
 554                                 " lio_cmd_cache\n");
 555                 goto ts_out2;
 556         }
 557
 558         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 559                         sizeof(struct iscsi_queue_req),
 560                         __alignof__(struct iscsi_queue_req), 0, NULL);
 561         if (!lio_qr_cache) {
 562                 pr_err("nable to kmem_cache_create() for"
 563                                 " lio_qr_cache\n");
 564                 goto cmd_out;
 565         }
 566
 567         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 568                         sizeof(struct iscsi_datain_req),
 569                         __alignof__(struct iscsi_datain_req), 0, NULL);
 570         if (!lio_dr_cache) {
 571                 pr_err("Unable to kmem_cache_create() for"
 572                                 " lio_dr_cache\n");
 573                 goto qr_out;
 574         }
 575
 576         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 577                         sizeof(struct iscsi_ooo_cmdsn),
 578                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 579         if (!lio_ooo_cache) {
 580                 pr_err("Unable to kmem_cache_create() for"
 581                                 " lio_ooo_cache\n");
 582                 goto dr_out;
 583         }
 584
 585         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 586                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 587                         0, NULL);
 588         if (!lio_r2t_cache) {
 589                 pr_err("Unable to kmem_cache_create() for"
 590                                 " lio_r2t_cache\n");
 591                 goto ooo_out;
 592         }
 593
 594         iscsit_register_transport(&iscsi_target_transport);
 595
 596         if (iscsit_load_discovery_tpg() < 0)
 597                 goto r2t_out;
 598
 599         return ret;
 600 r2t_out:
 601         kmem_cache_destroy(lio_r2t_cache);
 602 ooo_out:
 603         kmem_cache_destroy(lio_ooo_cache);
 604 dr_out:
 605         kmem_cache_destroy(lio_dr_cache);
 606 qr_out:
 607         kmem_cache_destroy(lio_qr_cache);
 608 cmd_out:
 609         kmem_cache_destroy(lio_cmd_cache);
 610 ts_out2:
 611         iscsi_deallocate_thread_sets();
 612 ts_out1:
 613         iscsi_thread_set_free();
 614 configfs_out:
 615         iscsi_target_deregister_configfs();
 616 out:
 617         kfree(iscsit_global);
 618         return -ENOMEM;
 619 }
 620
 621 static void __exit iscsi_target_cleanup_module(void)
 622 {
 623         iscsi_deallocate_thread_sets();
 624         iscsi_thread_set_free();
 625         iscsit_release_discovery_tpg();
 626         iscsit_unregister_transport(&iscsi_target_transport);
 627         kmem_cache_destroy(lio_cmd_cache);
 628         kmem_cache_destroy(lio_qr_cache);
 629         kmem_cache_destroy(lio_dr_cache);
 630         kmem_cache_destroy(lio_ooo_cache);
 631         kmem_cache_destroy(lio_r2t_cache);
 632
 633         iscsi_target_deregister_configfs();
 634
 635         kfree(iscsit_global);
 636 }
 637
 638 static int iscsit_add_reject(
 639         struct iscsi_conn *conn,
 640         u8 reason,
 641         unsigned char *buf)
 642 {
 643         struct iscsi_cmd *cmd;
 644
 645         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
 646         if (!cmd)
 647                 return -1;
 648
 649         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 650         cmd->reject_reason = reason;
 651
 652         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 653         if (!cmd->buf_ptr) {
 654                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 655                 iscsit_free_cmd(cmd, false);
 656                 return -1;
 657         }
 658
 659         spin_lock_bh(&conn->cmd_lock);
 660         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 661         spin_unlock_bh(&conn->cmd_lock);
 662
 663         cmd->i_state = ISTATE_SEND_REJECT;
 664         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 665
 666         return -1;
 667 }
 668
 669 static int iscsit_add_reject_from_cmd(
 670         struct iscsi_cmd *cmd,
 671         u8 reason,
 672         bool add_to_conn,
 673         unsigned char *buf)
 674 {
 675         struct iscsi_conn *conn;
 676
 677         if (!cmd->conn) {
 678                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 679                                 cmd->init_task_tag);
 680                 return -1;
 681         }
 682         conn = cmd->conn;
 683
 684         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 685         cmd->reject_reason = reason;
 686
 687         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 688         if (!cmd->buf_ptr) {
 689                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 690                 iscsit_free_cmd(cmd, false);
 691                 return -1;
 692         }
 693
 694         if (add_to_conn) {
 695                 spin_lock_bh(&conn->cmd_lock);
 696                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 697                 spin_unlock_bh(&conn->cmd_lock);
 698         }
 699
 700         cmd->i_state = ISTATE_SEND_REJECT;
 701         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 702         /*
 703          * Perform the kref_put now if se_cmd has already been setup by
 704          * scsit_setup_scsi_cmd()
 705          */
 706         if (cmd->se_cmd.se_tfo != NULL) {
 707                 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
 708                 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
 709         }
 710         return -1;
 711 }
 712
 713 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason,
 714                                  unsigned char *buf)
 715 {
 716         return iscsit_add_reject_from_cmd(cmd, reason, true, buf);
 717 }
 718
 719 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf)
 720 {
 721         return iscsit_add_reject_from_cmd(cmd, reason, false, buf);
 722 }
 723
 724 /*
 725  * Map some portion of the allocated scatterlist to an iovec, suitable for
 726  * kernel sockets to copy data in/out.
 727  */
 728 static int iscsit_map_iovec(
 729         struct iscsi_cmd *cmd,
 730         struct kvec *iov,
 731         u32 data_offset,
 732         u32 data_length)
 733 {
 734         u32 i = 0;
 735         struct scatterlist *sg;
 736         unsigned int page_off;
 737
 738         /*
 739          * We know each entry in t_data_sg contains a page.
 740          */
 741         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 742         page_off = (data_offset % PAGE_SIZE);
 743
 744         cmd->first_data_sg = sg;
 745         cmd->first_data_sg_off = page_off;
 746
 747         while (data_length) {
 748                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 749
 750                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 751                 iov[i].iov_len = cur_len;
 752
 753                 data_length -= cur_len;
 754                 page_off = 0;
 755                 sg = sg_next(sg);
 756                 i++;
 757         }
 758
 759         cmd->kmapped_nents = i;
 760
 761         return i;
 762 }
 763
 764 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 765 {
 766         u32 i;
 767         struct scatterlist *sg;
 768
 769         sg = cmd->first_data_sg;
 770
 771         for (i = 0; i < cmd->kmapped_nents; i++)
 772                 kunmap(sg_page(&sg[i]));
 773 }
 774
 775 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 776 {
 777         struct iscsi_cmd *cmd;
 778
 779         conn->exp_statsn = exp_statsn;
 780
 781         if (conn->sess->sess_ops->RDMAExtensions)
 782                 return;
 783
 784         spin_lock_bh(&conn->cmd_lock);
 785         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 786                 spin_lock(&cmd->istate_lock);
 787                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 788                     iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
 789                         cmd->i_state = ISTATE_REMOVE;
 790                         spin_unlock(&cmd->istate_lock);
 791                         iscsit_add_cmd_to_immediate_queue(cmd, conn,
 792                                                 cmd->i_state);
 793                         continue;
 794                 }
 795                 spin_unlock(&cmd->istate_lock);
 796         }
 797         spin_unlock_bh(&conn->cmd_lock);
 798 }
 799
 800 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 801 {
 802         u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
 803
 804         iov_count += ISCSI_IOV_DATA_BUFFER;
 805
 806         cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
 807         if (!cmd->iov_data) {
 808                 pr_err("Unable to allocate cmd->iov_data\n");
 809                 return -ENOMEM;
 810         }
 811
 812         cmd->orig_iov_data_count = iov_count;
 813         return 0;
 814 }
 815
 816 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 817                           unsigned char *buf)
 818 {
 819         int data_direction, payload_length;
 820         struct iscsi_scsi_req *hdr;
 821         int iscsi_task_attr;
 822         int sam_task_attr;
 823
 824         spin_lock_bh(&conn->sess->session_stats_lock);
 825         conn->sess->cmd_pdus++;
 826         if (conn->sess->se_sess->se_node_acl) {
 827                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
 828                 conn->sess->se_sess->se_node_acl->num_cmds++;
 829                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
 830         }
 831         spin_unlock_bh(&conn->sess->session_stats_lock);
 832
 833         hdr                     = (struct iscsi_scsi_req *) buf;
 834         payload_length          = ntoh24(hdr->dlength);
 835
 836         /* FIXME; Add checks for AdditionalHeaderSegment */
 837
 838         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
 839             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
 840                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
 841                                 " not set. Bad iSCSI Initiator.\n");
 842                 return iscsit_add_reject_cmd(cmd,
 843                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
 844         }
 845
 846         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
 847              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
 848                 /*
 849                  * From RFC-3720 Section 10.3.1:
 850                  *
 851                  * "Either or both of R and W MAY be 1 when either the
 852                  *  Expected Data Transfer Length and/or Bidirectional Read
 853                  *  Expected Data Transfer Length are 0"
 854                  *
 855                  * For this case, go ahead and clear the unnecssary bits
 856                  * to avoid any confusion with ->data_direction.
 857                  */
 858                 hdr->flags &= ~ISCSI_FLAG_CMD_READ;
 859                 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
 860
 861                 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
 862                         " set when Expected Data Transfer Length is 0 for"
 863                         " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]);
 864         }
 865
 866         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
 867             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
 868                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
 869                         " MUST be set if Expected Data Transfer Length is not 0."
 870                         " Bad iSCSI Initiator\n");
 871                 return iscsit_add_reject_cmd(cmd,
 872                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
 873         }
 874
 875         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
 876             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
 877                 pr_err("Bidirectional operations not supported!\n");
 878                 return iscsit_add_reject_cmd(cmd,
 879                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
 880         }
 881
 882         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
 883                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
 884                                 " Scsi Command PDU.\n");
 885                 return iscsit_add_reject_cmd(cmd,
 886                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
 887         }
 888
 889         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
 890                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
 891                         " protocol error.\n", payload_length);
 892                 return iscsit_add_reject_cmd(cmd,
 893                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
 894         }
 895
 896         if ((be32_to_cpu(hdr->data_length) == payload_length) &&
 897             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
 898                 pr_err("Expected Data Transfer Length and Length of"
 899                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
 900                         " bit is not set protocol error\n");
 901                 return iscsit_add_reject_cmd(cmd,
 902                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
 903         }
 904
 905         if (payload_length > be32_to_cpu(hdr->data_length)) {
 906                 pr_err("DataSegmentLength: %u is greater than"
 907                         " EDTL: %u, protocol error.\n", payload_length,
 908                                 hdr->data_length);
 909                 return iscsit_add_reject_cmd(cmd,
 910                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
 911         }
 912
 913         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
 914                 pr_err("DataSegmentLength: %u is greater than"
 915                         " MaxXmitDataSegmentLength: %u, protocol error.\n",
 916                         payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
 917                 return iscsit_add_reject_cmd(cmd,
 918                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
 919         }
 920
 921         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
 922                 pr_err("DataSegmentLength: %u is greater than"
 923                         " FirstBurstLength: %u, protocol error.\n",
 924                         payload_length, conn->sess->sess_ops->FirstBurstLength);
 925                 return iscsit_add_reject_cmd(cmd,
 926                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
 927         }
 928
 929         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
 930                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
 931                           DMA_NONE;
 932
 933         cmd->data_direction = data_direction;
 934         iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
 935         /*
 936          * Figure out the SAM Task Attribute for the incoming SCSI CDB
 937          */
 938         if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
 939             (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
 940                 sam_task_attr = MSG_SIMPLE_TAG;
 941         else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
 942                 sam_task_attr = MSG_ORDERED_TAG;
 943         else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
 944                 sam_task_attr = MSG_HEAD_TAG;
 945         else if (iscsi_task_attr == ISCSI_ATTR_ACA)
 946                 sam_task_attr = MSG_ACA_TAG;
 947         else {
 948                 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
 949                         " MSG_SIMPLE_TAG\n", iscsi_task_attr);
 950                 sam_task_attr = MSG_SIMPLE_TAG;
 951         }
 952
 953         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
 954         cmd->i_state            = ISTATE_NEW_CMD;
 955         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
 956         cmd->immediate_data     = (payload_length) ? 1 : 0;
 957         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
 958                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
 959         if (cmd->unsolicited_data)
 960                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
 961
 962         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
 963         if (hdr->flags & ISCSI_FLAG_CMD_READ) {
 964                 spin_lock_bh(&conn->sess->ttt_lock);
 965                 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 966                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
 967                         cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 968                 spin_unlock_bh(&conn->sess->ttt_lock);
 969         } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE)
 970                 cmd->targ_xfer_tag = 0xFFFFFFFF;
 971         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
 972         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
 973         cmd->first_burst_len    = payload_length;
 974
 975         if (!conn->sess->sess_ops->RDMAExtensions &&
 976              cmd->data_direction == DMA_FROM_DEVICE) {
 977                 struct iscsi_datain_req *dr;
 978
 979                 dr = iscsit_allocate_datain_req();
 980                 if (!dr)
 981                         return iscsit_add_reject_cmd(cmd,
 982                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
 983
 984                 iscsit_attach_datain_req(cmd, dr);
 985         }
 986
 987         /*
 988          * Initialize struct se_cmd descriptor from target_core_mod infrastructure
 989          */
 990         transport_init_se_cmd(&cmd->se_cmd, &lio_target_fabric_configfs->tf_ops,
 991                         conn->sess->se_sess, be32_to_cpu(hdr->data_length),
 992                         cmd->data_direction, sam_task_attr,
 993                         cmd->sense_buffer + 2);
 994
 995         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 996                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
 997                 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
 998                 conn->cid);
 999
1000         target_get_sess_cmd(conn->sess->se_sess, &cmd->se_cmd, true);
1001
1002         cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
1003                                                      scsilun_to_int(&hdr->lun));
1004         if (cmd->sense_reason)
1005                 goto attach_cmd;
1006
1007         cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb);
1008         if (cmd->sense_reason) {
1009                 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
1010                         return iscsit_add_reject_cmd(cmd,
1011                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1012                 }
1013
1014                 goto attach_cmd;
1015         }
1016
1017         if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
1018                 return iscsit_add_reject_cmd(cmd,
1019                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1020         }
1021
1022 attach_cmd:
1023         spin_lock_bh(&conn->cmd_lock);
1024         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1025         spin_unlock_bh(&conn->cmd_lock);
1026         /*
1027          * Check if we need to delay processing because of ALUA
1028          * Active/NonOptimized primary access state..
1029          */
1030         core_alua_check_nonop_delay(&cmd->se_cmd);
1031
1032         return 0;
1033 }
1034 EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
1035
1036 void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd)
1037 {
1038         iscsit_set_dataout_sequence_values(cmd);
1039
1040         spin_lock_bh(&cmd->dataout_timeout_lock);
1041         iscsit_start_dataout_timer(cmd, cmd->conn);
1042         spin_unlock_bh(&cmd->dataout_timeout_lock);
1043 }
1044 EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout);
1045
1046 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1047                             struct iscsi_scsi_req *hdr)
1048 {
1049         int cmdsn_ret = 0;
1050         /*
1051          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1052          * the Immediate Bit is not set, and no Immediate
1053          * Data is attached.
1054          *
1055          * A PDU/CmdSN carrying Immediate Data can only
1056          * be processed after the DataCRC has passed.
1057          * If the DataCRC fails, the CmdSN MUST NOT
1058          * be acknowledged. (See below)
1059          */
1060         if (!cmd->immediate_data) {
1061                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1062                                         (unsigned char *)hdr, hdr->cmdsn);
1063                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1064                         return -1;
1065                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1066                         target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1067                         return 0;
1068                 }
1069         }
1070
1071         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1072
1073         /*
1074          * If no Immediate Data is attached, it's OK to return now.
1075          */
1076         if (!cmd->immediate_data) {
1077                 if (!cmd->sense_reason && cmd->unsolicited_data)
1078                         iscsit_set_unsoliticed_dataout(cmd);
1079                 if (!cmd->sense_reason)
1080                         return 0;
1081
1082                 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1083                 return 0;
1084         }
1085
1086         /*
1087          * Early CHECK_CONDITIONs with ImmediateData never make it to command
1088          * execution.  These exceptions are processed in CmdSN order using
1089          * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
1090          */
1091         if (cmd->sense_reason) {
1092                 if (cmd->reject_reason)
1093                         return 0;
1094
1095                 return 1;
1096         }
1097         /*
1098          * Call directly into transport_generic_new_cmd() to perform
1099          * the backend memory allocation.
1100          */
1101         cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
1102         if (cmd->sense_reason)
1103                 return 1;
1104
1105         return 0;
1106 }
1107 EXPORT_SYMBOL(iscsit_process_scsi_cmd);
1108
1109 static int
1110 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
1111                           bool dump_payload)
1112 {
1113         struct iscsi_conn *conn = cmd->conn;
1114         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1115         /*
1116          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
1117          */
1118         if (dump_payload == true)
1119                 goto after_immediate_data;
1120
1121         immed_ret = iscsit_handle_immediate_data(cmd, hdr,
1122                                         cmd->first_burst_len);
1123 after_immediate_data:
1124         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1125                 /*
1126                  * A PDU/CmdSN carrying Immediate Data passed
1127                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1128                  * Immediate Bit is not set.
1129                  */
1130                 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd,
1131                                         (unsigned char *)hdr, hdr->cmdsn);
1132                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1133                         return -1;
1134
1135                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1136                         int rc;
1137
1138                         rc = iscsit_dump_data_payload(cmd->conn,
1139                                                       cmd->first_burst_len, 1);
1140                         target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1141                         return rc;
1142                 } else if (cmd->unsolicited_data)
1143                         iscsit_set_unsoliticed_dataout(cmd);
1144
1145         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1146                 /*
1147                  * Immediate Data failed DataCRC and ERL>=1,
1148                  * silently drop this PDU and let the initiator
1149                  * plug the CmdSN gap.
1150                  *
1151                  * FIXME: Send Unsolicited NOPIN with reserved
1152                  * TTT here to help the initiator figure out
1153                  * the missing CmdSN, although they should be
1154                  * intelligent enough to determine the missing
1155                  * CmdSN and issue a retry to plug the sequence.
1156                  */
1157                 cmd->i_state = ISTATE_REMOVE;
1158                 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
1159         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1160                 return -1;
1161
1162         return 0;
1163 }
1164
1165 static int
1166 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1167                            unsigned char *buf)
1168 {
1169         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1170         int rc, immed_data;
1171         bool dump_payload = false;
1172
1173         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1174         if (rc < 0)
1175                 return 0;
1176         /*
1177          * Allocation iovecs needed for struct socket operations for
1178          * traditional iSCSI block I/O.
1179          */
1180         if (iscsit_allocate_iovecs(cmd) < 0) {
1181                 return iscsit_add_reject_cmd(cmd,
1182                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1183         }
1184         immed_data = cmd->immediate_data;
1185
1186         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1187         if (rc < 0)
1188                 return rc;
1189         else if (rc > 0)
1190                 dump_payload = true;
1191
1192         if (!immed_data)
1193                 return 0;
1194
1195         return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1196 }
1197
1198 static u32 iscsit_do_crypto_hash_sg(
1199         struct hash_desc *hash,
1200         struct iscsi_cmd *cmd,
1201         u32 data_offset,
1202         u32 data_length,
1203         u32 padding,
1204         u8 *pad_bytes)
1205 {
1206         u32 data_crc;
1207         u32 i;
1208         struct scatterlist *sg;
1209         unsigned int page_off;
1210
1211         crypto_hash_init(hash);
1212
1213         sg = cmd->first_data_sg;
1214         page_off = cmd->first_data_sg_off;
1215
1216         i = 0;
1217         while (data_length) {
1218                 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off));
1219
1220                 crypto_hash_update(hash, &sg[i], cur_len);
1221
1222                 data_length -= cur_len;
1223                 page_off = 0;
1224                 i++;
1225         }
1226
1227         if (padding) {
1228                 struct scatterlist pad_sg;
1229
1230                 sg_init_one(&pad_sg, pad_bytes, padding);
1231                 crypto_hash_update(hash, &pad_sg, padding);
1232         }
1233         crypto_hash_final(hash, (u8 *) &data_crc);
1234
1235         return data_crc;
1236 }
1237
1238 static void iscsit_do_crypto_hash_buf(
1239         struct hash_desc *hash,
1240         const void *buf,
1241         u32 payload_length,
1242         u32 padding,
1243         u8 *pad_bytes,
1244         u8 *data_crc)
1245 {
1246         struct scatterlist sg;
1247
1248         crypto_hash_init(hash);
1249
1250         sg_init_one(&sg, buf, payload_length);
1251         crypto_hash_update(hash, &sg, payload_length);
1252
1253         if (padding) {
1254                 sg_init_one(&sg, pad_bytes, padding);
1255                 crypto_hash_update(hash, &sg, padding);
1256         }
1257         crypto_hash_final(hash, data_crc);
1258 }
1259
1260 int
1261 iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf,
1262                           struct iscsi_cmd **out_cmd)
1263 {
1264         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1265         struct iscsi_cmd *cmd = NULL;
1266         struct se_cmd *se_cmd;
1267         unsigned long flags;
1268         u32 payload_length = ntoh24(hdr->dlength);
1269         int rc;
1270
1271         if (!payload_length) {
1272                 pr_err("DataOUT payload is ZERO, protocol error.\n");
1273                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1274                                          buf);
1275         }
1276
1277         /* iSCSI write */
1278         spin_lock_bh(&conn->sess->session_stats_lock);
1279         conn->sess->rx_data_octets += payload_length;
1280         if (conn->sess->se_sess->se_node_acl) {
1281                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
1282                 conn->sess->se_sess->se_node_acl->write_bytes += payload_length;
1283                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
1284         }
1285         spin_unlock_bh(&conn->sess->session_stats_lock);
1286
1287         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1288                 pr_err("DataSegmentLength: %u is greater than"
1289                         " MaxXmitDataSegmentLength: %u\n", payload_length,
1290                         conn->conn_ops->MaxXmitDataSegmentLength);
1291                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1292                                          buf);
1293         }
1294
1295         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
1296                         payload_length);
1297         if (!cmd)
1298                 return 0;
1299
1300         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1301                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1302                 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
1303                 payload_length, conn->cid);
1304
1305         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1306                 pr_err("Command ITT: 0x%08x received DataOUT after"
1307                         " last DataOUT received, dumping payload\n",
1308                         cmd->init_task_tag);
1309                 return iscsit_dump_data_payload(conn, payload_length, 1);
1310         }
1311
1312         if (cmd->data_direction != DMA_TO_DEVICE) {
1313                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1314                         " NON-WRITE command.\n", cmd->init_task_tag);
1315                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
1316         }
1317         se_cmd = &cmd->se_cmd;
1318         iscsit_mod_dataout_timer(cmd);
1319
1320         if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
1321                 pr_err("DataOut Offset: %u, Length %u greater than"
1322                         " iSCSI Command EDTL %u, protocol error.\n",
1323                         hdr->offset, payload_length, cmd->se_cmd.data_length);
1324                 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf);
1325         }
1326
1327         if (cmd->unsolicited_data) {
1328                 int dump_unsolicited_data = 0;
1329
1330                 if (conn->sess->sess_ops->InitialR2T) {
1331                         pr_err("Received unexpected unsolicited data"
1332                                 " while InitialR2T=Yes, protocol error.\n");
1333                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1334                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1335                         return -1;
1336                 }
1337                 /*
1338                  * Special case for dealing with Unsolicited DataOUT
1339                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1340                  * failures;
1341                  */
1342
1343                 /* Something's amiss if we're not in WRITE_PENDING state... */
1344                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1345                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1346                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1347
1348                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1349                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
1350                         dump_unsolicited_data = 1;
1351                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1352
1353                 if (dump_unsolicited_data) {
1354                         /*
1355                          * Check if a delayed TASK_ABORTED status needs to
1356                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1357                          * received with the unsolicitied data out.
1358                          */
1359                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1360                                 iscsit_stop_dataout_timer(cmd);
1361
1362                         transport_check_aborted_status(se_cmd,
1363                                         (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1364                         return iscsit_dump_data_payload(conn, payload_length, 1);
1365                 }
1366         } else {
1367                 /*
1368                  * For the normal solicited data path:
1369                  *
1370                  * Check for a delayed TASK_ABORTED status and dump any
1371                  * incoming data out payload if one exists.  Also, when the
1372                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1373                  * data out sequence, we decrement outstanding_r2ts.  Once
1374                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1375                  * TASK_ABORTED status.
1376                  */
1377                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1378                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1379                                 if (--cmd->outstanding_r2ts < 1) {
1380                                         iscsit_stop_dataout_timer(cmd);
1381                                         transport_check_aborted_status(
1382                                                         se_cmd, 1);
1383                                 }
1384
1385                         return iscsit_dump_data_payload(conn, payload_length, 1);
1386                 }
1387         }
1388         /*
1389          * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1390          * within-command recovery checks before receiving the payload.
1391          */
1392         rc = iscsit_check_pre_dataout(cmd, buf);
1393         if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
1394                 return 0;
1395         else if (rc == DATAOUT_CANNOT_RECOVER)
1396                 return -1;
1397
1398         *out_cmd = cmd;
1399         return 0;
1400 }
1401 EXPORT_SYMBOL(iscsit_check_dataout_hdr);
1402
1403 static int
1404 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1405                    struct iscsi_data *hdr)
1406 {
1407         struct kvec *iov;
1408         u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
1409         u32 payload_length = ntoh24(hdr->dlength);
1410         int iov_ret, data_crc_failed = 0;
1411
1412         rx_size += payload_length;
1413         iov = &cmd->iov_data[0];
1414
1415         iov_ret = iscsit_map_iovec(cmd, iov, be32_to_cpu(hdr->offset),
1416                                    payload_length);
1417         if (iov_ret < 0)
1418                 return -1;
1419
1420         iov_count += iov_ret;
1421
1422         padding = ((-payload_length) & 3);
1423         if (padding != 0) {
1424                 iov[iov_count].iov_base = cmd->pad_bytes;
1425                 iov[iov_count++].iov_len = padding;
1426                 rx_size += padding;
1427                 pr_debug("Receiving %u padding bytes.\n", padding);
1428         }
1429
1430         if (conn->conn_ops->DataDigest) {
1431                 iov[iov_count].iov_base = &checksum;
1432                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1433                 rx_size += ISCSI_CRC_LEN;
1434         }
1435
1436         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1437
1438         iscsit_unmap_iovec(cmd);
1439
1440         if (rx_got != rx_size)
1441                 return -1;
1442
1443         if (conn->conn_ops->DataDigest) {
1444                 u32 data_crc;
1445
1446                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
1447                                                     be32_to_cpu(hdr->offset),
1448                                                     payload_length, padding,
1449                                                     cmd->pad_bytes);
1450
1451                 if (checksum != data_crc) {
1452                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1453                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1454                                 " does not match computed 0x%08x\n",
1455                                 hdr->itt, hdr->offset, payload_length,
1456                                 hdr->datasn, checksum, data_crc);
1457                         data_crc_failed = 1;
1458                 } else {
1459                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1460                                 " %u bytes of Data Out\n", checksum,
1461                                 payload_length);
1462                 }
1463         }
1464
1465         return data_crc_failed;
1466 }
1467
1468 int
1469 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
1470                              bool data_crc_failed)
1471 {
1472         struct iscsi_conn *conn = cmd->conn;
1473         int rc, ooo_cmdsn;
1474         /*
1475          * Increment post receive data and CRC values or perform
1476          * within-command recovery.
1477          */
1478         rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
1479         if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
1480                 return 0;
1481         else if (rc == DATAOUT_SEND_R2T) {
1482                 iscsit_set_dataout_sequence_values(cmd);
1483                 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
1484         } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
1485                 /*
1486                  * Handle extra special case for out of order
1487                  * Unsolicited Data Out.
1488                  */
1489                 spin_lock_bh(&cmd->istate_lock);
1490                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1491                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1492                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1493                 spin_unlock_bh(&cmd->istate_lock);
1494
1495                 iscsit_stop_dataout_timer(cmd);
1496                 if (ooo_cmdsn)
1497                         return 0;
1498                 target_execute_cmd(&cmd->se_cmd);
1499                 return 0;
1500         } else /* DATAOUT_CANNOT_RECOVER */
1501                 return -1;
1502
1503         return 0;
1504 }
1505 EXPORT_SYMBOL(iscsit_check_dataout_payload);
1506
1507 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1508 {
1509         struct iscsi_cmd *cmd;
1510         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1511         int rc;
1512         bool data_crc_failed = false;
1513
1514         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1515         if (rc < 0)
1516                 return 0;
1517         else if (!cmd)
1518                 return 0;
1519
1520         rc = iscsit_get_dataout(conn, cmd, hdr);
1521         if (rc < 0)
1522                 return rc;
1523         else if (rc > 0)
1524                 data_crc_failed = true;
1525
1526         return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
1527 }
1528
1529 int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1530                         unsigned char *buf)
1531 {
1532         unsigned char *ping_data = NULL;
1533         int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
1534         u32 checksum, data_crc, padding = 0, payload_length;
1535         struct iscsi_cmd *cmd_p = NULL;
1536         struct kvec *iov = NULL;
1537         struct iscsi_nopout *hdr;
1538
1539         hdr                     = (struct iscsi_nopout *) buf;
1540         payload_length          = ntoh24(hdr->dlength);
1541
1542         if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1543                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1544                         " not set, protocol error.\n");
1545                 if (!cmd)
1546                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1547                                                  (unsigned char *)hdr);
1548
1549                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1550                                          (unsigned char *)hdr);
1551         }
1552
1553         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1554                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1555                         " greater than MaxXmitDataSegmentLength: %u, protocol"
1556                         " error.\n", payload_length,
1557                         conn->conn_ops->MaxXmitDataSegmentLength);
1558                 if (!cmd)
1559                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1560                                                  (unsigned char *)hdr);
1561
1562                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1563                                          (unsigned char *)hdr);
1564         }
1565
1566         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
1567                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1568                 hdr->itt == RESERVED_ITT ? "Response" : "Request",
1569                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1570                 payload_length);
1571         /*
1572          * This is not a response to a Unsolicited NopIN, which means
1573          * it can either be a NOPOUT ping request (with a valid ITT),
1574          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1575          * Either way, make sure we allocate an struct iscsi_cmd, as both
1576          * can contain ping data.
1577          */
1578         if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1579                 if (!cmd)
1580                         return iscsit_reject_cmd(cmd,
1581                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1582                                         (unsigned char *)hdr);
1583
1584                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1585                 cmd->i_state            = ISTATE_SEND_NOPIN;
1586                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1587                                                 1 : 0);
1588                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1589                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1590                 cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1591                 cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1592                 cmd->data_direction     = DMA_NONE;
1593         }
1594
1595         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1596                 rx_size = payload_length;
1597                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1598                 if (!ping_data) {
1599                         pr_err("Unable to allocate memory for"
1600                                 " NOPOUT ping data.\n");
1601                         ret = -1;
1602                         goto out;
1603                 }
1604
1605                 iov = &cmd->iov_misc[0];
1606                 iov[niov].iov_base      = ping_data;
1607                 iov[niov++].iov_len     = payload_length;
1608
1609                 padding = ((-payload_length) & 3);
1610                 if (padding != 0) {
1611                         pr_debug("Receiving %u additional bytes"
1612                                 " for padding.\n", padding);
1613                         iov[niov].iov_base      = &cmd->pad_bytes;
1614                         iov[niov++].iov_len     = padding;
1615                         rx_size += padding;
1616                 }
1617                 if (conn->conn_ops->DataDigest) {
1618                         iov[niov].iov_base      = &checksum;
1619                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1620                         rx_size += ISCSI_CRC_LEN;
1621                 }
1622
1623                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1624                 if (rx_got != rx_size) {
1625                         ret = -1;
1626                         goto out;
1627                 }
1628
1629                 if (conn->conn_ops->DataDigest) {
1630                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1631                                         ping_data, payload_length,
1632                                         padding, cmd->pad_bytes,
1633                                         (u8 *)&data_crc);
1634
1635                         if (checksum != data_crc) {
1636                                 pr_err("Ping data CRC32C DataDigest"
1637                                 " 0x%08x does not match computed 0x%08x\n",
1638                                         checksum, data_crc);
1639                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1640                                         pr_err("Unable to recover from"
1641                                         " NOPOUT Ping DataCRC failure while in"
1642                                                 " ERL=0.\n");
1643                                         ret = -1;
1644                                         goto out;
1645                                 } else {
1646                                         /*
1647                                          * Silently drop this PDU and let the
1648                                          * initiator plug the CmdSN gap.
1649                                          */
1650                                         pr_debug("Dropping NOPOUT"
1651                                         " Command CmdSN: 0x%08x due to"
1652                                         " DataCRC error.\n", hdr->cmdsn);
1653                                         ret = 0;
1654                                         goto out;
1655                                 }
1656                         } else {
1657                                 pr_debug("Got CRC32C DataDigest"
1658                                 " 0x%08x for %u bytes of ping data.\n",
1659                                         checksum, payload_length);
1660                         }
1661                 }
1662
1663                 ping_data[payload_length] = '\0';
1664                 /*
1665                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1666                  */
1667                 cmd->buf_ptr = ping_data;
1668                 cmd->buf_ptr_size = payload_length;
1669
1670                 pr_debug("Got %u bytes of NOPOUT ping"
1671                         " data.\n", payload_length);
1672                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1673         }
1674
1675         if (hdr->itt != RESERVED_ITT) {
1676                 if (!cmd) {
1677                         pr_err("Checking CmdSN for NOPOUT,"
1678                                 " but cmd is NULL!\n");
1679                         return -1;
1680                 }
1681                 /*
1682                  * Initiator is expecting a NopIN ping reply,
1683                  */
1684                 spin_lock_bh(&conn->cmd_lock);
1685                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1686                 spin_unlock_bh(&conn->cmd_lock);
1687
1688                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1689
1690                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1691                         iscsit_add_cmd_to_response_queue(cmd, conn,
1692                                         cmd->i_state);
1693                         return 0;
1694                 }
1695
1696                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1697                                 (unsigned char *)hdr, hdr->cmdsn);
1698                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1699                         ret = 0;
1700                         goto ping_out;
1701                 }
1702                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1703                         return -1;
1704
1705                 return 0;
1706         }
1707
1708         if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1709                 /*
1710                  * This was a response to a unsolicited NOPIN ping.
1711                  */
1712                 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
1713                 if (!cmd_p)
1714                         return -1;
1715
1716                 iscsit_stop_nopin_response_timer(conn);
1717
1718                 cmd_p->i_state = ISTATE_REMOVE;
1719                 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
1720                 iscsit_start_nopin_timer(conn);
1721         } else {
1722                 /*
1723                  * Initiator is not expecting a NOPIN is response.
1724                  * Just ignore for now.
1725                  *
1726                  * iSCSI v19-91 10.18
1727                  * "A NOP-OUT may also be used to confirm a changed
1728                  *  ExpStatSN if another PDU will not be available
1729                  *  for a long time."
1730                  */
1731                 ret = 0;
1732                 goto out;
1733         }
1734
1735         return 0;
1736 out:
1737         if (cmd)
1738                 iscsit_free_cmd(cmd, false);
1739 ping_out:
1740         kfree(ping_data);
1741         return ret;
1742 }
1743 EXPORT_SYMBOL(iscsit_handle_nop_out);
1744
1745 int
1746 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1747                            unsigned char *buf)
1748 {
1749         struct se_tmr_req *se_tmr;
1750         struct iscsi_tmr_req *tmr_req;
1751         struct iscsi_tm *hdr;
1752         int out_of_order_cmdsn = 0, ret;
1753         bool sess_ref = false;
1754         u8 function;
1755
1756         hdr                     = (struct iscsi_tm *) buf;
1757         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1758         function = hdr->flags;
1759
1760         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1761                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1762                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1763                 hdr->rtt, hdr->refcmdsn, conn->cid);
1764
1765         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1766             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1767              hdr->rtt != RESERVED_ITT)) {
1768                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1769                 hdr->rtt = RESERVED_ITT;
1770         }
1771
1772         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1773                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1774                 pr_err("Task Management Request TASK_REASSIGN not"
1775                         " issued as immediate command, bad iSCSI Initiator"
1776                                 "implementation\n");
1777                 return iscsit_add_reject_cmd(cmd,
1778                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1779         }
1780         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1781             be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
1782                 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
1783
1784         cmd->data_direction = DMA_NONE;
1785
1786         cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL);
1787         if (!cmd->tmr_req) {
1788                 pr_err("Unable to allocate memory for"
1789                         " Task Management command!\n");
1790                 return iscsit_add_reject_cmd(cmd,
1791                                              ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1792                                              buf);
1793         }
1794
1795         /*
1796          * TASK_REASSIGN for ERL=2 / connection stays inside of
1797          * LIO-Target $FABRIC_MOD
1798          */
1799         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1800
1801                 u8 tcm_function;
1802                 int ret;
1803
1804                 transport_init_se_cmd(&cmd->se_cmd,
1805                                       &lio_target_fabric_configfs->tf_ops,
1806                                       conn->sess->se_sess, 0, DMA_NONE,
1807                                       MSG_SIMPLE_TAG, cmd->sense_buffer + 2);
1808
1809                 target_get_sess_cmd(conn->sess->se_sess, &cmd->se_cmd, true);
1810                 sess_ref = true;
1811
1812                 switch (function) {
1813                 case ISCSI_TM_FUNC_ABORT_TASK:
1814                         tcm_function = TMR_ABORT_TASK;
1815                         break;
1816                 case ISCSI_TM_FUNC_ABORT_TASK_SET:
1817                         tcm_function = TMR_ABORT_TASK_SET;
1818                         break;
1819                 case ISCSI_TM_FUNC_CLEAR_ACA:
1820                         tcm_function = TMR_CLEAR_ACA;
1821                         break;
1822                 case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1823                         tcm_function = TMR_CLEAR_TASK_SET;
1824                         break;
1825                 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1826                         tcm_function = TMR_LUN_RESET;
1827                         break;
1828                 case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1829                         tcm_function = TMR_TARGET_WARM_RESET;
1830                         break;
1831                 case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1832                         tcm_function = TMR_TARGET_COLD_RESET;
1833                         break;
1834                 default:
1835                         pr_err("Unknown iSCSI TMR Function:"
1836                                " 0x%02x\n", function);
1837                         return iscsit_add_reject_cmd(cmd,
1838                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1839                 }
1840
1841                 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req,
1842                                          tcm_function, GFP_KERNEL);
1843                 if (ret < 0)
1844                         return iscsit_add_reject_cmd(cmd,
1845                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1846
1847                 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
1848         }
1849
1850         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
1851         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
1852         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1853         cmd->init_task_tag      = hdr->itt;
1854         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1855         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1856         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1857         se_tmr                  = cmd->se_cmd.se_tmr_req;
1858         tmr_req                 = cmd->tmr_req;
1859         /*
1860          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
1861          */
1862         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1863                 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
1864                                                scsilun_to_int(&hdr->lun));
1865                 if (ret < 0) {
1866                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
1867                         goto attach;
1868                 }
1869         }
1870
1871         switch (function) {
1872         case ISCSI_TM_FUNC_ABORT_TASK:
1873                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
1874                 if (se_tmr->response)
1875                         goto attach;
1876                 break;
1877         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1878         case ISCSI_TM_FUNC_CLEAR_ACA:
1879         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1880         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1881                 break;
1882         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1883                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
1884                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1885                         goto attach;
1886                 }
1887                 break;
1888         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1889                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
1890                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1891                         goto attach;
1892                 }
1893                 break;
1894         case ISCSI_TM_FUNC_TASK_REASSIGN:
1895                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
1896                 /*
1897                  * Perform sanity checks on the ExpDataSN only if the
1898                  * TASK_REASSIGN was successful.
1899                  */
1900                 if (se_tmr->response)
1901                         break;
1902
1903                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
1904                         return iscsit_add_reject_cmd(cmd,
1905                                         ISCSI_REASON_BOOKMARK_INVALID, buf);
1906                 break;
1907         default:
1908                 pr_err("Unknown TMR function: 0x%02x, protocol"
1909                         " error.\n", function);
1910                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
1911                 goto attach;
1912         }
1913
1914         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1915             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
1916                 se_tmr->call_transport = 1;
1917 attach:
1918         spin_lock_bh(&conn->cmd_lock);
1919         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1920         spin_unlock_bh(&conn->cmd_lock);
1921
1922         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1923                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1924                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
1925                         out_of_order_cmdsn = 1;
1926                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1927                         return 0;
1928                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1929                         return -1;
1930         }
1931         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1932
1933         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
1934                 return 0;
1935         /*
1936          * Found the referenced task, send to transport for processing.
1937          */
1938         if (se_tmr->call_transport)
1939                 return transport_generic_handle_tmr(&cmd->se_cmd);
1940
1941         /*
1942          * Could not find the referenced LUN, task, or Task Management
1943          * command not authorized or supported.  Change state and
1944          * let the tx_thread send the response.
1945          *
1946          * For connection recovery, this is also the default action for
1947          * TMR TASK_REASSIGN.
1948          */
1949         if (sess_ref) {
1950                 pr_debug("Handle TMR, using sess_ref=true check\n");
1951                 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1952         }
1953
1954         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
1955         return 0;
1956 }
1957 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
1958
1959 /* #warning FIXME: Support Text Command parameters besides SendTargets */
1960 static int iscsit_handle_text_cmd(
1961         struct iscsi_conn *conn,
1962         unsigned char *buf)
1963 {
1964         char *text_ptr, *text_in;
1965         int cmdsn_ret, niov = 0, rx_got, rx_size;
1966         u32 checksum = 0, data_crc = 0, payload_length;
1967         u32 padding = 0, pad_bytes = 0, text_length = 0;
1968         struct iscsi_cmd *cmd;
1969         struct kvec iov[3];
1970         struct iscsi_text *hdr;
1971
1972         hdr                     = (struct iscsi_text *) buf;
1973         payload_length          = ntoh24(hdr->dlength);
1974
1975         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1976                 pr_err("Unable to accept text parameter length: %u"
1977                         "greater than MaxXmitDataSegmentLength %u.\n",
1978                        payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
1979                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
1980         }
1981
1982         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
1983                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
1984                 hdr->exp_statsn, payload_length);
1985
1986         rx_size = text_length = payload_length;
1987         if (text_length) {
1988                 text_in = kzalloc(text_length, GFP_KERNEL);
1989                 if (!text_in) {
1990                         pr_err("Unable to allocate memory for"
1991                                 " incoming text parameters\n");
1992                         return -1;
1993                 }
1994
1995                 memset(iov, 0, 3 * sizeof(struct kvec));
1996                 iov[niov].iov_base      = text_in;
1997                 iov[niov++].iov_len     = text_length;
1998
1999                 padding = ((-payload_length) & 3);
2000                 if (padding != 0) {
2001                         iov[niov].iov_base = &pad_bytes;
2002                         iov[niov++].iov_len  = padding;
2003                         rx_size += padding;
2004                         pr_debug("Receiving %u additional bytes"
2005                                         " for padding.\n", padding);
2006                 }
2007                 if (conn->conn_ops->DataDigest) {
2008                         iov[niov].iov_base      = &checksum;
2009                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
2010                         rx_size += ISCSI_CRC_LEN;
2011                 }
2012
2013                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
2014                 if (rx_got != rx_size) {
2015                         kfree(text_in);
2016                         return -1;
2017                 }
2018
2019                 if (conn->conn_ops->DataDigest) {
2020                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
2021                                         text_in, text_length,
2022                                         padding, (u8 *)&pad_bytes,
2023                                         (u8 *)&data_crc);
2024
2025                         if (checksum != data_crc) {
2026                                 pr_err("Text data CRC32C DataDigest"
2027                                         " 0x%08x does not match computed"
2028                                         " 0x%08x\n", checksum, data_crc);
2029                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2030                                         pr_err("Unable to recover from"
2031                                         " Text Data digest failure while in"
2032                                                 " ERL=0.\n");
2033                                         kfree(text_in);
2034                                         return -1;
2035                                 } else {
2036                                         /*
2037                                          * Silently drop this PDU and let the
2038                                          * initiator plug the CmdSN gap.
2039                                          */
2040                                         pr_debug("Dropping Text"
2041                                         " Command CmdSN: 0x%08x due to"
2042                                         " DataCRC error.\n", hdr->cmdsn);
2043                                         kfree(text_in);
2044                                         return 0;
2045                                 }
2046                         } else {
2047                                 pr_debug("Got CRC32C DataDigest"
2048                                         " 0x%08x for %u bytes of text data.\n",
2049                                                 checksum, text_length);
2050                         }
2051                 }
2052                 text_in[text_length - 1] = '\0';
2053                 pr_debug("Successfully read %d bytes of text"
2054                                 " data.\n", text_length);
2055
2056                 if (strncmp("SendTargets", text_in, 11) != 0) {
2057                         pr_err("Received Text Data that is not"
2058                                 " SendTargets, cannot continue.\n");
2059                         kfree(text_in);
2060                         return -1;
2061                 }
2062                 text_ptr = strchr(text_in, '=');
2063                 if (!text_ptr) {
2064                         pr_err("No \"=\" separator found in Text Data,"
2065                                 "  cannot continue.\n");
2066                         kfree(text_in);
2067                         return -1;
2068                 }
2069                 if (strncmp("=All", text_ptr, 4) != 0) {
2070                         pr_err("Unable to locate All value for"
2071                                 " SendTargets key,  cannot continue.\n");
2072                         kfree(text_in);
2073                         return -1;
2074                 }
2075 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */
2076                 kfree(text_in);
2077         }
2078
2079         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
2080         if (!cmd)
2081                 return iscsit_add_reject(conn,
2082                                          ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2083
2084         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
2085         cmd->i_state            = ISTATE_SEND_TEXTRSP;
2086         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2087         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2088         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2089         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2090         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2091         cmd->data_direction     = DMA_NONE;
2092
2093         spin_lock_bh(&conn->cmd_lock);
2094         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2095         spin_unlock_bh(&conn->cmd_lock);
2096
2097         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2098
2099         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2100                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
2101                                 (unsigned char *)hdr, hdr->cmdsn);
2102                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2103                         return -1;
2104
2105                 return 0;
2106         }
2107
2108         return iscsit_execute_cmd(cmd, 0);
2109 }
2110
2111 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2112 {
2113         struct iscsi_conn *conn_p;
2114         struct iscsi_session *sess = conn->sess;
2115
2116         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2117                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2118
2119         atomic_set(&sess->session_logout, 1);
2120         atomic_set(&conn->conn_logout_remove, 1);
2121         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2122
2123         iscsit_inc_conn_usage_count(conn);
2124         iscsit_inc_session_usage_count(sess);
2125
2126         spin_lock_bh(&sess->conn_lock);
2127         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2128                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2129                         continue;
2130
2131                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2132                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2133         }
2134         spin_unlock_bh(&sess->conn_lock);
2135
2136         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2137
2138         return 0;
2139 }
2140
2141 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2142 {
2143         struct iscsi_conn *l_conn;
2144         struct iscsi_session *sess = conn->sess;
2145
2146         pr_debug("Received logout request CLOSECONNECTION for CID:"
2147                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2148
2149         /*
2150          * A Logout Request with a CLOSECONNECTION reason code for a CID
2151          * can arrive on a connection with a differing CID.
2152          */
2153         if (conn->cid == cmd->logout_cid) {
2154                 spin_lock_bh(&conn->state_lock);
2155                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2156                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2157
2158                 atomic_set(&conn->conn_logout_remove, 1);
2159                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2160                 iscsit_inc_conn_usage_count(conn);
2161
2162                 spin_unlock_bh(&conn->state_lock);
2163         } else {
2164                 /*
2165                  * Handle all different cid CLOSECONNECTION requests in
2166                  * iscsit_logout_post_handler_diffcid() as to give enough
2167                  * time for any non immediate command's CmdSN to be
2168                  * acknowledged on the connection in question.
2169                  *
2170                  * Here we simply make sure the CID is still around.
2171                  */
2172                 l_conn = iscsit_get_conn_from_cid(sess,
2173                                 cmd->logout_cid);
2174                 if (!l_conn) {
2175                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2176                         iscsit_add_cmd_to_response_queue(cmd, conn,
2177                                         cmd->i_state);
2178                         return 0;
2179                 }
2180
2181                 iscsit_dec_conn_usage_count(l_conn);
2182         }
2183
2184         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2185
2186         return 0;
2187 }
2188
2189 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2190 {
2191         struct iscsi_session *sess = conn->sess;
2192
2193         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2194                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2195
2196         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2197                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2198                         " while ERL!=2.\n");
2199                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2200                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2201                 return 0;
2202         }
2203
2204         if (conn->cid == cmd->logout_cid) {
2205                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2206                         " with CID: %hu on CID: %hu, implementation error.\n",
2207                                 cmd->logout_cid, conn->cid);
2208                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2209                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2210                 return 0;
2211         }
2212
2213         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2214
2215         return 0;
2216 }
2217
2218 int
2219 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2220                         unsigned char *buf)
2221 {
2222         int cmdsn_ret, logout_remove = 0;
2223         u8 reason_code = 0;
2224         struct iscsi_logout *hdr;
2225         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2226
2227         hdr                     = (struct iscsi_logout *) buf;
2228         reason_code             = (hdr->flags & 0x7f);
2229
2230         if (tiqn) {
2231                 spin_lock(&tiqn->logout_stats.lock);
2232                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2233                         tiqn->logout_stats.normal_logouts++;
2234                 else
2235                         tiqn->logout_stats.abnormal_logouts++;
2236                 spin_unlock(&tiqn->logout_stats.lock);
2237         }
2238
2239         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2240                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2241                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2242                 hdr->cid, conn->cid);
2243
2244         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2245                 pr_err("Received logout request on connection that"
2246                         " is not in logged in state, ignoring request.\n");
2247                 iscsit_free_cmd(cmd, false);
2248                 return 0;
2249         }
2250
2251         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2252         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2253         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2254         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2255         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2256         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2257         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2258         cmd->logout_cid         = be16_to_cpu(hdr->cid);
2259         cmd->logout_reason      = reason_code;
2260         cmd->data_direction     = DMA_NONE;
2261
2262         /*
2263          * We need to sleep in these cases (by returning 1) until the Logout
2264          * Response gets sent in the tx thread.
2265          */
2266         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2267            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2268             be16_to_cpu(hdr->cid) == conn->cid))
2269                 logout_remove = 1;
2270
2271         spin_lock_bh(&conn->cmd_lock);
2272         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2273         spin_unlock_bh(&conn->cmd_lock);
2274
2275         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2276                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2277
2278         /*
2279          * Immediate commands are executed, well, immediately.
2280          * Non-Immediate Logout Commands are executed in CmdSN order.
2281          */
2282         if (cmd->immediate_cmd) {
2283                 int ret = iscsit_execute_cmd(cmd, 0);
2284
2285                 if (ret < 0)
2286                         return ret;
2287         } else {
2288                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2289                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
2290                         logout_remove = 0;
2291                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2292                         return -1;
2293         }
2294
2295         return logout_remove;
2296 }
2297 EXPORT_SYMBOL(iscsit_handle_logout_cmd);
2298
2299 static int iscsit_handle_snack(
2300         struct iscsi_conn *conn,
2301         unsigned char *buf)
2302 {
2303         struct iscsi_snack *hdr;
2304
2305         hdr                     = (struct iscsi_snack *) buf;
2306         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2307
2308         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2309                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2310                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2311                         hdr->begrun, hdr->runlength, conn->cid);
2312
2313         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2314                 pr_err("Initiator sent SNACK request while in"
2315                         " ErrorRecoveryLevel=0.\n");
2316                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2317                                          buf);
2318         }
2319         /*
2320          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2321          * call from inside iscsi_send_recovery_datain_or_r2t().
2322          */
2323         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2324         case 0:
2325                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2326                         hdr->itt,
2327                         be32_to_cpu(hdr->ttt),
2328                         be32_to_cpu(hdr->begrun),
2329                         be32_to_cpu(hdr->runlength));
2330         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2331                 return iscsit_handle_status_snack(conn, hdr->itt,
2332                         be32_to_cpu(hdr->ttt),
2333                         be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
2334         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2335                 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
2336                         be32_to_cpu(hdr->begrun),
2337                         be32_to_cpu(hdr->runlength));
2338         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2339                 /* FIXME: Support R-Data SNACK */
2340                 pr_err("R-Data SNACK Not Supported.\n");
2341                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2342                                          buf);
2343         default:
2344                 pr_err("Unknown SNACK type 0x%02x, protocol"
2345                         " error.\n", hdr->flags & 0x0f);
2346                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2347                                          buf);
2348         }
2349
2350         return 0;
2351 }
2352
2353 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2354 {
2355         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2356             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2357                 wait_for_completion_interruptible_timeout(
2358                                         &conn->rx_half_close_comp,
2359                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2360         }
2361 }
2362
2363 static int iscsit_handle_immediate_data(
2364         struct iscsi_cmd *cmd,
2365         struct iscsi_scsi_req *hdr,
2366         u32 length)
2367 {
2368         int iov_ret, rx_got = 0, rx_size = 0;
2369         u32 checksum, iov_count = 0, padding = 0;
2370         struct iscsi_conn *conn = cmd->conn;
2371         struct kvec *iov;
2372
2373         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2374         if (iov_ret < 0)
2375                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2376
2377         rx_size = length;
2378         iov_count = iov_ret;
2379         iov = &cmd->iov_data[0];
2380
2381         padding = ((-length) & 3);
2382         if (padding != 0) {
2383                 iov[iov_count].iov_base = cmd->pad_bytes;
2384                 iov[iov_count++].iov_len = padding;
2385                 rx_size += padding;
2386         }
2387
2388         if (conn->conn_ops->DataDigest) {
2389                 iov[iov_count].iov_base         = &checksum;
2390                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2391                 rx_size += ISCSI_CRC_LEN;
2392         }
2393
2394         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2395
2396         iscsit_unmap_iovec(cmd);
2397
2398         if (rx_got != rx_size) {
2399                 iscsit_rx_thread_wait_for_tcp(conn);
2400                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2401         }
2402
2403         if (conn->conn_ops->DataDigest) {
2404                 u32 data_crc;
2405
2406                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
2407                                                     cmd->write_data_done, length, padding,
2408                                                     cmd->pad_bytes);
2409
2410                 if (checksum != data_crc) {
2411                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2412                                 " does not match computed 0x%08x\n", checksum,
2413                                 data_crc);
2414
2415                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2416                                 pr_err("Unable to recover from"
2417                                         " Immediate Data digest failure while"
2418                                         " in ERL=0.\n");
2419                                 iscsit_reject_cmd(cmd,
2420                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2421                                                 (unsigned char *)hdr);
2422                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2423                         } else {
2424                                 iscsit_reject_cmd(cmd,
2425                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2426                                                 (unsigned char *)hdr);
2427                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2428                         }
2429                 } else {
2430                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2431                                 " %u bytes of Immediate Data\n", checksum,
2432                                 length);
2433                 }
2434         }
2435
2436         cmd->write_data_done += length;
2437
2438         if (cmd->write_data_done == cmd->se_cmd.data_length) {
2439                 spin_lock_bh(&cmd->istate_lock);
2440                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2441                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2442                 spin_unlock_bh(&cmd->istate_lock);
2443         }
2444
2445         return IMMEDIATE_DATA_NORMAL_OPERATION;
2446 }
2447
2448 /*
2449  *      Called with sess->conn_lock held.
2450  */
2451 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2452         with active network interface */
2453 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2454 {
2455         struct iscsi_cmd *cmd;
2456         struct iscsi_conn *conn_p;
2457         bool found = false;
2458
2459         /*
2460          * Only send a Asynchronous Message on connections whos network
2461          * interface is still functional.
2462          */
2463         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2464                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2465                         iscsit_inc_conn_usage_count(conn_p);
2466                         found = true;
2467                         break;
2468                 }
2469         }
2470
2471         if (!found)
2472                 return;
2473
2474         cmd = iscsit_allocate_cmd(conn_p, GFP_ATOMIC);
2475         if (!cmd) {
2476                 iscsit_dec_conn_usage_count(conn_p);
2477                 return;
2478         }
2479
2480         cmd->logout_cid = conn->cid;
2481         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2482         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2483
2484         spin_lock_bh(&conn_p->cmd_lock);
2485         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2486         spin_unlock_bh(&conn_p->cmd_lock);
2487
2488         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2489         iscsit_dec_conn_usage_count(conn_p);
2490 }
2491
2492 static int iscsit_send_conn_drop_async_message(
2493         struct iscsi_cmd *cmd,
2494         struct iscsi_conn *conn)
2495 {
2496         struct iscsi_async *hdr;
2497
2498         cmd->tx_size = ISCSI_HDR_LEN;
2499         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2500
2501         hdr                     = (struct iscsi_async *) cmd->pdu;
2502         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2503         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2504         cmd->init_task_tag      = RESERVED_ITT;
2505         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2506         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2507         cmd->stat_sn            = conn->stat_sn++;
2508         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2509         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2510         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2511         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2512         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2513         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2514         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2515
2516         if (conn->conn_ops->HeaderDigest) {
2517                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2518
2519                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
2520                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
2521
2522                 cmd->tx_size += ISCSI_CRC_LEN;
2523                 pr_debug("Attaching CRC32C HeaderDigest to"
2524                         " Async Message 0x%08x\n", *header_digest);
2525         }
2526
2527         cmd->iov_misc[0].iov_base       = cmd->pdu;
2528         cmd->iov_misc[0].iov_len        = cmd->tx_size;
2529         cmd->iov_misc_count             = 1;
2530
2531         pr_debug("Sending Connection Dropped Async Message StatSN:"
2532                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2533                         cmd->logout_cid, conn->cid);
2534         return 0;
2535 }
2536
2537 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2538 {
2539         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2540             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2541                 wait_for_completion_interruptible_timeout(
2542                                         &conn->tx_half_close_comp,
2543                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2544         }
2545 }
2546
2547 static void
2548 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2549                         struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
2550                         bool set_statsn)
2551 {
2552         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2553         hdr->flags              = datain->flags;
2554         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2555                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2556                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2557                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2558                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2559                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2560                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2561                 }
2562         }
2563         hton24(hdr->dlength, datain->length);
2564         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2565                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2566                                 (struct scsi_lun *)&hdr->lun);
2567         else
2568                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2569
2570         hdr->itt                = cmd->init_task_tag;
2571
2572         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2573                 hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2574         else
2575                 hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
2576         if (set_statsn)
2577                 hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2578         else
2579                 hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
2580
2581         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2582         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2583         hdr->datasn             = cpu_to_be32(datain->data_sn);
2584         hdr->offset             = cpu_to_be32(datain->offset);
2585
2586         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2587                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2588                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2589                 ntohl(hdr->offset), datain->length, conn->cid);
2590 }
2591
2592 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2593 {
2594         struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
2595         struct iscsi_datain datain;
2596         struct iscsi_datain_req *dr;
2597         struct kvec *iov;
2598         u32 iov_count = 0, tx_size = 0;
2599         int eodr = 0, ret, iov_ret;
2600         bool set_statsn = false;
2601
2602         memset(&datain, 0, sizeof(struct iscsi_datain));
2603         dr = iscsit_get_datain_values(cmd, &datain);
2604         if (!dr) {
2605                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2606                                 cmd->init_task_tag);
2607                 return -1;
2608         }
2609         /*
2610          * Be paranoid and double check the logic for now.
2611          */
2612         if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
2613                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2614                         " datain.length: %u exceeds cmd->data_length: %u\n",
2615                         cmd->init_task_tag, datain.offset, datain.length,
2616                         cmd->se_cmd.data_length);
2617                 return -1;
2618         }
2619
2620         spin_lock_bh(&conn->sess->session_stats_lock);
2621         conn->sess->tx_data_octets += datain.length;
2622         if (conn->sess->se_sess->se_node_acl) {
2623                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
2624                 conn->sess->se_sess->se_node_acl->read_bytes += datain.length;
2625                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
2626         }
2627         spin_unlock_bh(&conn->sess->session_stats_lock);
2628         /*
2629          * Special case for successfully execution w/ both DATAIN
2630          * and Sense Data.
2631          */
2632         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2633             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2634                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2635         else {
2636                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2637                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2638                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2639                         cmd->stat_sn = conn->stat_sn++;
2640                         set_statsn = true;
2641                 } else if (dr->dr_complete ==
2642                            DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2643                         set_statsn = true;
2644         }
2645
2646         iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
2647
2648         iov = &cmd->iov_data[0];
2649         iov[iov_count].iov_base = cmd->pdu;
2650         iov[iov_count++].iov_len        = ISCSI_HDR_LEN;
2651         tx_size += ISCSI_HDR_LEN;
2652
2653         if (conn->conn_ops->HeaderDigest) {
2654                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2655
2656                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, cmd->pdu,
2657                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
2658
2659                 iov[0].iov_len += ISCSI_CRC_LEN;
2660                 tx_size += ISCSI_CRC_LEN;
2661
2662                 pr_debug("Attaching CRC32 HeaderDigest"
2663                         " for DataIN PDU 0x%08x\n", *header_digest);
2664         }
2665
2666         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1],
2667                                 datain.offset, datain.length);
2668         if (iov_ret < 0)
2669                 return -1;
2670
2671         iov_count += iov_ret;
2672         tx_size += datain.length;
2673
2674         cmd->padding = ((-datain.length) & 3);
2675         if (cmd->padding) {
2676                 iov[iov_count].iov_base         = cmd->pad_bytes;
2677                 iov[iov_count++].iov_len        = cmd->padding;
2678                 tx_size += cmd->padding;
2679
2680                 pr_debug("Attaching %u padding bytes\n",
2681                                 cmd->padding);
2682         }
2683         if (conn->conn_ops->DataDigest) {
2684                 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd,
2685                          datain.offset, datain.length, cmd->padding, cmd->pad_bytes);
2686
2687                 iov[iov_count].iov_base = &cmd->data_crc;
2688                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2689                 tx_size += ISCSI_CRC_LEN;
2690
2691                 pr_debug("Attached CRC32C DataDigest %d bytes, crc"
2692                         " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc);
2693         }
2694
2695         cmd->iov_data_count = iov_count;
2696         cmd->tx_size = tx_size;
2697
2698         /* sendpage is preferred but can't insert markers */
2699         if (!conn->conn_ops->IFMarker)
2700                 ret = iscsit_fe_sendpage_sg(cmd, conn);
2701         else
2702                 ret = iscsit_send_tx_data(cmd, conn, 0);
2703
2704         iscsit_unmap_iovec(cmd);
2705
2706         if (ret < 0) {
2707                 iscsit_tx_thread_wait_for_tcp(conn);
2708                 return ret;
2709         }
2710
2711         if (dr->dr_complete) {
2712                 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2713                                 2 : 1;
2714                 iscsit_free_datain_req(cmd, dr);
2715         }
2716
2717         return eodr;
2718 }
2719
2720 int
2721 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2722                         struct iscsi_logout_rsp *hdr)
2723 {
2724         struct iscsi_conn *logout_conn = NULL;
2725         struct iscsi_conn_recovery *cr = NULL;
2726         struct iscsi_session *sess = conn->sess;
2727         /*
2728          * The actual shutting down of Sessions and/or Connections
2729          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2730          * is done in scsi_logout_post_handler().
2731          */
2732         switch (cmd->logout_reason) {
2733         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2734                 pr_debug("iSCSI session logout successful, setting"
2735                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2736                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2737                 break;
2738         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2739                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2740                         break;
2741                 /*
2742                  * For CLOSECONNECTION logout requests carrying
2743                  * a matching logout CID -> local CID, the reference
2744                  * for the local CID will have been incremented in
2745                  * iscsi_logout_closeconnection().
2746                  *
2747                  * For CLOSECONNECTION logout requests carrying
2748                  * a different CID than the connection it arrived
2749                  * on, the connection responding to cmd->logout_cid
2750                  * is stopped in iscsit_logout_post_handler_diffcid().
2751                  */
2752
2753                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2754                         " successful.\n", cmd->logout_cid, conn->cid);
2755                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2756                 break;
2757         case ISCSI_LOGOUT_REASON_RECOVERY:
2758                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2759                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2760                         break;
2761                 /*
2762                  * If the connection is still active from our point of view
2763                  * force connection recovery to occur.
2764                  */
2765                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2766                                 cmd->logout_cid);
2767                 if (logout_conn) {
2768                         iscsit_connection_reinstatement_rcfr(logout_conn);
2769                         iscsit_dec_conn_usage_count(logout_conn);
2770                 }
2771
2772                 cr = iscsit_get_inactive_connection_recovery_entry(
2773                                 conn->sess, cmd->logout_cid);
2774                 if (!cr) {
2775                         pr_err("Unable to locate CID: %hu for"
2776                         " REMOVECONNFORRECOVERY Logout Request.\n",
2777                                 cmd->logout_cid);
2778                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2779                         break;
2780                 }
2781
2782                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2783
2784                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2785                         " for recovery for CID: %hu on CID: %hu successful.\n",
2786                                 cmd->logout_cid, conn->cid);
2787                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2788                 break;
2789         default:
2790                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2791                                 cmd->logout_reason);
2792                 return -1;
2793         }
2794
2795         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2796         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2797         hdr->response           = cmd->logout_response;
2798         hdr->itt                = cmd->init_task_tag;
2799         cmd->stat_sn            = conn->stat_sn++;
2800         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2801
2802         iscsit_increment_maxcmdsn(cmd, conn->sess);
2803         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2804         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2805
2806         pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
2807                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2808                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2809                 cmd->logout_cid, conn->cid);
2810
2811         return 0;
2812 }
2813 EXPORT_SYMBOL(iscsit_build_logout_rsp);
2814
2815 static int
2816 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2817 {
2818         struct kvec *iov;
2819         int niov = 0, tx_size, rc;
2820
2821         rc = iscsit_build_logout_rsp(cmd, conn,
2822                         (struct iscsi_logout_rsp *)&cmd->pdu[0]);
2823         if (rc < 0)
2824                 return rc;
2825
2826         tx_size = ISCSI_HDR_LEN;
2827         iov = &cmd->iov_misc[0];
2828         iov[niov].iov_base      = cmd->pdu;
2829         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2830
2831         if (conn->conn_ops->HeaderDigest) {
2832                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2833
2834                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, &cmd->pdu[0],
2835                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
2836
2837                 iov[0].iov_len += ISCSI_CRC_LEN;
2838                 tx_size += ISCSI_CRC_LEN;
2839                 pr_debug("Attaching CRC32C HeaderDigest to"
2840                         " Logout Response 0x%08x\n", *header_digest);
2841         }
2842         cmd->iov_misc_count = niov;
2843         cmd->tx_size = tx_size;
2844
2845         return 0;
2846 }
2847
2848 void
2849 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2850                        struct iscsi_nopin *hdr, bool nopout_response)
2851 {
2852         hdr->opcode             = ISCSI_OP_NOOP_IN;
2853         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2854         hton24(hdr->dlength, cmd->buf_ptr_size);
2855         if (nopout_response)
2856                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2857         hdr->itt                = cmd->init_task_tag;
2858         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2859         cmd->stat_sn            = (nopout_response) ? conn->stat_sn++ :
2860                                   conn->stat_sn;
2861         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2862
2863         if (nopout_response)
2864                 iscsit_increment_maxcmdsn(cmd, conn->sess);
2865
2866         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2867         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2868
2869         pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
2870                 " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
2871                 "Solicitied" : "Unsolicitied", cmd->init_task_tag,
2872                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
2873 }
2874 EXPORT_SYMBOL(iscsit_build_nopin_rsp);
2875
2876 /*
2877  *      Unsolicited NOPIN, either requesting a response or not.
2878  */
2879 static int iscsit_send_unsolicited_nopin(
2880         struct iscsi_cmd *cmd,
2881         struct iscsi_conn *conn,
2882         int want_response)
2883 {
2884         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
2885         int tx_size = ISCSI_HDR_LEN, ret;
2886
2887         iscsit_build_nopin_rsp(cmd, conn, hdr, false);
2888
2889         if (conn->conn_ops->HeaderDigest) {
2890                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2891
2892                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
2893                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
2894
2895                 tx_size += ISCSI_CRC_LEN;
2896                 pr_debug("Attaching CRC32C HeaderDigest to"
2897                         " NopIN 0x%08x\n", *header_digest);
2898         }
2899
2900         cmd->iov_misc[0].iov_base       = cmd->pdu;
2901         cmd->iov_misc[0].iov_len        = tx_size;
2902         cmd->iov_misc_count     = 1;
2903         cmd->tx_size            = tx_size;
2904
2905         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
2906                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
2907
2908         ret = iscsit_send_tx_data(cmd, conn, 1);
2909         if (ret < 0) {
2910                 iscsit_tx_thread_wait_for_tcp(conn);
2911                 return ret;
2912         }
2913
2914         spin_lock_bh(&cmd->istate_lock);
2915         cmd->i_state = want_response ?
2916                 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
2917         spin_unlock_bh(&cmd->istate_lock);
2918
2919         return 0;
2920 }
2921
2922 static int
2923 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2924 {
2925         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
2926         struct kvec *iov;
2927         u32 padding = 0;
2928         int niov = 0, tx_size;
2929
2930         iscsit_build_nopin_rsp(cmd, conn, hdr, true);
2931
2932         tx_size = ISCSI_HDR_LEN;
2933         iov = &cmd->iov_misc[0];
2934         iov[niov].iov_base      = cmd->pdu;
2935         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2936
2937         if (conn->conn_ops->HeaderDigest) {
2938                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2939
2940                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
2941                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
2942
2943                 iov[0].iov_len += ISCSI_CRC_LEN;
2944                 tx_size += ISCSI_CRC_LEN;
2945                 pr_debug("Attaching CRC32C HeaderDigest"
2946                         " to NopIn 0x%08x\n", *header_digest);
2947         }
2948
2949         /*
2950          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
2951          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
2952          */
2953         if (cmd->buf_ptr_size) {
2954                 iov[niov].iov_base      = cmd->buf_ptr;
2955                 iov[niov++].iov_len     = cmd->buf_ptr_size;
2956                 tx_size += cmd->buf_ptr_size;
2957
2958                 pr_debug("Echoing back %u bytes of ping"
2959                         " data.\n", cmd->buf_ptr_size);
2960
2961                 padding = ((-cmd->buf_ptr_size) & 3);
2962                 if (padding != 0) {
2963                         iov[niov].iov_base = &cmd->pad_bytes;
2964                         iov[niov++].iov_len = padding;
2965                         tx_size += padding;
2966                         pr_debug("Attaching %u additional"
2967                                 " padding bytes.\n", padding);
2968                 }
2969                 if (conn->conn_ops->DataDigest) {
2970                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2971                                 cmd->buf_ptr, cmd->buf_ptr_size,
2972                                 padding, (u8 *)&cmd->pad_bytes,
2973                                 (u8 *)&cmd->data_crc);
2974
2975                         iov[niov].iov_base = &cmd->data_crc;
2976                         iov[niov++].iov_len = ISCSI_CRC_LEN;
2977                         tx_size += ISCSI_CRC_LEN;
2978                         pr_debug("Attached DataDigest for %u"
2979                                 " bytes of ping data, CRC 0x%08x\n",
2980                                 cmd->buf_ptr_size, cmd->data_crc);
2981                 }
2982         }
2983
2984         cmd->iov_misc_count = niov;
2985         cmd->tx_size = tx_size;
2986
2987         return 0;
2988 }
2989
2990 static int iscsit_send_r2t(
2991         struct iscsi_cmd *cmd,
2992         struct iscsi_conn *conn)
2993 {
2994         int tx_size = 0;
2995         struct iscsi_r2t *r2t;
2996         struct iscsi_r2t_rsp *hdr;
2997         int ret;
2998
2999         r2t = iscsit_get_r2t_from_list(cmd);
3000         if (!r2t)
3001                 return -1;
3002
3003         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
3004         memset(hdr, 0, ISCSI_HDR_LEN);
3005         hdr->opcode             = ISCSI_OP_R2T;
3006         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3007         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
3008                         (struct scsi_lun *)&hdr->lun);
3009         hdr->itt                = cmd->init_task_tag;
3010         spin_lock_bh(&conn->sess->ttt_lock);
3011         r2t->targ_xfer_tag      = conn->sess->targ_xfer_tag++;
3012         if (r2t->targ_xfer_tag == 0xFFFFFFFF)
3013                 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
3014         spin_unlock_bh(&conn->sess->ttt_lock);
3015         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
3016         hdr->statsn             = cpu_to_be32(conn->stat_sn);
3017         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3018         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3019         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
3020         hdr->data_offset        = cpu_to_be32(r2t->offset);
3021         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
3022
3023         cmd->iov_misc[0].iov_base       = cmd->pdu;
3024         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
3025         tx_size += ISCSI_HDR_LEN;
3026
3027         if (conn->conn_ops->HeaderDigest) {
3028                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3029
3030                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
3031                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
3032
3033                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3034                 tx_size += ISCSI_CRC_LEN;
3035                 pr_debug("Attaching CRC32 HeaderDigest for R2T"
3036                         " PDU 0x%08x\n", *header_digest);
3037         }
3038
3039         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
3040                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
3041                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
3042                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
3043                         r2t->offset, r2t->xfer_len, conn->cid);
3044
3045         cmd->iov_misc_count = 1;
3046         cmd->tx_size = tx_size;
3047
3048         spin_lock_bh(&cmd->r2t_lock);
3049         r2t->sent_r2t = 1;
3050         spin_unlock_bh(&cmd->r2t_lock);
3051
3052         ret = iscsit_send_tx_data(cmd, conn, 1);
3053         if (ret < 0) {
3054                 iscsit_tx_thread_wait_for_tcp(conn);
3055                 return ret;
3056         }
3057
3058         spin_lock_bh(&cmd->dataout_timeout_lock);
3059         iscsit_start_dataout_timer(cmd, conn);
3060         spin_unlock_bh(&cmd->dataout_timeout_lock);
3061
3062         return 0;
3063 }
3064
3065 /*
3066  *      @recovery: If called from iscsi_task_reassign_complete_write() for
3067  *              connection recovery.
3068  */
3069 int iscsit_build_r2ts_for_cmd(
3070         struct iscsi_conn *conn,
3071         struct iscsi_cmd *cmd,
3072         bool recovery)
3073 {
3074         int first_r2t = 1;
3075         u32 offset = 0, xfer_len = 0;
3076
3077         spin_lock_bh(&cmd->r2t_lock);
3078         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
3079                 spin_unlock_bh(&cmd->r2t_lock);
3080                 return 0;
3081         }
3082
3083         if (conn->sess->sess_ops->DataSequenceInOrder &&
3084             !recovery)
3085                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
3086
3087         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
3088                 if (conn->sess->sess_ops->DataSequenceInOrder) {
3089                         offset = cmd->r2t_offset;
3090
3091                         if (first_r2t && recovery) {
3092                                 int new_data_end = offset +
3093                                         conn->sess->sess_ops->MaxBurstLength -
3094                                         cmd->next_burst_len;
3095
3096                                 if (new_data_end > cmd->se_cmd.data_length)
3097                                         xfer_len = cmd->se_cmd.data_length - offset;
3098                                 else
3099                                         xfer_len =
3100                                                 conn->sess->sess_ops->MaxBurstLength -
3101                                                 cmd->next_burst_len;
3102                         } else {
3103                                 int new_data_end = offset +
3104                                         conn->sess->sess_ops->MaxBurstLength;
3105
3106                                 if (new_data_end > cmd->se_cmd.data_length)
3107                                         xfer_len = cmd->se_cmd.data_length - offset;
3108                                 else
3109                                         xfer_len = conn->sess->sess_ops->MaxBurstLength;
3110                         }
3111                         cmd->r2t_offset += xfer_len;
3112
3113                         if (cmd->r2t_offset == cmd->se_cmd.data_length)
3114                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3115                 } else {
3116                         struct iscsi_seq *seq;
3117
3118                         seq = iscsit_get_seq_holder_for_r2t(cmd);
3119                         if (!seq) {
3120                                 spin_unlock_bh(&cmd->r2t_lock);
3121                                 return -1;
3122                         }
3123
3124                         offset = seq->offset;
3125                         xfer_len = seq->xfer_len;
3126
3127                         if (cmd->seq_send_order == cmd->seq_count)
3128                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3129                 }
3130                 cmd->outstanding_r2ts++;
3131                 first_r2t = 0;
3132
3133                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3134                         spin_unlock_bh(&cmd->r2t_lock);
3135                         return -1;
3136                 }
3137
3138                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3139                         break;
3140         }
3141         spin_unlock_bh(&cmd->r2t_lock);
3142
3143         return 0;
3144 }
3145
3146 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3147                         bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
3148 {
3149         if (inc_stat_sn)
3150                 cmd->stat_sn = conn->stat_sn++;
3151
3152         spin_lock_bh(&conn->sess->session_stats_lock);
3153         conn->sess->rsp_pdus++;
3154         spin_unlock_bh(&conn->sess->session_stats_lock);
3155
3156         memset(hdr, 0, ISCSI_HDR_LEN);
3157         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3158         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3159         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3160                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3161                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3162         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3163                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3164                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3165         }
3166         hdr->response           = cmd->iscsi_response;
3167         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3168         hdr->itt                = cmd->init_task_tag;
3169         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3170
3171         iscsit_increment_maxcmdsn(cmd, conn->sess);
3172         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3173         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3174
3175         pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3176                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3177                 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
3178                 cmd->se_cmd.scsi_status, conn->cid);
3179 }
3180 EXPORT_SYMBOL(iscsit_build_rsp_pdu);
3181
3182 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3183 {
3184         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
3185         struct kvec *iov;
3186         u32 padding = 0, tx_size = 0;
3187         int iov_count = 0;
3188         bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
3189
3190         iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
3191
3192         iov = &cmd->iov_misc[0];
3193         iov[iov_count].iov_base = cmd->pdu;
3194         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3195         tx_size += ISCSI_HDR_LEN;
3196
3197         /*
3198          * Attach SENSE DATA payload to iSCSI Response PDU
3199          */
3200         if (cmd->se_cmd.sense_buffer &&
3201            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3202             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3203                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
3204                 cmd->se_cmd.scsi_sense_length += sizeof (__be16);
3205
3206                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3207                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
3208                 iov[iov_count].iov_base = cmd->sense_buffer;
3209                 iov[iov_count++].iov_len =
3210                                 (cmd->se_cmd.scsi_sense_length + padding);
3211                 tx_size += cmd->se_cmd.scsi_sense_length;
3212
3213                 if (padding) {
3214                         memset(cmd->sense_buffer +
3215                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3216                         tx_size += padding;
3217                         pr_debug("Adding %u bytes of padding to"
3218                                 " SENSE.\n", padding);
3219                 }
3220
3221                 if (conn->conn_ops->DataDigest) {
3222                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3223                                 cmd->sense_buffer,
3224                                 (cmd->se_cmd.scsi_sense_length + padding),
3225                                 0, NULL, (u8 *)&cmd->data_crc);
3226
3227                         iov[iov_count].iov_base    = &cmd->data_crc;
3228                         iov[iov_count++].iov_len     = ISCSI_CRC_LEN;
3229                         tx_size += ISCSI_CRC_LEN;
3230
3231                         pr_debug("Attaching CRC32 DataDigest for"
3232                                 " SENSE, %u bytes CRC 0x%08x\n",
3233                                 (cmd->se_cmd.scsi_sense_length + padding),
3234                                 cmd->data_crc);
3235                 }
3236
3237                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3238                                 " Response PDU\n",
3239                                 cmd->se_cmd.scsi_sense_length);
3240         }
3241
3242         if (conn->conn_ops->HeaderDigest) {
3243                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3244
3245                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, cmd->pdu,
3246                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
3247
3248                 iov[0].iov_len += ISCSI_CRC_LEN;
3249                 tx_size += ISCSI_CRC_LEN;
3250                 pr_debug("Attaching CRC32 HeaderDigest for Response"
3251                                 " PDU 0x%08x\n", *header_digest);
3252         }
3253
3254         cmd->iov_misc_count = iov_count;
3255         cmd->tx_size = tx_size;
3256
3257         return 0;
3258 }
3259
3260 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3261 {
3262         switch (se_tmr->response) {
3263         case TMR_FUNCTION_COMPLETE:
3264                 return ISCSI_TMF_RSP_COMPLETE;
3265         case TMR_TASK_DOES_NOT_EXIST:
3266                 return ISCSI_TMF_RSP_NO_TASK;
3267         case TMR_LUN_DOES_NOT_EXIST:
3268                 return ISCSI_TMF_RSP_NO_LUN;
3269         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3270                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3271         case TMR_FUNCTION_AUTHORIZATION_FAILED:
3272                 return ISCSI_TMF_RSP_AUTH_FAILED;
3273         case TMR_FUNCTION_REJECTED:
3274         default:
3275                 return ISCSI_TMF_RSP_REJECTED;
3276         }
3277 }
3278
3279 void
3280 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3281                           struct iscsi_tm_rsp *hdr)
3282 {
3283         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3284
3285         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3286         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3287         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3288         hdr->itt                = cmd->init_task_tag;
3289         cmd->stat_sn            = conn->stat_sn++;
3290         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3291
3292         iscsit_increment_maxcmdsn(cmd, conn->sess);
3293         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3294         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3295
3296         pr_debug("Built Task Management Response ITT: 0x%08x,"
3297                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3298                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3299 }
3300 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
3301
3302 static int
3303 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3304 {
3305         struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
3306         u32 tx_size = 0;
3307
3308         iscsit_build_task_mgt_rsp(cmd, conn, hdr);
3309
3310         cmd->iov_misc[0].iov_base       = cmd->pdu;
3311         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
3312         tx_size += ISCSI_HDR_LEN;
3313
3314         if (conn->conn_ops->HeaderDigest) {
3315                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3316
3317                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
3318                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
3319
3320                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3321                 tx_size += ISCSI_CRC_LEN;
3322                 pr_debug("Attaching CRC32 HeaderDigest for Task"
3323                         " Mgmt Response PDU 0x%08x\n", *header_digest);
3324         }
3325
3326         cmd->iov_misc_count = 1;
3327         cmd->tx_size = tx_size;
3328
3329         return 0;
3330 }
3331
3332 static bool iscsit_check_inaddr_any(struct iscsi_np *np)
3333 {
3334         bool ret = false;
3335
3336         if (np->np_sockaddr.ss_family == AF_INET6) {
3337                 const struct sockaddr_in6 sin6 = {
3338                         .sin6_addr = IN6ADDR_ANY_INIT };
3339                 struct sockaddr_in6 *sock_in6 =
3340                          (struct sockaddr_in6 *)&np->np_sockaddr;
3341
3342                 if (!memcmp(sock_in6->sin6_addr.s6_addr,
3343                                 sin6.sin6_addr.s6_addr, 16))
3344                         ret = true;
3345         } else {
3346                 struct sockaddr_in * sock_in =
3347                         (struct sockaddr_in *)&np->np_sockaddr;
3348
3349                 if (sock_in->sin_addr.s_addr == htonl(INADDR_ANY))
3350                         ret = true;
3351         }
3352
3353         return ret;
3354 }
3355
3356 #define SENDTARGETS_BUF_LIMIT 32768U
3357
3358 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd)
3359 {
3360         char *payload = NULL;
3361         struct iscsi_conn *conn = cmd->conn;
3362         struct iscsi_portal_group *tpg;
3363         struct iscsi_tiqn *tiqn;
3364         struct iscsi_tpg_np *tpg_np;
3365         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3366         unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
3367
3368         buffer_len = max(conn->conn_ops->MaxRecvDataSegmentLength,
3369                          SENDTARGETS_BUF_LIMIT);
3370
3371         payload = kzalloc(buffer_len, GFP_KERNEL);
3372         if (!payload) {
3373                 pr_err("Unable to allocate memory for sendtargets"
3374                                 " response.\n");
3375                 return -ENOMEM;
3376         }
3377
3378         spin_lock(&tiqn_lock);
3379         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3380                 len = sprintf(buf, "TargetName=%s", tiqn->tiqn);
3381                 len += 1;
3382
3383                 if ((len + payload_len) > buffer_len) {
3384                         end_of_buf = 1;
3385                         goto eob;
3386                 }
3387                 memcpy(payload + payload_len, buf, len);
3388                 payload_len += len;
3389
3390                 spin_lock(&tiqn->tiqn_tpg_lock);
3391                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3392
3393                         spin_lock(&tpg->tpg_state_lock);
3394                         if ((tpg->tpg_state == TPG_STATE_FREE) ||
3395                             (tpg->tpg_state == TPG_STATE_INACTIVE)) {
3396                                 spin_unlock(&tpg->tpg_state_lock);
3397                                 continue;
3398                         }
3399                         spin_unlock(&tpg->tpg_state_lock);
3400
3401                         spin_lock(&tpg->tpg_np_lock);
3402                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3403                                                 tpg_np_list) {
3404                                 struct iscsi_np *np = tpg_np->tpg_np;
3405                                 bool inaddr_any = iscsit_check_inaddr_any(np);
3406
3407                                 len = sprintf(buf, "TargetAddress="
3408                                         "%s%s%s:%hu,%hu",
3409                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3410                                         "[" : "", (inaddr_any == false) ?
3411                                                 np->np_ip : conn->local_ip,
3412                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3413                                         "]" : "", (inaddr_any == false) ?
3414                                                 np->np_port : conn->local_port,
3415                                         tpg->tpgt);
3416                                 len += 1;
3417
3418                                 if ((len + payload_len) > buffer_len) {
3419                                         spin_unlock(&tpg->tpg_np_lock);
3420                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3421                                         end_of_buf = 1;
3422                                         goto eob;
3423                                 }
3424                                 memcpy(payload + payload_len, buf, len);
3425                                 payload_len += len;
3426                         }
3427                         spin_unlock(&tpg->tpg_np_lock);
3428                 }
3429                 spin_unlock(&tiqn->tiqn_tpg_lock);
3430 eob:
3431                 if (end_of_buf)
3432                         break;
3433         }
3434         spin_unlock(&tiqn_lock);
3435
3436         cmd->buf_ptr = payload;
3437
3438         return payload_len;
3439 }
3440
3441 /*
3442  *      FIXME: Add support for F_BIT and C_BIT when the length is longer than
3443  *      MaxRecvDataSegmentLength.
3444  */
3445 static int iscsit_send_text_rsp(
3446         struct iscsi_cmd *cmd,
3447         struct iscsi_conn *conn)
3448 {
3449         struct iscsi_text_rsp *hdr;
3450         struct kvec *iov;
3451         u32 padding = 0, tx_size = 0;
3452         int text_length, iov_count = 0;
3453
3454         text_length = iscsit_build_sendtargets_response(cmd);
3455         if (text_length < 0)
3456                 return text_length;
3457
3458         padding = ((-text_length) & 3);
3459         if (padding != 0) {
3460                 memset(cmd->buf_ptr + text_length, 0, padding);
3461                 pr_debug("Attaching %u additional bytes for"
3462                         " padding.\n", padding);
3463         }
3464
3465         hdr                     = (struct iscsi_text_rsp *) cmd->pdu;
3466         memset(hdr, 0, ISCSI_HDR_LEN);
3467         hdr->opcode             = ISCSI_OP_TEXT_RSP;
3468         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3469         hton24(hdr->dlength, text_length);
3470         hdr->itt                = cmd->init_task_tag;
3471         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3472         cmd->stat_sn            = conn->stat_sn++;
3473         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3474
3475         iscsit_increment_maxcmdsn(cmd, conn->sess);
3476         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3477         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3478
3479         iov = &cmd->iov_misc[0];
3480
3481         iov[iov_count].iov_base = cmd->pdu;
3482         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3483         iov[iov_count].iov_base = cmd->buf_ptr;
3484         iov[iov_count++].iov_len = text_length + padding;
3485
3486         tx_size += (ISCSI_HDR_LEN + text_length + padding);
3487
3488         if (conn->conn_ops->HeaderDigest) {
3489                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3490
3491                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
3492                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
3493
3494                 iov[0].iov_len += ISCSI_CRC_LEN;
3495                 tx_size += ISCSI_CRC_LEN;
3496                 pr_debug("Attaching CRC32 HeaderDigest for"
3497                         " Text Response PDU 0x%08x\n", *header_digest);
3498         }
3499
3500         if (conn->conn_ops->DataDigest) {
3501                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3502                                 cmd->buf_ptr, (text_length + padding),
3503                                 0, NULL, (u8 *)&cmd->data_crc);
3504
3505                 iov[iov_count].iov_base = &cmd->data_crc;
3506                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3507                 tx_size += ISCSI_CRC_LEN;
3508
3509                 pr_debug("Attaching DataDigest for %u bytes of text"
3510                         " data, CRC 0x%08x\n", (text_length + padding),
3511                         cmd->data_crc);
3512         }
3513
3514         cmd->iov_misc_count = iov_count;
3515         cmd->tx_size = tx_size;
3516
3517         pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x,"
3518                 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn,
3519                         text_length, conn->cid);
3520         return 0;
3521 }
3522
3523 void
3524 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3525                     struct iscsi_reject *hdr)
3526 {
3527         hdr->opcode             = ISCSI_OP_REJECT;
3528         hdr->reason             = cmd->reject_reason;
3529         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3530         hton24(hdr->dlength, ISCSI_HDR_LEN);
3531         hdr->ffffffff           = cpu_to_be32(0xffffffff);
3532         cmd->stat_sn            = conn->stat_sn++;
3533         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3534         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3535         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3536
3537 }
3538 EXPORT_SYMBOL(iscsit_build_reject);
3539
3540 static int iscsit_send_reject(
3541         struct iscsi_cmd *cmd,
3542         struct iscsi_conn *conn)
3543 {
3544         struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0];
3545         struct kvec *iov;
3546         u32 iov_count = 0, tx_size;
3547
3548         iscsit_build_reject(cmd, conn, hdr);
3549
3550         iov = &cmd->iov_misc[0];
3551         iov[iov_count].iov_base = cmd->pdu;
3552         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3553         iov[iov_count].iov_base = cmd->buf_ptr;
3554         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3555
3556         tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN);
3557
3558         if (conn->conn_ops->HeaderDigest) {
3559                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3560
3561                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, hdr,
3562                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest);
3563
3564                 iov[0].iov_len += ISCSI_CRC_LEN;
3565                 tx_size += ISCSI_CRC_LEN;
3566                 pr_debug("Attaching CRC32 HeaderDigest for"
3567                         " REJECT PDU 0x%08x\n", *header_digest);
3568         }
3569
3570         if (conn->conn_ops->DataDigest) {
3571                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, cmd->buf_ptr,
3572                                 ISCSI_HDR_LEN, 0, NULL, (u8 *)&cmd->data_crc);
3573
3574                 iov[iov_count].iov_base = &cmd->data_crc;
3575                 iov[iov_count++].iov_len  = ISCSI_CRC_LEN;
3576                 tx_size += ISCSI_CRC_LEN;
3577                 pr_debug("Attaching CRC32 DataDigest for REJECT"
3578                                 " PDU 0x%08x\n", cmd->data_crc);
3579         }
3580
3581         cmd->iov_misc_count = iov_count;
3582         cmd->tx_size = tx_size;
3583
3584         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3585                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3586
3587         return 0;
3588 }
3589
3590 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3591 {
3592         struct iscsi_thread_set *ts = conn->thread_set;
3593         int ord, cpu;
3594         /*
3595          * thread_id is assigned from iscsit_global->ts_bitmap from
3596          * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
3597          *
3598          * Here we use thread_id to determine which CPU that this
3599          * iSCSI connection's iscsi_thread_set will be scheduled to
3600          * execute upon.
3601          */
3602         ord = ts->thread_id % cpumask_weight(cpu_online_mask);
3603         for_each_online_cpu(cpu) {
3604                 if (ord-- == 0) {
3605                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3606                         return;
3607                 }
3608         }
3609         /*
3610          * This should never be reached..
3611          */
3612         dump_stack();
3613         cpumask_setall(conn->conn_cpumask);
3614 }
3615
3616 static inline void iscsit_thread_check_cpumask(
3617         struct iscsi_conn *conn,
3618         struct task_struct *p,
3619         int mode)
3620 {
3621         char buf[128];
3622         /*
3623          * mode == 1 signals iscsi_target_tx_thread() usage.
3624          * mode == 0 signals iscsi_target_rx_thread() usage.
3625          */
3626         if (mode == 1) {
3627                 if (!conn->conn_tx_reset_cpumask)
3628                         return;
3629                 conn->conn_tx_reset_cpumask = 0;
3630         } else {
3631                 if (!conn->conn_rx_reset_cpumask)
3632                         return;
3633                 conn->conn_rx_reset_cpumask = 0;
3634         }
3635         /*
3636          * Update the CPU mask for this single kthread so that
3637          * both TX and RX kthreads are scheduled to run on the
3638          * same CPU.
3639          */
3640         memset(buf, 0, 128);
3641         cpumask_scnprintf(buf, 128, conn->conn_cpumask);
3642         set_cpus_allowed_ptr(p, conn->conn_cpumask);
3643 }
3644
3645 static int
3646 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3647 {
3648         int ret;
3649
3650         switch (state) {
3651         case ISTATE_SEND_R2T:
3652                 ret = iscsit_send_r2t(cmd, conn);
3653                 if (ret < 0)
3654                         goto err;
3655                 break;
3656         case ISTATE_REMOVE:
3657                 spin_lock_bh(&conn->cmd_lock);
3658                 list_del_init(&cmd->i_conn_node);
3659                 spin_unlock_bh(&conn->cmd_lock);
3660
3661                 iscsit_free_cmd(cmd, false);
3662                 break;
3663         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3664                 iscsit_mod_nopin_response_timer(conn);
3665                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
3666                 if (ret < 0)
3667                         goto err;
3668                 break;
3669         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3670                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
3671                 if (ret < 0)
3672                         goto err;
3673                 break;
3674         default:
3675                 pr_err("Unknown Opcode: 0x%02x ITT:"
3676                        " 0x%08x, i_state: %d on CID: %hu\n",
3677                        cmd->iscsi_opcode, cmd->init_task_tag, state,
3678                        conn->cid);
3679                 goto err;
3680         }
3681
3682         return 0;
3683
3684 err:
3685         return -1;
3686 }
3687
3688 static int
3689 iscsit_handle_immediate_queue(struct iscsi_conn *conn)
3690 {
3691         struct iscsit_transport *t = conn->conn_transport;
3692         struct iscsi_queue_req *qr;
3693         struct iscsi_cmd *cmd;
3694         u8 state;
3695         int ret;
3696
3697         while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3698                 atomic_set(&conn->check_immediate_queue, 0);
3699                 cmd = qr->cmd;
3700                 state = qr->state;
3701                 kmem_cache_free(lio_qr_cache, qr);
3702
3703                 ret = t->iscsit_immediate_queue(conn, cmd, state);
3704                 if (ret < 0)
3705                         return ret;
3706         }
3707
3708         return 0;
3709 }
3710
3711 static int
3712 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3713 {
3714         int ret;
3715
3716 check_rsp_state:
3717         switch (state) {
3718         case ISTATE_SEND_DATAIN:
3719                 ret = iscsit_send_datain(cmd, conn);
3720                 if (ret < 0)
3721                         goto err;
3722                 else if (!ret)
3723                         /* more drs */
3724                         goto check_rsp_state;
3725                 else if (ret == 1) {
3726                         /* all done */
3727                         spin_lock_bh(&cmd->istate_lock);
3728                         cmd->i_state = ISTATE_SENT_STATUS;
3729                         spin_unlock_bh(&cmd->istate_lock);
3730
3731                         if (atomic_read(&conn->check_immediate_queue))
3732                                 return 1;
3733
3734                         return 0;
3735                 } else if (ret == 2) {
3736                         /* Still must send status,
3737                            SCF_TRANSPORT_TASK_SENSE was set */
3738                         spin_lock_bh(&cmd->istate_lock);
3739                         cmd->i_state = ISTATE_SEND_STATUS;
3740                         spin_unlock_bh(&cmd->istate_lock);
3741                         state = ISTATE_SEND_STATUS;
3742                         goto check_rsp_state;
3743                 }
3744
3745                 break;
3746         case ISTATE_SEND_STATUS:
3747         case ISTATE_SEND_STATUS_RECOVERY:
3748                 ret = iscsit_send_response(cmd, conn);
3749                 break;
3750         case ISTATE_SEND_LOGOUTRSP:
3751                 ret = iscsit_send_logout(cmd, conn);
3752                 break;
3753         case ISTATE_SEND_ASYNCMSG:
3754                 ret = iscsit_send_conn_drop_async_message(
3755                         cmd, conn);
3756                 break;
3757         case ISTATE_SEND_NOPIN:
3758                 ret = iscsit_send_nopin(cmd, conn);
3759                 break;
3760         case ISTATE_SEND_REJECT:
3761                 ret = iscsit_send_reject(cmd, conn);
3762                 break;
3763         case ISTATE_SEND_TASKMGTRSP:
3764                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3765                 if (ret != 0)
3766                         break;
3767                 ret = iscsit_tmr_post_handler(cmd, conn);
3768                 if (ret != 0)
3769                         iscsit_fall_back_to_erl0(conn->sess);
3770                 break;
3771         case ISTATE_SEND_TEXTRSP:
3772                 ret = iscsit_send_text_rsp(cmd, conn);
3773                 break;
3774         default:
3775                 pr_err("Unknown Opcode: 0x%02x ITT:"
3776                        " 0x%08x, i_state: %d on CID: %hu\n",
3777                        cmd->iscsi_opcode, cmd->init_task_tag,
3778                        state, conn->cid);
3779                 goto err;
3780         }
3781         if (ret < 0)
3782                 goto err;
3783
3784         if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
3785                 iscsit_tx_thread_wait_for_tcp(conn);
3786                 iscsit_unmap_iovec(cmd);
3787                 goto err;
3788         }
3789         iscsit_unmap_iovec(cmd);
3790
3791         switch (state) {
3792         case ISTATE_SEND_LOGOUTRSP:
3793                 if (!iscsit_logout_post_handler(cmd, conn))
3794                         goto restart;
3795                 /* fall through */
3796         case ISTATE_SEND_STATUS:
3797         case ISTATE_SEND_ASYNCMSG:
3798         case ISTATE_SEND_NOPIN:
3799         case ISTATE_SEND_STATUS_RECOVERY:
3800         case ISTATE_SEND_TEXTRSP:
3801         case ISTATE_SEND_TASKMGTRSP:
3802         case ISTATE_SEND_REJECT:
3803                 spin_lock_bh(&cmd->istate_lock);
3804                 cmd->i_state = ISTATE_SENT_STATUS;
3805                 spin_unlock_bh(&cmd->istate_lock);
3806                 break;
3807         default:
3808                 pr_err("Unknown Opcode: 0x%02x ITT:"
3809                        " 0x%08x, i_state: %d on CID: %hu\n",
3810                        cmd->iscsi_opcode, cmd->init_task_tag,
3811                        cmd->i_state, conn->cid);
3812                 goto err;
3813         }
3814
3815         if (atomic_read(&conn->check_immediate_queue))
3816                 return 1;
3817
3818         return 0;
3819
3820 err:
3821         return -1;
3822 restart:
3823         return -EAGAIN;
3824 }
3825
3826 static int iscsit_handle_response_queue(struct iscsi_conn *conn)
3827 {
3828         struct iscsit_transport *t = conn->conn_transport;
3829         struct iscsi_queue_req *qr;
3830         struct iscsi_cmd *cmd;
3831         u8 state;
3832         int ret;
3833
3834         while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3835                 cmd = qr->cmd;
3836                 state = qr->state;
3837                 kmem_cache_free(lio_qr_cache, qr);
3838
3839                 ret = t->iscsit_response_queue(conn, cmd, state);
3840                 if (ret == 1 || ret < 0)
3841                         return ret;
3842         }
3843
3844         return 0;
3845 }
3846
3847 int iscsi_target_tx_thread(void *arg)
3848 {
3849         int ret = 0;
3850         struct iscsi_conn *conn;
3851         struct iscsi_thread_set *ts = arg;
3852         /*
3853          * Allow ourselves to be interrupted by SIGINT so that a
3854          * connection recovery / failure event can be triggered externally.
3855          */
3856         allow_signal(SIGINT);
3857
3858 restart:
3859         conn = iscsi_tx_thread_pre_handler(ts);
3860         if (!conn)
3861                 goto out;
3862
3863         ret = 0;
3864
3865         while (!kthread_should_stop()) {
3866                 /*
3867                  * Ensure that both TX and RX per connection kthreads
3868                  * are scheduled to run on the same CPU.
3869                  */
3870                 iscsit_thread_check_cpumask(conn, current, 1);
3871
3872                 wait_event_interruptible(conn->queues_wq,
3873                                          !iscsit_conn_all_queues_empty(conn) ||
3874                                          ts->status == ISCSI_THREAD_SET_RESET);
3875
3876                 if ((ts->status == ISCSI_THREAD_SET_RESET) ||
3877                      signal_pending(current))
3878                         goto transport_err;
3879
3880 get_immediate:
3881                 ret = iscsit_handle_immediate_queue(conn);
3882                 if (ret < 0)
3883                         goto transport_err;
3884
3885                 ret = iscsit_handle_response_queue(conn);
3886                 if (ret == 1)
3887                         goto get_immediate;
3888                 else if (ret == -EAGAIN)
3889                         goto restart;
3890                 else if (ret < 0)
3891                         goto transport_err;
3892         }
3893
3894 transport_err:
3895         iscsit_take_action_for_connection_exit(conn);
3896         goto restart;
3897 out:
3898         return 0;
3899 }
3900
3901 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
3902 {
3903         struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
3904         struct iscsi_cmd *cmd;
3905         int ret = 0;
3906
3907         switch (hdr->opcode & ISCSI_OPCODE_MASK) {
3908         case ISCSI_OP_SCSI_CMD:
3909                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3910                 if (!cmd)
3911                         goto reject;
3912
3913                 ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
3914                 break;
3915         case ISCSI_OP_SCSI_DATA_OUT:
3916                 ret = iscsit_handle_data_out(conn, buf);
3917                 break;
3918         case ISCSI_OP_NOOP_OUT:
3919                 cmd = NULL;
3920                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
3921                         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3922                         if (!cmd)
3923                                 goto reject;
3924                 }
3925                 ret = iscsit_handle_nop_out(conn, cmd, buf);
3926                 break;
3927         case ISCSI_OP_SCSI_TMFUNC:
3928                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3929                 if (!cmd)
3930                         goto reject;
3931
3932                 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
3933                 break;
3934         case ISCSI_OP_TEXT:
3935                 ret = iscsit_handle_text_cmd(conn, buf);
3936                 break;
3937         case ISCSI_OP_LOGOUT:
3938                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3939                 if (!cmd)
3940                         goto reject;
3941
3942                 ret = iscsit_handle_logout_cmd(conn, cmd, buf);
3943                 if (ret > 0)
3944                         wait_for_completion_timeout(&conn->conn_logout_comp,
3945                                         SECONDS_FOR_LOGOUT_COMP * HZ);
3946                 break;
3947         case ISCSI_OP_SNACK:
3948                 ret = iscsit_handle_snack(conn, buf);
3949                 break;
3950         default:
3951                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
3952                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3953                         pr_err("Cannot recover from unknown"
3954                         " opcode while ERL=0, closing iSCSI connection.\n");
3955                         return -1;
3956                 }
3957                 if (!conn->conn_ops->OFMarker) {
3958                         pr_err("Unable to recover from unknown"
3959                         " opcode while OFMarker=No, closing iSCSI"
3960                                 " connection.\n");
3961                         return -1;
3962                 }
3963                 if (iscsit_recover_from_unknown_opcode(conn) < 0) {
3964                         pr_err("Unable to recover from unknown"
3965                                 " opcode, closing iSCSI connection.\n");
3966                         return -1;
3967                 }
3968                 break;
3969         }
3970
3971         return ret;
3972 reject:
3973         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
3974 }
3975
3976 int iscsi_target_rx_thread(void *arg)
3977 {
3978         int ret;
3979         u8 buffer[ISCSI_HDR_LEN], opcode;
3980         u32 checksum = 0, digest = 0;
3981         struct iscsi_conn *conn = NULL;
3982         struct iscsi_thread_set *ts = arg;
3983         struct kvec iov;
3984         /*
3985          * Allow ourselves to be interrupted by SIGINT so that a
3986          * connection recovery / failure event can be triggered externally.
3987          */
3988         allow_signal(SIGINT);
3989
3990 restart:
3991         conn = iscsi_rx_thread_pre_handler(ts);
3992         if (!conn)
3993                 goto out;
3994
3995         if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) {
3996                 struct completion comp;
3997                 int rc;
3998
3999                 init_completion(&comp);
4000                 rc = wait_for_completion_interruptible(&comp);
4001                 if (rc < 0)
4002                         goto transport_err;
4003
4004                 goto out;
4005         }
4006
4007         while (!kthread_should_stop()) {
4008                 /*
4009                  * Ensure that both TX and RX per connection kthreads
4010                  * are scheduled to run on the same CPU.
4011                  */
4012                 iscsit_thread_check_cpumask(conn, current, 0);
4013
4014                 memset(buffer, 0, ISCSI_HDR_LEN);
4015                 memset(&iov, 0, sizeof(struct kvec));
4016
4017                 iov.iov_base    = buffer;
4018                 iov.iov_len     = ISCSI_HDR_LEN;
4019
4020                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
4021                 if (ret != ISCSI_HDR_LEN) {
4022                         iscsit_rx_thread_wait_for_tcp(conn);
4023                         goto transport_err;
4024                 }
4025
4026                 /*
4027                  * Set conn->bad_hdr for use with REJECT PDUs.
4028                  */
4029                 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN);
4030
4031                 if (conn->conn_ops->HeaderDigest) {
4032                         iov.iov_base    = &digest;
4033                         iov.iov_len     = ISCSI_CRC_LEN;
4034
4035                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
4036                         if (ret != ISCSI_CRC_LEN) {
4037                                 iscsit_rx_thread_wait_for_tcp(conn);
4038                                 goto transport_err;
4039                         }
4040
4041                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
4042                                         buffer, ISCSI_HDR_LEN,
4043                                         0, NULL, (u8 *)&checksum);
4044
4045                         if (digest != checksum) {
4046                                 pr_err("HeaderDigest CRC32C failed,"
4047                                         " received 0x%08x, computed 0x%08x\n",
4048                                         digest, checksum);
4049                                 /*
4050                                  * Set the PDU to 0xff so it will intentionally
4051                                  * hit default in the switch below.
4052                                  */
4053                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
4054                                 spin_lock_bh(&conn->sess->session_stats_lock);
4055                                 conn->sess->conn_digest_errors++;
4056                                 spin_unlock_bh(&conn->sess->session_stats_lock);
4057                         } else {
4058                                 pr_debug("Got HeaderDigest CRC32C"
4059                                                 " 0x%08x\n", checksum);
4060                         }
4061                 }
4062
4063                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
4064                         goto transport_err;
4065
4066                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
4067
4068                 if (conn->sess->sess_ops->SessionType &&
4069                    ((!(opcode & ISCSI_OP_TEXT)) ||
4070                     (!(opcode & ISCSI_OP_LOGOUT)))) {
4071                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
4072                         " while in Discovery Session, rejecting.\n", opcode);
4073                         iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
4074                                           buffer);
4075                         goto transport_err;
4076                 }
4077
4078                 ret = iscsi_target_rx_opcode(conn, buffer);
4079                 if (ret < 0)
4080                         goto transport_err;
4081         }
4082
4083 transport_err:
4084         if (!signal_pending(current))
4085                 atomic_set(&conn->transport_failed, 1);
4086         iscsit_take_action_for_connection_exit(conn);
4087         goto restart;
4088 out:
4089         return 0;
4090 }
4091
4092 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
4093 {
4094         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
4095         struct iscsi_session *sess = conn->sess;
4096         /*
4097          * We expect this function to only ever be called from either RX or TX
4098          * thread context via iscsit_close_connection() once the other context
4099          * has been reset -> returned sleeping pre-handler state.
4100          */
4101         spin_lock_bh(&conn->cmd_lock);
4102         list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
4103
4104                 list_del_init(&cmd->i_conn_node);
4105                 spin_unlock_bh(&conn->cmd_lock);
4106
4107                 iscsit_increment_maxcmdsn(cmd, sess);
4108
4109                 iscsit_free_cmd(cmd, true);
4110
4111                 spin_lock_bh(&conn->cmd_lock);
4112         }
4113         spin_unlock_bh(&conn->cmd_lock);
4114 }
4115
4116 static void iscsit_stop_timers_for_cmds(
4117         struct iscsi_conn *conn)
4118 {
4119         struct iscsi_cmd *cmd;
4120
4121         spin_lock_bh(&conn->cmd_lock);
4122         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
4123                 if (cmd->data_direction == DMA_TO_DEVICE)
4124                         iscsit_stop_dataout_timer(cmd);
4125         }
4126         spin_unlock_bh(&conn->cmd_lock);
4127 }
4128
4129 int iscsit_close_connection(
4130         struct iscsi_conn *conn)
4131 {
4132         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
4133         struct iscsi_session    *sess = conn->sess;
4134
4135         pr_debug("Closing iSCSI connection CID %hu on SID:"
4136                 " %u\n", conn->cid, sess->sid);
4137         /*
4138          * Always up conn_logout_comp just in case the RX Thread is sleeping
4139          * and the logout response never got sent because the connection
4140          * failed.
4141          */
4142         complete(&conn->conn_logout_comp);
4143
4144         iscsi_release_thread_set(conn);
4145
4146         iscsit_stop_timers_for_cmds(conn);
4147         iscsit_stop_nopin_response_timer(conn);
4148         iscsit_stop_nopin_timer(conn);
4149
4150         if (conn->conn_transport->iscsit_wait_conn)
4151                 conn->conn_transport->iscsit_wait_conn(conn);
4152
4153         iscsit_free_queue_reqs_for_conn(conn);
4154
4155         /*
4156          * During Connection recovery drop unacknowledged out of order
4157          * commands for this connection, and prepare the other commands
4158          * for realligence.
4159          *
4160          * During normal operation clear the out of order commands (but
4161          * do not free the struct iscsi_ooo_cmdsn's) and release all
4162          * struct iscsi_cmds.
4163          */
4164         if (atomic_read(&conn->connection_recovery)) {
4165                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
4166                 iscsit_prepare_cmds_for_realligance(conn);
4167         } else {
4168                 iscsit_clear_ooo_cmdsns_for_conn(conn);
4169                 iscsit_release_commands_from_conn(conn);
4170         }
4171
4172         /*
4173          * Handle decrementing session or connection usage count if
4174          * a logout response was not able to be sent because the
4175          * connection failed.  Fall back to Session Recovery here.
4176          */
4177         if (atomic_read(&conn->conn_logout_remove)) {
4178                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4179                         iscsit_dec_conn_usage_count(conn);
4180                         iscsit_dec_session_usage_count(sess);
4181                 }
4182                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4183                         iscsit_dec_conn_usage_count(conn);
4184
4185                 atomic_set(&conn->conn_logout_remove, 0);
4186                 atomic_set(&sess->session_reinstatement, 0);
4187                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4188         }
4189
4190         spin_lock_bh(&sess->conn_lock);
4191         list_del(&conn->conn_list);
4192
4193         /*
4194          * Attempt to let the Initiator know this connection failed by
4195          * sending an Connection Dropped Async Message on another
4196          * active connection.
4197          */
4198         if (atomic_read(&conn->connection_recovery))
4199                 iscsit_build_conn_drop_async_message(conn);
4200
4201         spin_unlock_bh(&sess->conn_lock);
4202
4203         /*
4204          * If connection reinstatement is being performed on this connection,
4205          * up the connection reinstatement semaphore that is being blocked on
4206          * in iscsit_cause_connection_reinstatement().
4207          */
4208         spin_lock_bh(&conn->state_lock);
4209         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4210                 spin_unlock_bh(&conn->state_lock);
4211                 complete(&conn->conn_wait_comp);
4212                 wait_for_completion(&conn->conn_post_wait_comp);
4213                 spin_lock_bh(&conn->state_lock);
4214         }
4215
4216         /*
4217          * If connection reinstatement is being performed on this connection
4218          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4219          * connection wait rcfr semaphore that is being blocked on
4220          * an iscsit_connection_reinstatement_rcfr().
4221          */
4222         if (atomic_read(&conn->connection_wait_rcfr)) {
4223                 spin_unlock_bh(&conn->state_lock);
4224                 complete(&conn->conn_wait_rcfr_comp);
4225                 wait_for_completion(&conn->conn_post_wait_comp);
4226                 spin_lock_bh(&conn->state_lock);
4227         }
4228         atomic_set(&conn->connection_reinstatement, 1);
4229         spin_unlock_bh(&conn->state_lock);
4230
4231         /*
4232          * If any other processes are accessing this connection pointer we
4233          * must wait until they have completed.
4234          */
4235         iscsit_check_conn_usage_count(conn);
4236
4237         if (conn->conn_rx_hash.tfm)
4238                 crypto_free_hash(conn->conn_rx_hash.tfm);
4239         if (conn->conn_tx_hash.tfm)
4240                 crypto_free_hash(conn->conn_tx_hash.tfm);
4241
4242         if (conn->conn_cpumask)
4243                 free_cpumask_var(conn->conn_cpumask);
4244
4245         kfree(conn->conn_ops);
4246         conn->conn_ops = NULL;
4247
4248         if (conn->sock)
4249                 sock_release(conn->sock);
4250
4251         if (conn->conn_transport->iscsit_free_conn)
4252                 conn->conn_transport->iscsit_free_conn(conn);
4253
4254         iscsit_put_transport(conn->conn_transport);
4255
4256         conn->thread_set = NULL;
4257
4258         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4259         conn->conn_state = TARG_CONN_STATE_FREE;
4260         kfree(conn);
4261
4262         spin_lock_bh(&sess->conn_lock);
4263         atomic_dec(&sess->nconn);
4264         pr_debug("Decremented iSCSI connection count to %hu from node:"
4265                 " %s\n", atomic_read(&sess->nconn),
4266                 sess->sess_ops->InitiatorName);
4267         /*
4268          * Make sure that if one connection fails in an non ERL=2 iSCSI
4269          * Session that they all fail.
4270          */
4271         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4272              !atomic_read(&sess->session_logout))
4273                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4274
4275         /*
4276          * If this was not the last connection in the session, and we are
4277          * performing session reinstatement or falling back to ERL=0, call
4278          * iscsit_stop_session() without sleeping to shutdown the other
4279          * active connections.
4280          */
4281         if (atomic_read(&sess->nconn)) {
4282                 if (!atomic_read(&sess->session_reinstatement) &&
4283                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4284                         spin_unlock_bh(&sess->conn_lock);
4285                         return 0;
4286                 }
4287                 if (!atomic_read(&sess->session_stop_active)) {
4288                         atomic_set(&sess->session_stop_active, 1);
4289                         spin_unlock_bh(&sess->conn_lock);
4290                         iscsit_stop_session(sess, 0, 0);
4291                         return 0;
4292                 }
4293                 spin_unlock_bh(&sess->conn_lock);
4294                 return 0;
4295         }
4296
4297         /*
4298          * If this was the last connection in the session and one of the
4299          * following is occurring:
4300          *
4301          * Session Reinstatement is not being performed, and are falling back
4302          * to ERL=0 call iscsit_close_session().
4303          *
4304          * Session Logout was requested.  iscsit_close_session() will be called
4305          * elsewhere.
4306          *
4307          * Session Continuation is not being performed, start the Time2Retain
4308          * handler and check if sleep_on_sess_wait_sem is active.
4309          */
4310         if (!atomic_read(&sess->session_reinstatement) &&
4311              atomic_read(&sess->session_fall_back_to_erl0)) {
4312                 spin_unlock_bh(&sess->conn_lock);
4313                 target_put_session(sess->se_sess);
4314
4315                 return 0;
4316         } else if (atomic_read(&sess->session_logout)) {
4317                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4318                 sess->session_state = TARG_SESS_STATE_FREE;
4319                 spin_unlock_bh(&sess->conn_lock);
4320
4321                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4322                         complete(&sess->session_wait_comp);
4323
4324                 return 0;
4325         } else {
4326                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4327                 sess->session_state = TARG_SESS_STATE_FAILED;
4328
4329                 if (!atomic_read(&sess->session_continuation)) {
4330                         spin_unlock_bh(&sess->conn_lock);
4331                         iscsit_start_time2retain_handler(sess);
4332                 } else
4333                         spin_unlock_bh(&sess->conn_lock);
4334
4335                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4336                         complete(&sess->session_wait_comp);
4337
4338                 return 0;
4339         }
4340         spin_unlock_bh(&sess->conn_lock);
4341
4342         return 0;
4343 }
4344
4345 int iscsit_close_session(struct iscsi_session *sess)
4346 {
4347         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
4348         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4349
4350         if (atomic_read(&sess->nconn)) {
4351                 pr_err("%d connection(s) still exist for iSCSI session"
4352                         " to %s\n", atomic_read(&sess->nconn),
4353                         sess->sess_ops->InitiatorName);
4354                 BUG();
4355         }
4356
4357         spin_lock_bh(&se_tpg->session_lock);
4358         atomic_set(&sess->session_logout, 1);
4359         atomic_set(&sess->session_reinstatement, 1);
4360         iscsit_stop_time2retain_timer(sess);
4361         spin_unlock_bh(&se_tpg->session_lock);
4362
4363         /*
4364          * transport_deregister_session_configfs() will clear the
4365          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4366          * can be setting it again with __transport_register_session() in
4367          * iscsi_post_login_handler() again after the iscsit_stop_session()
4368          * completes in iscsi_np context.
4369          */
4370         transport_deregister_session_configfs(sess->se_sess);
4371
4372         /*
4373          * If any other processes are accessing this session pointer we must
4374          * wait until they have completed.  If we are in an interrupt (the
4375          * time2retain handler) and contain and active session usage count we
4376          * restart the timer and exit.
4377          */
4378         if (!in_interrupt()) {
4379                 if (iscsit_check_session_usage_count(sess) == 1)
4380                         iscsit_stop_session(sess, 1, 1);
4381         } else {
4382                 if (iscsit_check_session_usage_count(sess) == 2) {
4383                         atomic_set(&sess->session_logout, 0);
4384                         iscsit_start_time2retain_handler(sess);
4385                         return 0;
4386                 }
4387         }
4388
4389         transport_deregister_session(sess->se_sess);
4390
4391         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4392                 iscsit_free_connection_recovery_entires(sess);
4393
4394         iscsit_free_all_ooo_cmdsns(sess);
4395
4396         spin_lock_bh(&se_tpg->session_lock);
4397         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4398         sess->session_state = TARG_SESS_STATE_FREE;
4399         pr_debug("Released iSCSI session from node: %s\n",
4400                         sess->sess_ops->InitiatorName);
4401         tpg->nsessions--;
4402         if (tpg->tpg_tiqn)
4403                 tpg->tpg_tiqn->tiqn_nsessions--;
4404
4405         pr_debug("Decremented number of active iSCSI Sessions on"
4406                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4407
4408         spin_lock(&sess_idr_lock);
4409         idr_remove(&sess_idr, sess->session_index);
4410         spin_unlock(&sess_idr_lock);
4411
4412         kfree(sess->sess_ops);
4413         sess->sess_ops = NULL;
4414         spin_unlock_bh(&se_tpg->session_lock);
4415
4416         kfree(sess);
4417         return 0;
4418 }
4419
4420 static void iscsit_logout_post_handler_closesession(
4421         struct iscsi_conn *conn)
4422 {
4423         struct iscsi_session *sess = conn->sess;
4424
4425         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4426         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4427
4428         atomic_set(&conn->conn_logout_remove, 0);
4429         complete(&conn->conn_logout_comp);
4430
4431         iscsit_dec_conn_usage_count(conn);
4432         iscsit_stop_session(sess, 1, 1);
4433         iscsit_dec_session_usage_count(sess);
4434         target_put_session(sess->se_sess);
4435 }
4436
4437 static void iscsit_logout_post_handler_samecid(
4438         struct iscsi_conn *conn)
4439 {
4440         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4441         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4442
4443         atomic_set(&conn->conn_logout_remove, 0);
4444         complete(&conn->conn_logout_comp);
4445
4446         iscsit_cause_connection_reinstatement(conn, 1);
4447         iscsit_dec_conn_usage_count(conn);
4448 }
4449
4450 static void iscsit_logout_post_handler_diffcid(
4451         struct iscsi_conn *conn,
4452         u16 cid)
4453 {
4454         struct iscsi_conn *l_conn;
4455         struct iscsi_session *sess = conn->sess;
4456
4457         if (!sess)
4458                 return;
4459
4460         spin_lock_bh(&sess->conn_lock);
4461         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4462                 if (l_conn->cid == cid) {
4463                         iscsit_inc_conn_usage_count(l_conn);
4464                         break;
4465                 }
4466         }
4467         spin_unlock_bh(&sess->conn_lock);
4468
4469         if (!l_conn)
4470                 return;
4471
4472         if (l_conn->sock)
4473                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4474
4475         spin_lock_bh(&l_conn->state_lock);
4476         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4477         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4478         spin_unlock_bh(&l_conn->state_lock);
4479
4480         iscsit_cause_connection_reinstatement(l_conn, 1);
4481         iscsit_dec_conn_usage_count(l_conn);
4482 }
4483
4484 /*
4485  *      Return of 0 causes the TX thread to restart.
4486  */
4487 int iscsit_logout_post_handler(
4488         struct iscsi_cmd *cmd,
4489         struct iscsi_conn *conn)
4490 {
4491         int ret = 0;
4492
4493         switch (cmd->logout_reason) {
4494         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4495                 switch (cmd->logout_response) {
4496                 case ISCSI_LOGOUT_SUCCESS:
4497                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4498                 default:
4499                         iscsit_logout_post_handler_closesession(conn);
4500                         break;
4501                 }
4502                 ret = 0;
4503                 break;
4504         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4505                 if (conn->cid == cmd->logout_cid) {
4506                         switch (cmd->logout_response) {
4507                         case ISCSI_LOGOUT_SUCCESS:
4508                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4509                         default:
4510                                 iscsit_logout_post_handler_samecid(conn);
4511                                 break;
4512                         }
4513                         ret = 0;
4514                 } else {
4515                         switch (cmd->logout_response) {
4516                         case ISCSI_LOGOUT_SUCCESS:
4517                                 iscsit_logout_post_handler_diffcid(conn,
4518                                         cmd->logout_cid);
4519                                 break;
4520                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4521                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4522                         default:
4523                                 break;
4524                         }
4525                         ret = 1;
4526                 }
4527                 break;
4528         case ISCSI_LOGOUT_REASON_RECOVERY:
4529                 switch (cmd->logout_response) {
4530                 case ISCSI_LOGOUT_SUCCESS:
4531                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4532                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4533                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4534                 default:
4535                         break;
4536                 }
4537                 ret = 1;
4538                 break;
4539         default:
4540                 break;
4541
4542         }
4543         return ret;
4544 }
4545 EXPORT_SYMBOL(iscsit_logout_post_handler);
4546
4547 void iscsit_fail_session(struct iscsi_session *sess)
4548 {
4549         struct iscsi_conn *conn;
4550
4551         spin_lock_bh(&sess->conn_lock);
4552         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4553                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4554                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4555         }
4556         spin_unlock_bh(&sess->conn_lock);
4557
4558         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4559         sess->session_state = TARG_SESS_STATE_FAILED;
4560 }
4561
4562 int iscsit_free_session(struct iscsi_session *sess)
4563 {
4564         u16 conn_count = atomic_read(&sess->nconn);
4565         struct iscsi_conn *conn, *conn_tmp = NULL;
4566         int is_last;
4567
4568         spin_lock_bh(&sess->conn_lock);
4569         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4570
4571         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4572                         conn_list) {
4573                 if (conn_count == 0)
4574                         break;
4575
4576                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4577                         is_last = 1;
4578                 } else {
4579                         iscsit_inc_conn_usage_count(conn_tmp);
4580                         is_last = 0;
4581                 }
4582                 iscsit_inc_conn_usage_count(conn);
4583
4584                 spin_unlock_bh(&sess->conn_lock);
4585                 iscsit_cause_connection_reinstatement(conn, 1);
4586                 spin_lock_bh(&sess->conn_lock);
4587
4588                 iscsit_dec_conn_usage_count(conn);
4589                 if (is_last == 0)
4590                         iscsit_dec_conn_usage_count(conn_tmp);
4591
4592                 conn_count--;
4593         }
4594
4595         if (atomic_read(&sess->nconn)) {
4596                 spin_unlock_bh(&sess->conn_lock);
4597                 wait_for_completion(&sess->session_wait_comp);
4598         } else
4599                 spin_unlock_bh(&sess->conn_lock);
4600
4601         target_put_session(sess->se_sess);
4602         return 0;
4603 }
4604
4605 void iscsit_stop_session(
4606         struct iscsi_session *sess,
4607         int session_sleep,
4608         int connection_sleep)
4609 {
4610         u16 conn_count = atomic_read(&sess->nconn);
4611         struct iscsi_conn *conn, *conn_tmp = NULL;
4612         int is_last;
4613
4614         spin_lock_bh(&sess->conn_lock);
4615         if (session_sleep)
4616                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4617
4618         if (connection_sleep) {
4619                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4620                                 conn_list) {
4621                         if (conn_count == 0)
4622                                 break;
4623
4624                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4625                                 is_last = 1;
4626                         } else {
4627                                 iscsit_inc_conn_usage_count(conn_tmp);
4628                                 is_last = 0;
4629                         }
4630                         iscsit_inc_conn_usage_count(conn);
4631
4632                         spin_unlock_bh(&sess->conn_lock);
4633                         iscsit_cause_connection_reinstatement(conn, 1);
4634                         spin_lock_bh(&sess->conn_lock);
4635
4636                         iscsit_dec_conn_usage_count(conn);
4637                         if (is_last == 0)
4638                                 iscsit_dec_conn_usage_count(conn_tmp);
4639                         conn_count--;
4640                 }
4641         } else {
4642                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4643                         iscsit_cause_connection_reinstatement(conn, 0);
4644         }
4645
4646         if (session_sleep && atomic_read(&sess->nconn)) {
4647                 spin_unlock_bh(&sess->conn_lock);
4648                 wait_for_completion(&sess->session_wait_comp);
4649         } else
4650                 spin_unlock_bh(&sess->conn_lock);
4651 }
4652
4653 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4654 {
4655         struct iscsi_session *sess;
4656         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4657         struct se_session *se_sess, *se_sess_tmp;
4658         int session_count = 0;
4659
4660         spin_lock_bh(&se_tpg->session_lock);
4661         if (tpg->nsessions && !force) {
4662                 spin_unlock_bh(&se_tpg->session_lock);
4663                 return -1;
4664         }
4665
4666         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4667                         sess_list) {
4668                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4669
4670                 spin_lock(&sess->conn_lock);
4671                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4672                     atomic_read(&sess->session_logout) ||
4673                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4674                         spin_unlock(&sess->conn_lock);
4675                         continue;
4676                 }
4677                 atomic_set(&sess->session_reinstatement, 1);
4678                 spin_unlock(&sess->conn_lock);
4679                 spin_unlock_bh(&se_tpg->session_lock);
4680
4681                 iscsit_free_session(sess);
4682                 spin_lock_bh(&se_tpg->session_lock);
4683
4684                 session_count++;
4685         }
4686         spin_unlock_bh(&se_tpg->session_lock);
4687
4688         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4689                         " Group: %hu\n", session_count, tpg->tpgt);
4690         return 0;
4691 }
4692
4693 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4694 MODULE_VERSION("4.1.x");
4695 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4696 MODULE_LICENSE("GPL");
4697
4698 module_init(iscsi_target_init_module);
4699 module_exit(iscsi_target_cleanup_module);