drivers/infiniband/ulp/isert/ib_isert.c

   1 /*******************************************************************************
   2  * This file contains iSCSI extentions for RDMA (iSER) Verbs
   3  *
   4  * (c) Copyright 2013 RisingTide Systems LLC.
   5  *
   6  * Nicholas A. Bellinger <nab@linux-iscsi.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  ****************************************************************************/
  18
  19 #include <linux/string.h>
  20 #include <linux/module.h>
  21 #include <linux/scatterlist.h>
  22 #include <linux/socket.h>
  23 #include <linux/in.h>
  24 #include <linux/in6.h>
  25 #include <rdma/ib_verbs.h>
  26 #include <rdma/rdma_cm.h>
  27 #include <target/target_core_base.h>
  28 #include <target/target_core_fabric.h>
  29 #include <target/iscsi/iscsi_transport.h>
  30
  31 #include "isert_proto.h"
  32 #include "ib_isert.h"
  33
  34 #define ISERT_MAX_CONN          8
  35 #define ISER_MAX_RX_CQ_LEN      (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
  36 #define ISER_MAX_TX_CQ_LEN      (ISERT_QP_MAX_REQ_DTOS  * ISERT_MAX_CONN)
  37
  38 static DEFINE_MUTEX(device_list_mutex);
  39 static LIST_HEAD(device_list);
  40 static struct workqueue_struct *isert_rx_wq;
  41 static struct workqueue_struct *isert_comp_wq;
  42 static struct kmem_cache *isert_cmd_cache;
  43
  44 static void
  45 isert_qp_event_callback(struct ib_event *e, void *context)
  46 {
  47         struct isert_conn *isert_conn = (struct isert_conn *)context;
  48
  49         pr_err("isert_qp_event_callback event: %d\n", e->event);
  50         switch (e->event) {
  51         case IB_EVENT_COMM_EST:
  52                 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
  53                 break;
  54         case IB_EVENT_QP_LAST_WQE_REACHED:
  55                 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
  56                 break;
  57         default:
  58                 break;
  59         }
  60 }
  61
  62 static int
  63 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
  64 {
  65         int ret;
  66
  67         ret = ib_query_device(ib_dev, devattr);
  68         if (ret) {
  69                 pr_err("ib_query_device() failed: %d\n", ret);
  70                 return ret;
  71         }
  72         pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
  73         pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
  74
  75         return 0;
  76 }
  77
  78 static int
  79 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
  80 {
  81         struct isert_device *device = isert_conn->conn_device;
  82         struct ib_qp_init_attr attr;
  83         struct ib_device_attr devattr;
  84         int ret, index, min_index = 0;
  85
  86         memset(&devattr, 0, sizeof(struct ib_device_attr));
  87         ret = isert_query_device(cma_id->device, &devattr);
  88         if (ret)
  89                 return ret;
  90
  91         mutex_lock(&device_list_mutex);
  92         for (index = 0; index < device->cqs_used; index++)
  93                 if (device->cq_active_qps[index] <
  94                     device->cq_active_qps[min_index])
  95                         min_index = index;
  96         device->cq_active_qps[min_index]++;
  97         pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
  98         mutex_unlock(&device_list_mutex);
  99
 100         memset(&attr, 0, sizeof(struct ib_qp_init_attr));
 101         attr.event_handler = isert_qp_event_callback;
 102         attr.qp_context = isert_conn;
 103         attr.send_cq = device->dev_tx_cq[min_index];
 104         attr.recv_cq = device->dev_rx_cq[min_index];
 105         attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
 106         attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
 107         /*
 108          * FIXME: Use devattr.max_sge - 2 for max_send_sge as
 109          * work-around for RDMA_READ..
 110          */
 111         attr.cap.max_send_sge = devattr.max_sge - 2;
 112         isert_conn->max_sge = attr.cap.max_send_sge;
 113
 114         attr.cap.max_recv_sge = 1;
 115         attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 116         attr.qp_type = IB_QPT_RC;
 117
 118         pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
 119                  cma_id->device);
 120         pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
 121                  isert_conn->conn_pd->device);
 122
 123         ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
 124         if (ret) {
 125                 pr_err("rdma_create_qp failed for cma_id %d\n", ret);
 126                 return ret;
 127         }
 128         isert_conn->conn_qp = cma_id->qp;
 129         pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
 130
 131         return 0;
 132 }
 133
 134 static void
 135 isert_cq_event_callback(struct ib_event *e, void *context)
 136 {
 137         pr_debug("isert_cq_event_callback event: %d\n", e->event);
 138 }
 139
 140 static int
 141 isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
 142 {
 143         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 144         struct iser_rx_desc *rx_desc;
 145         struct ib_sge *rx_sg;
 146         u64 dma_addr;
 147         int i, j;
 148
 149         isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
 150                                 sizeof(struct iser_rx_desc), GFP_KERNEL);
 151         if (!isert_conn->conn_rx_descs)
 152                 goto fail;
 153
 154         rx_desc = isert_conn->conn_rx_descs;
 155
 156         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 157                 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
 158                                         ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 159                 if (ib_dma_mapping_error(ib_dev, dma_addr))
 160                         goto dma_map_fail;
 161
 162                 rx_desc->dma_addr = dma_addr;
 163
 164                 rx_sg = &rx_desc->rx_sg;
 165                 rx_sg->addr = rx_desc->dma_addr;
 166                 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
 167                 rx_sg->lkey = isert_conn->conn_mr->lkey;
 168         }
 169
 170         isert_conn->conn_rx_desc_head = 0;
 171         return 0;
 172
 173 dma_map_fail:
 174         rx_desc = isert_conn->conn_rx_descs;
 175         for (j = 0; j < i; j++, rx_desc++) {
 176                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 177                                     ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 178         }
 179         kfree(isert_conn->conn_rx_descs);
 180         isert_conn->conn_rx_descs = NULL;
 181 fail:
 182         return -ENOMEM;
 183 }
 184
 185 static void
 186 isert_free_rx_descriptors(struct isert_conn *isert_conn)
 187 {
 188         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 189         struct iser_rx_desc *rx_desc;
 190         int i;
 191
 192         if (!isert_conn->conn_rx_descs)
 193                 return;
 194
 195         rx_desc = isert_conn->conn_rx_descs;
 196         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 197                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 198                                     ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 199         }
 200
 201         kfree(isert_conn->conn_rx_descs);
 202         isert_conn->conn_rx_descs = NULL;
 203 }
 204
 205 static void isert_cq_tx_callback(struct ib_cq *, void *);
 206 static void isert_cq_rx_callback(struct ib_cq *, void *);
 207
 208 static int
 209 isert_create_device_ib_res(struct isert_device *device)
 210 {
 211         struct ib_device *ib_dev = device->ib_device;
 212         struct isert_cq_desc *cq_desc;
 213         int ret = 0, i, j;
 214
 215         device->cqs_used = min_t(int, num_online_cpus(),
 216                                  device->ib_device->num_comp_vectors);
 217         device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
 218         pr_debug("Using %d CQs, device %s supports %d vectors\n",
 219                  device->cqs_used, device->ib_device->name,
 220                  device->ib_device->num_comp_vectors);
 221         device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
 222                                 device->cqs_used, GFP_KERNEL);
 223         if (!device->cq_desc) {
 224                 pr_err("Unable to allocate device->cq_desc\n");
 225                 return -ENOMEM;
 226         }
 227         cq_desc = device->cq_desc;
 228
 229         device->dev_pd = ib_alloc_pd(ib_dev);
 230         if (IS_ERR(device->dev_pd)) {
 231                 ret = PTR_ERR(device->dev_pd);
 232                 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
 233                 goto out_cq_desc;
 234         }
 235
 236         for (i = 0; i < device->cqs_used; i++) {
 237                 cq_desc[i].device = device;
 238                 cq_desc[i].cq_index = i;
 239
 240                 device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
 241                                                 isert_cq_rx_callback,
 242                                                 isert_cq_event_callback,
 243                                                 (void *)&cq_desc[i],
 244                                                 ISER_MAX_RX_CQ_LEN, i);
 245                 if (IS_ERR(device->dev_rx_cq[i]))
 246                         goto out_cq;
 247
 248                 device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
 249                                                 isert_cq_tx_callback,
 250                                                 isert_cq_event_callback,
 251                                                 (void *)&cq_desc[i],
 252                                                 ISER_MAX_TX_CQ_LEN, i);
 253                 if (IS_ERR(device->dev_tx_cq[i]))
 254                         goto out_cq;
 255
 256                 if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP))
 257                         goto out_cq;
 258
 259                 if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP))
 260                         goto out_cq;
 261         }
 262
 263         device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
 264         if (IS_ERR(device->dev_mr)) {
 265                 ret = PTR_ERR(device->dev_mr);
 266                 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
 267                 goto out_cq;
 268         }
 269
 270         return 0;
 271
 272 out_cq:
 273         for (j = 0; j < i; j++) {
 274                 cq_desc = &device->cq_desc[j];
 275
 276                 if (device->dev_rx_cq[j]) {
 277                         cancel_work_sync(&cq_desc->cq_rx_work);
 278                         ib_destroy_cq(device->dev_rx_cq[j]);
 279                 }
 280                 if (device->dev_tx_cq[j]) {
 281                         cancel_work_sync(&cq_desc->cq_tx_work);
 282                         ib_destroy_cq(device->dev_tx_cq[j]);
 283                 }
 284         }
 285         ib_dealloc_pd(device->dev_pd);
 286
 287 out_cq_desc:
 288         kfree(device->cq_desc);
 289
 290         return ret;
 291 }
 292
 293 static void
 294 isert_free_device_ib_res(struct isert_device *device)
 295 {
 296         struct isert_cq_desc *cq_desc;
 297         int i;
 298
 299         for (i = 0; i < device->cqs_used; i++) {
 300                 cq_desc = &device->cq_desc[i];
 301
 302                 cancel_work_sync(&cq_desc->cq_rx_work);
 303                 cancel_work_sync(&cq_desc->cq_tx_work);
 304                 ib_destroy_cq(device->dev_rx_cq[i]);
 305                 ib_destroy_cq(device->dev_tx_cq[i]);
 306                 device->dev_rx_cq[i] = NULL;
 307                 device->dev_tx_cq[i] = NULL;
 308         }
 309
 310         ib_dereg_mr(device->dev_mr);
 311         ib_dealloc_pd(device->dev_pd);
 312         kfree(device->cq_desc);
 313 }
 314
 315 static void
 316 isert_device_try_release(struct isert_device *device)
 317 {
 318         mutex_lock(&device_list_mutex);
 319         device->refcount--;
 320         if (!device->refcount) {
 321                 isert_free_device_ib_res(device);
 322                 list_del(&device->dev_node);
 323                 kfree(device);
 324         }
 325         mutex_unlock(&device_list_mutex);
 326 }
 327
 328 static struct isert_device *
 329 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
 330 {
 331         struct isert_device *device;
 332         int ret;
 333
 334         mutex_lock(&device_list_mutex);
 335         list_for_each_entry(device, &device_list, dev_node) {
 336                 if (device->ib_device->node_guid == cma_id->device->node_guid) {
 337                         device->refcount++;
 338                         mutex_unlock(&device_list_mutex);
 339                         return device;
 340                 }
 341         }
 342
 343         device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
 344         if (!device) {
 345                 mutex_unlock(&device_list_mutex);
 346                 return ERR_PTR(-ENOMEM);
 347         }
 348
 349         INIT_LIST_HEAD(&device->dev_node);
 350
 351         device->ib_device = cma_id->device;
 352         ret = isert_create_device_ib_res(device);
 353         if (ret) {
 354                 kfree(device);
 355                 mutex_unlock(&device_list_mutex);
 356                 return ERR_PTR(ret);
 357         }
 358
 359         device->refcount++;
 360         list_add_tail(&device->dev_node, &device_list);
 361         mutex_unlock(&device_list_mutex);
 362
 363         return device;
 364 }
 365
 366 static int
 367 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 368 {
 369         struct iscsi_np *np = cma_id->context;
 370         struct isert_np *isert_np = np->np_context;
 371         struct isert_conn *isert_conn;
 372         struct isert_device *device;
 373         struct ib_device *ib_dev = cma_id->device;
 374         int ret = 0;
 375
 376         pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
 377                  cma_id, cma_id->context);
 378
 379         isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
 380         if (!isert_conn) {
 381                 pr_err("Unable to allocate isert_conn\n");
 382                 return -ENOMEM;
 383         }
 384         isert_conn->state = ISER_CONN_INIT;
 385         INIT_LIST_HEAD(&isert_conn->conn_accept_node);
 386         init_completion(&isert_conn->conn_login_comp);
 387         init_waitqueue_head(&isert_conn->conn_wait);
 388         init_waitqueue_head(&isert_conn->conn_wait_comp_err);
 389         kref_init(&isert_conn->conn_kref);
 390         kref_get(&isert_conn->conn_kref);
 391
 392         cma_id->context = isert_conn;
 393         isert_conn->conn_cm_id = cma_id;
 394         isert_conn->responder_resources = event->param.conn.responder_resources;
 395         isert_conn->initiator_depth = event->param.conn.initiator_depth;
 396         pr_debug("Using responder_resources: %u initiator_depth: %u\n",
 397                  isert_conn->responder_resources, isert_conn->initiator_depth);
 398
 399         isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
 400                                         ISER_RX_LOGIN_SIZE, GFP_KERNEL);
 401         if (!isert_conn->login_buf) {
 402                 pr_err("Unable to allocate isert_conn->login_buf\n");
 403                 ret = -ENOMEM;
 404                 goto out;
 405         }
 406
 407         isert_conn->login_req_buf = isert_conn->login_buf;
 408         isert_conn->login_rsp_buf = isert_conn->login_buf +
 409                                     ISCSI_DEF_MAX_RECV_SEG_LEN;
 410         pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
 411                  isert_conn->login_buf, isert_conn->login_req_buf,
 412                  isert_conn->login_rsp_buf);
 413
 414         isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
 415                                 (void *)isert_conn->login_req_buf,
 416                                 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
 417
 418         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
 419         if (ret) {
 420                 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
 421                        ret);
 422                 isert_conn->login_req_dma = 0;
 423                 goto out_login_buf;
 424         }
 425
 426         isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
 427                                         (void *)isert_conn->login_rsp_buf,
 428                                         ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 429
 430         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
 431         if (ret) {
 432                 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
 433                        ret);
 434                 isert_conn->login_rsp_dma = 0;
 435                 goto out_req_dma_map;
 436         }
 437
 438         device = isert_device_find_by_ib_dev(cma_id);
 439         if (IS_ERR(device)) {
 440                 ret = PTR_ERR(device);
 441                 goto out_rsp_dma_map;
 442         }
 443
 444         isert_conn->conn_device = device;
 445         isert_conn->conn_pd = device->dev_pd;
 446         isert_conn->conn_mr = device->dev_mr;
 447
 448         ret = isert_conn_setup_qp(isert_conn, cma_id);
 449         if (ret)
 450                 goto out_conn_dev;
 451
 452         mutex_lock(&isert_np->np_accept_mutex);
 453         list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node);
 454         mutex_unlock(&isert_np->np_accept_mutex);
 455
 456         pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np);
 457         wake_up(&isert_np->np_accept_wq);
 458         return 0;
 459
 460 out_conn_dev:
 461         isert_device_try_release(device);
 462 out_rsp_dma_map:
 463         ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
 464                             ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 465 out_req_dma_map:
 466         ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
 467                             ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
 468 out_login_buf:
 469         kfree(isert_conn->login_buf);
 470 out:
 471         kfree(isert_conn);
 472         return ret;
 473 }
 474
 475 static void
 476 isert_connect_release(struct isert_conn *isert_conn)
 477 {
 478         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 479         struct isert_device *device = isert_conn->conn_device;
 480         int cq_index;
 481
 482         pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
 483
 484         if (isert_conn->conn_qp) {
 485                 cq_index = ((struct isert_cq_desc *)
 486                         isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
 487                 pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
 488                 isert_conn->conn_device->cq_active_qps[cq_index]--;
 489
 490                 rdma_destroy_qp(isert_conn->conn_cm_id);
 491         }
 492
 493         isert_free_rx_descriptors(isert_conn);
 494         rdma_destroy_id(isert_conn->conn_cm_id);
 495
 496         if (isert_conn->login_buf) {
 497                 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
 498                                     ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 499                 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
 500                                     ISCSI_DEF_MAX_RECV_SEG_LEN,
 501                                     DMA_FROM_DEVICE);
 502                 kfree(isert_conn->login_buf);
 503         }
 504         kfree(isert_conn);
 505
 506         if (device)
 507                 isert_device_try_release(device);
 508
 509         pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
 510 }
 511
 512 static void
 513 isert_connected_handler(struct rdma_cm_id *cma_id)
 514 {
 515         return;
 516 }
 517
 518 static void
 519 isert_release_conn_kref(struct kref *kref)
 520 {
 521         struct isert_conn *isert_conn = container_of(kref,
 522                                 struct isert_conn, conn_kref);
 523
 524         pr_debug("Calling isert_connect_release for final kref %s/%d\n",
 525                  current->comm, current->pid);
 526
 527         isert_connect_release(isert_conn);
 528 }
 529
 530 static void
 531 isert_put_conn(struct isert_conn *isert_conn)
 532 {
 533         kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
 534 }
 535
 536 static void
 537 isert_disconnect_work(struct work_struct *work)
 538 {
 539         struct isert_conn *isert_conn = container_of(work,
 540                                 struct isert_conn, conn_logout_work);
 541
 542         pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
 543
 544         isert_conn->state = ISER_CONN_DOWN;
 545
 546         if (isert_conn->post_recv_buf_count == 0 &&
 547             atomic_read(&isert_conn->post_send_buf_count) == 0) {
 548                 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n");
 549                 wake_up(&isert_conn->conn_wait);
 550         }
 551
 552         isert_put_conn(isert_conn);
 553 }
 554
 555 static void
 556 isert_disconnected_handler(struct rdma_cm_id *cma_id)
 557 {
 558         struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;
 559
 560         INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
 561         schedule_work(&isert_conn->conn_logout_work);
 562 }
 563
 564 static int
 565 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 566 {
 567         int ret = 0;
 568
 569         pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
 570                  event->event, event->status, cma_id->context, cma_id);
 571
 572         switch (event->event) {
 573         case RDMA_CM_EVENT_CONNECT_REQUEST:
 574                 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
 575                 ret = isert_connect_request(cma_id, event);
 576                 break;
 577         case RDMA_CM_EVENT_ESTABLISHED:
 578                 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
 579                 isert_connected_handler(cma_id);
 580                 break;
 581         case RDMA_CM_EVENT_DISCONNECTED:
 582                 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
 583                 isert_disconnected_handler(cma_id);
 584                 break;
 585         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 586         case RDMA_CM_EVENT_ADDR_CHANGE:
 587                 break;
 588         case RDMA_CM_EVENT_CONNECT_ERROR:
 589         default:
 590                 pr_err("Unknown RDMA CMA event: %d\n", event->event);
 591                 break;
 592         }
 593
 594         if (ret != 0) {
 595                 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
 596                        event->event, ret);
 597                 dump_stack();
 598         }
 599
 600         return ret;
 601 }
 602
 603 static int
 604 isert_post_recv(struct isert_conn *isert_conn, u32 count)
 605 {
 606         struct ib_recv_wr *rx_wr, *rx_wr_failed;
 607         int i, ret;
 608         unsigned int rx_head = isert_conn->conn_rx_desc_head;
 609         struct iser_rx_desc *rx_desc;
 610
 611         for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
 612                 rx_desc         = &isert_conn->conn_rx_descs[rx_head];
 613                 rx_wr->wr_id    = (unsigned long)rx_desc;
 614                 rx_wr->sg_list  = &rx_desc->rx_sg;
 615                 rx_wr->num_sge  = 1;
 616                 rx_wr->next     = rx_wr + 1;
 617                 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
 618         }
 619
 620         rx_wr--;
 621         rx_wr->next = NULL; /* mark end of work requests list */
 622
 623         isert_conn->post_recv_buf_count += count;
 624         ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
 625                                 &rx_wr_failed);
 626         if (ret) {
 627                 pr_err("ib_post_recv() failed with ret: %d\n", ret);
 628                 isert_conn->post_recv_buf_count -= count;
 629         } else {
 630                 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
 631                 isert_conn->conn_rx_desc_head = rx_head;
 632         }
 633         return ret;
 634 }
 635
 636 static int
 637 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
 638 {
 639         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 640         struct ib_send_wr send_wr, *send_wr_failed;
 641         int ret;
 642
 643         ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
 644                                       ISER_HEADERS_LEN, DMA_TO_DEVICE);
 645
 646         send_wr.next    = NULL;
 647         send_wr.wr_id   = (unsigned long)tx_desc;
 648         send_wr.sg_list = tx_desc->tx_sg;
 649         send_wr.num_sge = tx_desc->num_sge;
 650         send_wr.opcode  = IB_WR_SEND;
 651         send_wr.send_flags = IB_SEND_SIGNALED;
 652
 653         atomic_inc(&isert_conn->post_send_buf_count);
 654
 655         ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
 656         if (ret) {
 657                 pr_err("ib_post_send() failed, ret: %d\n", ret);
 658                 atomic_dec(&isert_conn->post_send_buf_count);
 659         }
 660
 661         return ret;
 662 }
 663
 664 static void
 665 isert_create_send_desc(struct isert_conn *isert_conn,
 666                        struct isert_cmd *isert_cmd,
 667                        struct iser_tx_desc *tx_desc)
 668 {
 669         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 670
 671         ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
 672                                    ISER_HEADERS_LEN, DMA_TO_DEVICE);
 673
 674         memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
 675         tx_desc->iser_header.flags = ISER_VER;
 676
 677         tx_desc->num_sge = 1;
 678         tx_desc->isert_cmd = isert_cmd;
 679
 680         if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
 681                 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
 682                 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
 683         }
 684 }
 685
 686 static int
 687 isert_init_tx_hdrs(struct isert_conn *isert_conn,
 688                    struct iser_tx_desc *tx_desc)
 689 {
 690         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 691         u64 dma_addr;
 692
 693         dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
 694                         ISER_HEADERS_LEN, DMA_TO_DEVICE);
 695         if (ib_dma_mapping_error(ib_dev, dma_addr)) {
 696                 pr_err("ib_dma_mapping_error() failed\n");
 697                 return -ENOMEM;
 698         }
 699
 700         tx_desc->dma_addr = dma_addr;
 701         tx_desc->tx_sg[0].addr  = tx_desc->dma_addr;
 702         tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
 703         tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
 704
 705         pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
 706                  " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
 707                  tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
 708
 709         return 0;
 710 }
 711
 712 static void
 713 isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
 714 {
 715         isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
 716         send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
 717         send_wr->opcode = IB_WR_SEND;
 718         send_wr->send_flags = IB_SEND_SIGNALED;
 719         send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
 720         send_wr->num_sge = isert_cmd->tx_desc.num_sge;
 721 }
 722
 723 static int
 724 isert_rdma_post_recvl(struct isert_conn *isert_conn)
 725 {
 726         struct ib_recv_wr rx_wr, *rx_wr_fail;
 727         struct ib_sge sge;
 728         int ret;
 729
 730         memset(&sge, 0, sizeof(struct ib_sge));
 731         sge.addr = isert_conn->login_req_dma;
 732         sge.length = ISER_RX_LOGIN_SIZE;
 733         sge.lkey = isert_conn->conn_mr->lkey;
 734
 735         pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
 736                 sge.addr, sge.length, sge.lkey);
 737
 738         memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
 739         rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
 740         rx_wr.sg_list = &sge;
 741         rx_wr.num_sge = 1;
 742
 743         isert_conn->post_recv_buf_count++;
 744         ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
 745         if (ret) {
 746                 pr_err("ib_post_recv() failed: %d\n", ret);
 747                 isert_conn->post_recv_buf_count--;
 748         }
 749
 750         pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
 751         return ret;
 752 }
 753
 754 static int
 755 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 756                    u32 length)
 757 {
 758         struct isert_conn *isert_conn = conn->context;
 759         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 760         struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
 761         int ret;
 762
 763         isert_create_send_desc(isert_conn, NULL, tx_desc);
 764
 765         memcpy(&tx_desc->iscsi_header, &login->rsp[0],
 766                sizeof(struct iscsi_hdr));
 767
 768         isert_init_tx_hdrs(isert_conn, tx_desc);
 769
 770         if (length > 0) {
 771                 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
 772
 773                 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
 774                                            length, DMA_TO_DEVICE);
 775
 776                 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
 777
 778                 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
 779                                               length, DMA_TO_DEVICE);
 780
 781                 tx_dsg->addr    = isert_conn->login_rsp_dma;
 782                 tx_dsg->length  = length;
 783                 tx_dsg->lkey    = isert_conn->conn_mr->lkey;
 784                 tx_desc->num_sge = 2;
 785         }
 786         if (!login->login_failed) {
 787                 if (login->login_complete) {
 788                         ret = isert_alloc_rx_descriptors(isert_conn);
 789                         if (ret)
 790                                 return ret;
 791
 792                         ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
 793                         if (ret)
 794                                 return ret;
 795
 796                         isert_conn->state = ISER_CONN_UP;
 797                         goto post_send;
 798                 }
 799
 800                 ret = isert_rdma_post_recvl(isert_conn);
 801                 if (ret)
 802                         return ret;
 803         }
 804 post_send:
 805         ret = isert_post_send(isert_conn, tx_desc);
 806         if (ret)
 807                 return ret;
 808
 809         return 0;
 810 }
 811
 812 static void
 813 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
 814                    struct isert_conn *isert_conn)
 815 {
 816         struct iscsi_conn *conn = isert_conn->conn;
 817         struct iscsi_login *login = conn->conn_login;
 818         int size;
 819
 820         if (!login) {
 821                 pr_err("conn->conn_login is NULL\n");
 822                 dump_stack();
 823                 return;
 824         }
 825
 826         if (login->first_request) {
 827                 struct iscsi_login_req *login_req =
 828                         (struct iscsi_login_req *)&rx_desc->iscsi_header;
 829                 /*
 830                  * Setup the initial iscsi_login values from the leading
 831                  * login request PDU.
 832                  */
 833                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
 834                 login->current_stage =
 835                         (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
 836                          >> 2;
 837                 login->version_min      = login_req->min_version;
 838                 login->version_max      = login_req->max_version;
 839                 memcpy(login->isid, login_req->isid, 6);
 840                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
 841                 login->init_task_tag    = login_req->itt;
 842                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
 843                 login->cid              = be16_to_cpu(login_req->cid);
 844                 login->tsih             = be16_to_cpu(login_req->tsih);
 845         }
 846
 847         memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
 848
 849         size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
 850         pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
 851                  size, rx_buflen, MAX_KEY_VALUE_PAIRS);
 852         memcpy(login->req_buf, &rx_desc->data[0], size);
 853
 854         complete(&isert_conn->conn_login_comp);
 855 }
 856
 857 static void
 858 isert_release_cmd(struct iscsi_cmd *cmd)
 859 {
 860         struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
 861                                                    iscsi_cmd);
 862
 863         pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);
 864
 865         kfree(cmd->buf_ptr);
 866         kfree(cmd->tmr_req);
 867
 868         kmem_cache_free(isert_cmd_cache, isert_cmd);
 869 }
 870
 871 static struct iscsi_cmd
 872 *isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
 873 {
 874         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
 875         struct isert_cmd *isert_cmd;
 876
 877         isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
 878         if (!isert_cmd) {
 879                 pr_err("Unable to allocate isert_cmd\n");
 880                 return NULL;
 881         }
 882         isert_cmd->conn = isert_conn;
 883         isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;
 884
 885         return &isert_cmd->iscsi_cmd;
 886 }
 887
 888 static int
 889 isert_handle_scsi_cmd(struct isert_conn *isert_conn,
 890                       struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
 891                       unsigned char *buf)
 892 {
 893         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
 894         struct iscsi_conn *conn = isert_conn->conn;
 895         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
 896         struct scatterlist *sg;
 897         int imm_data, imm_data_len, unsol_data, sg_nents, rc;
 898         bool dump_payload = false;
 899
 900         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
 901         if (rc < 0)
 902                 return rc;
 903
 904         imm_data = cmd->immediate_data;
 905         imm_data_len = cmd->first_burst_len;
 906         unsol_data = cmd->unsolicited_data;
 907
 908         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
 909         if (rc < 0) {
 910                 return 0;
 911         } else if (rc > 0) {
 912                 dump_payload = true;
 913                 goto sequence_cmd;
 914         }
 915
 916         if (!imm_data)
 917                 return 0;
 918
 919         sg = &cmd->se_cmd.t_data_sg[0];
 920         sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
 921
 922         pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
 923                  sg, sg_nents, &rx_desc->data[0], imm_data_len);
 924
 925         sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
 926
 927         cmd->write_data_done += imm_data_len;
 928
 929         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 930                 spin_lock_bh(&cmd->istate_lock);
 931                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 932                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 933                 spin_unlock_bh(&cmd->istate_lock);
 934         }
 935
 936 sequence_cmd:
 937         rc = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
 938
 939         if (!rc && dump_payload == false && unsol_data)
 940                 iscsit_set_unsoliticed_dataout(cmd);
 941
 942         if (rc == CMDSN_ERROR_CANNOT_RECOVER)
 943                 return iscsit_add_reject_from_cmd(
 944                            ISCSI_REASON_PROTOCOL_ERROR,
 945                            1, 0, (unsigned char *)hdr, cmd);
 946
 947         return 0;
 948 }
 949
 950 static int
 951 isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
 952                            struct iser_rx_desc *rx_desc, unsigned char *buf)
 953 {
 954         struct scatterlist *sg_start;
 955         struct iscsi_conn *conn = isert_conn->conn;
 956         struct iscsi_cmd *cmd = NULL;
 957         struct iscsi_data *hdr = (struct iscsi_data *)buf;
 958         u32 unsol_data_len = ntoh24(hdr->dlength);
 959         int rc, sg_nents, sg_off, page_off;
 960
 961         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
 962         if (rc < 0)
 963                 return rc;
 964         else if (!cmd)
 965                 return 0;
 966         /*
 967          * FIXME: Unexpected unsolicited_data out
 968          */
 969         if (!cmd->unsolicited_data) {
 970                 pr_err("Received unexpected solicited data payload\n");
 971                 dump_stack();
 972                 return -1;
 973         }
 974
 975         pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
 976                  unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
 977
 978         sg_off = cmd->write_data_done / PAGE_SIZE;
 979         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
 980         sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
 981         page_off = cmd->write_data_done % PAGE_SIZE;
 982         /*
 983          * FIXME: Non page-aligned unsolicited_data out
 984          */
 985         if (page_off) {
 986                 pr_err("Received unexpected non-page aligned data payload\n");
 987                 dump_stack();
 988                 return -1;
 989         }
 990         pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
 991                  sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
 992
 993         sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
 994                             unsol_data_len);
 995
 996         rc = iscsit_check_dataout_payload(cmd, hdr, false);
 997         if (rc < 0)
 998                 return rc;
 999
1000         return 0;
1001 }
1002
1003 static int
1004 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1005                 uint32_t read_stag, uint64_t read_va,
1006                 uint32_t write_stag, uint64_t write_va)
1007 {
1008         struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1009         struct iscsi_conn *conn = isert_conn->conn;
1010         struct iscsi_cmd *cmd;
1011         struct isert_cmd *isert_cmd;
1012         int ret = -EINVAL;
1013         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1014
1015         switch (opcode) {
1016         case ISCSI_OP_SCSI_CMD:
1017                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1018                 if (!cmd)
1019                         break;
1020
1021                 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
1022                 isert_cmd->read_stag = read_stag;
1023                 isert_cmd->read_va = read_va;
1024                 isert_cmd->write_stag = write_stag;
1025                 isert_cmd->write_va = write_va;
1026
1027                 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
1028                                         rx_desc, (unsigned char *)hdr);
1029                 break;
1030         case ISCSI_OP_NOOP_OUT:
1031                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1032                 if (!cmd)
1033                         break;
1034
1035                 ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr);
1036                 break;
1037         case ISCSI_OP_SCSI_DATA_OUT:
1038                 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1039                                                 (unsigned char *)hdr);
1040                 break;
1041         case ISCSI_OP_SCSI_TMFUNC:
1042                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1043                 if (!cmd)
1044                         break;
1045
1046                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1047                                                 (unsigned char *)hdr);
1048                 break;
1049         case ISCSI_OP_LOGOUT:
1050                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1051                 if (!cmd)
1052                         break;
1053
1054                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1055                 if (ret > 0)
1056                         wait_for_completion_timeout(&conn->conn_logout_comp,
1057                                                     SECONDS_FOR_LOGOUT_COMP *
1058                                                     HZ);
1059                 break;
1060         default:
1061                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1062                 dump_stack();
1063                 break;
1064         }
1065
1066         return ret;
1067 }
1068
1069 static void
1070 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1071 {
1072         struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1073         uint64_t read_va = 0, write_va = 0;
1074         uint32_t read_stag = 0, write_stag = 0;
1075         int rc;
1076
1077         switch (iser_hdr->flags & 0xF0) {
1078         case ISCSI_CTRL:
1079                 if (iser_hdr->flags & ISER_RSV) {
1080                         read_stag = be32_to_cpu(iser_hdr->read_stag);
1081                         read_va = be64_to_cpu(iser_hdr->read_va);
1082                         pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1083                                  read_stag, (unsigned long long)read_va);
1084                 }
1085                 if (iser_hdr->flags & ISER_WSV) {
1086                         write_stag = be32_to_cpu(iser_hdr->write_stag);
1087                         write_va = be64_to_cpu(iser_hdr->write_va);
1088                         pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1089                                  write_stag, (unsigned long long)write_va);
1090                 }
1091
1092                 pr_debug("ISER ISCSI_CTRL PDU\n");
1093                 break;
1094         case ISER_HELLO:
1095                 pr_err("iSER Hello message\n");
1096                 break;
1097         default:
1098                 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
1099                 break;
1100         }
1101
1102         rc = isert_rx_opcode(isert_conn, rx_desc,
1103                              read_stag, read_va, write_stag, write_va);
1104 }
1105
1106 static void
1107 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
1108                     unsigned long xfer_len)
1109 {
1110         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1111         struct iscsi_hdr *hdr;
1112         u64 rx_dma;
1113         int rx_buflen, outstanding;
1114
1115         if ((char *)desc == isert_conn->login_req_buf) {
1116                 rx_dma = isert_conn->login_req_dma;
1117                 rx_buflen = ISER_RX_LOGIN_SIZE;
1118                 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1119                          rx_dma, rx_buflen);
1120         } else {
1121                 rx_dma = desc->dma_addr;
1122                 rx_buflen = ISER_RX_PAYLOAD_SIZE;
1123                 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1124                          rx_dma, rx_buflen);
1125         }
1126
1127         ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1128
1129         hdr = &desc->iscsi_header;
1130         pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1131                  hdr->opcode, hdr->itt, hdr->flags,
1132                  (int)(xfer_len - ISER_HEADERS_LEN));
1133
1134         if ((char *)desc == isert_conn->login_req_buf)
1135                 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
1136                                    isert_conn);
1137         else
1138                 isert_rx_do_work(desc, isert_conn);
1139
1140         ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1141                                       DMA_FROM_DEVICE);
1142
1143         isert_conn->post_recv_buf_count--;
1144         pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1145                  isert_conn->post_recv_buf_count);
1146
1147         if ((char *)desc == isert_conn->login_req_buf)
1148                 return;
1149
1150         outstanding = isert_conn->post_recv_buf_count;
1151         if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
1152                 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
1153                                 ISERT_MIN_POSTED_RX);
1154                 err = isert_post_recv(isert_conn, count);
1155                 if (err) {
1156                         pr_err("isert_post_recv() count: %d failed, %d\n",
1157                                count, err);
1158                 }
1159         }
1160 }
1161
1162 static void
1163 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1164 {
1165         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1166         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1167
1168         pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");
1169
1170         if (wr->sge) {
1171                 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1172                 wr->sge = NULL;
1173         }
1174
1175         kfree(wr->send_wr);
1176         wr->send_wr = NULL;
1177
1178         kfree(isert_cmd->ib_sge);
1179         isert_cmd->ib_sge = NULL;
1180 }
1181
1182 static void
1183 isert_put_cmd(struct isert_cmd *isert_cmd)
1184 {
1185         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1186         struct isert_conn *isert_conn = isert_cmd->conn;
1187         struct iscsi_conn *conn;
1188
1189         pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
1190
1191         switch (cmd->iscsi_opcode) {
1192         case ISCSI_OP_SCSI_CMD:
1193                 conn = isert_conn->conn;
1194
1195                 spin_lock_bh(&conn->cmd_lock);
1196                 if (!list_empty(&cmd->i_conn_node))
1197                         list_del(&cmd->i_conn_node);
1198                 spin_unlock_bh(&conn->cmd_lock);
1199
1200                 if (cmd->data_direction == DMA_TO_DEVICE)
1201                         iscsit_stop_dataout_timer(cmd);
1202
1203                 isert_unmap_cmd(isert_cmd, isert_conn);
1204                 /*
1205                  * Fall-through
1206                  */
1207         case ISCSI_OP_SCSI_TMFUNC:
1208                 transport_generic_free_cmd(&cmd->se_cmd, 0);
1209                 break;
1210         case ISCSI_OP_REJECT:
1211         case ISCSI_OP_NOOP_OUT:
1212                 conn = isert_conn->conn;
1213
1214                 spin_lock_bh(&conn->cmd_lock);
1215                 if (!list_empty(&cmd->i_conn_node))
1216                         list_del(&cmd->i_conn_node);
1217                 spin_unlock_bh(&conn->cmd_lock);
1218
1219                 /*
1220                  * Handle special case for REJECT when iscsi_add_reject*() has
1221                  * overwritten the original iscsi_opcode assignment, and the
1222                  * associated cmd->se_cmd needs to be released.
1223                  */
1224                 if (cmd->se_cmd.se_tfo != NULL) {
1225                         transport_generic_free_cmd(&cmd->se_cmd, 0);
1226                         break;
1227                 }
1228                 /*
1229                  * Fall-through
1230                  */
1231         default:
1232                 isert_release_cmd(cmd);
1233                 break;
1234         }
1235 }
1236
1237 static void
1238 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1239 {
1240         if (tx_desc->dma_addr != 0) {
1241                 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1242                 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1243                                     ISER_HEADERS_LEN, DMA_TO_DEVICE);
1244                 tx_desc->dma_addr = 0;
1245         }
1246 }
1247
1248 static void
1249 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1250                      struct ib_device *ib_dev)
1251 {
1252         if (isert_cmd->sense_buf_dma != 0) {
1253                 pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
1254                 ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
1255                                     isert_cmd->sense_buf_len, DMA_TO_DEVICE);
1256                 isert_cmd->sense_buf_dma = 0;
1257         }
1258
1259         isert_unmap_tx_desc(tx_desc, ib_dev);
1260         isert_put_cmd(isert_cmd);
1261 }
1262
1263 static void
1264 isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1265                            struct isert_cmd *isert_cmd)
1266 {
1267         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1268         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1269         struct se_cmd *se_cmd = &cmd->se_cmd;
1270         struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;
1271
1272         iscsit_stop_dataout_timer(cmd);
1273
1274         if (wr->sge) {
1275                 pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
1276                 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1277                 wr->sge = NULL;
1278         }
1279
1280         if (isert_cmd->ib_sge) {
1281                 pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
1282                 kfree(isert_cmd->ib_sge);
1283                 isert_cmd->ib_sge = NULL;
1284         }
1285
1286         cmd->write_data_done = se_cmd->data_length;
1287
1288         pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
1289         spin_lock_bh(&cmd->istate_lock);
1290         cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1291         cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1292         spin_unlock_bh(&cmd->istate_lock);
1293
1294         target_execute_cmd(se_cmd);
1295 }
1296
1297 static void
1298 isert_do_control_comp(struct work_struct *work)
1299 {
1300         struct isert_cmd *isert_cmd = container_of(work,
1301                         struct isert_cmd, comp_work);
1302         struct isert_conn *isert_conn = isert_cmd->conn;
1303         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1304         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1305
1306         switch (cmd->i_state) {
1307         case ISTATE_SEND_TASKMGTRSP:
1308                 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1309
1310                 atomic_dec(&isert_conn->post_send_buf_count);
1311                 iscsit_tmr_post_handler(cmd, cmd->conn);
1312
1313                 cmd->i_state = ISTATE_SENT_STATUS;
1314                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
1315                 break;
1316         case ISTATE_SEND_REJECT:
1317                 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1318                 atomic_dec(&isert_conn->post_send_buf_count);
1319
1320                 cmd->i_state = ISTATE_SENT_STATUS;
1321                 complete(&cmd->reject_comp);
1322                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
1323         case ISTATE_SEND_LOGOUTRSP:
1324                 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
1325                 /*
1326                  * Call atomic_dec(&isert_conn->post_send_buf_count)
1327                  * from isert_free_conn()
1328                  */
1329                 isert_conn->logout_posted = true;
1330                 iscsit_logout_post_handler(cmd, cmd->conn);
1331                 break;
1332         default:
1333                 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
1334                 dump_stack();
1335                 break;
1336         }
1337 }
1338
1339 static void
1340 isert_response_completion(struct iser_tx_desc *tx_desc,
1341                           struct isert_cmd *isert_cmd,
1342                           struct isert_conn *isert_conn,
1343                           struct ib_device *ib_dev)
1344 {
1345         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1346
1347         if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
1348             cmd->i_state == ISTATE_SEND_LOGOUTRSP) {
1349                 isert_unmap_tx_desc(tx_desc, ib_dev);
1350
1351                 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1352                 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1353                 return;
1354         }
1355         atomic_dec(&isert_conn->post_send_buf_count);
1356
1357         cmd->i_state = ISTATE_SENT_STATUS;
1358         isert_completion_put(tx_desc, isert_cmd, ib_dev);
1359 }
1360
1361 static void
1362 isert_send_completion(struct iser_tx_desc *tx_desc,
1363                       struct isert_conn *isert_conn)
1364 {
1365         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1366         struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1367         struct isert_rdma_wr *wr;
1368
1369         if (!isert_cmd) {
1370                 atomic_dec(&isert_conn->post_send_buf_count);
1371                 isert_unmap_tx_desc(tx_desc, ib_dev);
1372                 return;
1373         }
1374         wr = &isert_cmd->rdma_wr;
1375
1376         switch (wr->iser_ib_op) {
1377         case ISER_IB_RECV:
1378                 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1379                 dump_stack();
1380                 break;
1381         case ISER_IB_SEND:
1382                 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1383                 isert_response_completion(tx_desc, isert_cmd,
1384                                           isert_conn, ib_dev);
1385                 break;
1386         case ISER_IB_RDMA_WRITE:
1387                 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1388                 dump_stack();
1389                 break;
1390         case ISER_IB_RDMA_READ:
1391                 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1392
1393                 atomic_dec(&isert_conn->post_send_buf_count);
1394                 isert_completion_rdma_read(tx_desc, isert_cmd);
1395                 break;
1396         default:
1397                 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
1398                 dump_stack();
1399                 break;
1400         }
1401 }
1402
1403 static void
1404 isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
1405 {
1406         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1407
1408         if (tx_desc) {
1409                 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1410
1411                 if (!isert_cmd)
1412                         isert_unmap_tx_desc(tx_desc, ib_dev);
1413                 else
1414                         isert_completion_put(tx_desc, isert_cmd, ib_dev);
1415         }
1416
1417         if (isert_conn->post_recv_buf_count == 0 &&
1418             atomic_read(&isert_conn->post_send_buf_count) == 0) {
1419                 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1420                 pr_debug("Calling wake_up from isert_cq_comp_err\n");
1421
1422                 isert_conn->state = ISER_CONN_TERMINATING;
1423                 wake_up(&isert_conn->conn_wait_comp_err);
1424         }
1425 }
1426
1427 static void
1428 isert_cq_tx_work(struct work_struct *work)
1429 {
1430         struct isert_cq_desc *cq_desc = container_of(work,
1431                                 struct isert_cq_desc, cq_tx_work);
1432         struct isert_device *device = cq_desc->device;
1433         int cq_index = cq_desc->cq_index;
1434         struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
1435         struct isert_conn *isert_conn;
1436         struct iser_tx_desc *tx_desc;
1437         struct ib_wc wc;
1438
1439         while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
1440                 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
1441                 isert_conn = wc.qp->qp_context;
1442
1443                 if (wc.status == IB_WC_SUCCESS) {
1444                         isert_send_completion(tx_desc, isert_conn);
1445                 } else {
1446                         pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1447                         pr_debug("TX wc.status: 0x%08x\n", wc.status);
1448                         atomic_dec(&isert_conn->post_send_buf_count);
1449                         isert_cq_comp_err(tx_desc, isert_conn);
1450                 }
1451         }
1452
1453         ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
1454 }
1455
1456 static void
1457 isert_cq_tx_callback(struct ib_cq *cq, void *context)
1458 {
1459         struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1460
1461         INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
1462         queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
1463 }
1464
1465 static void
1466 isert_cq_rx_work(struct work_struct *work)
1467 {
1468         struct isert_cq_desc *cq_desc = container_of(work,
1469                         struct isert_cq_desc, cq_rx_work);
1470         struct isert_device *device = cq_desc->device;
1471         int cq_index = cq_desc->cq_index;
1472         struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
1473         struct isert_conn *isert_conn;
1474         struct iser_rx_desc *rx_desc;
1475         struct ib_wc wc;
1476         unsigned long xfer_len;
1477
1478         while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
1479                 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
1480                 isert_conn = wc.qp->qp_context;
1481
1482                 if (wc.status == IB_WC_SUCCESS) {
1483                         xfer_len = (unsigned long)wc.byte_len;
1484                         isert_rx_completion(rx_desc, isert_conn, xfer_len);
1485                 } else {
1486                         pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1487                         if (wc.status != IB_WC_WR_FLUSH_ERR)
1488                                 pr_debug("RX wc.status: 0x%08x\n", wc.status);
1489
1490                         isert_conn->post_recv_buf_count--;
1491                         isert_cq_comp_err(NULL, isert_conn);
1492                 }
1493         }
1494
1495         ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
1496 }
1497
1498 static void
1499 isert_cq_rx_callback(struct ib_cq *cq, void *context)
1500 {
1501         struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1502
1503         INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
1504         queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
1505 }
1506
1507 static int
1508 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1509 {
1510         struct ib_send_wr *wr_failed;
1511         int ret;
1512
1513         atomic_inc(&isert_conn->post_send_buf_count);
1514
1515         ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
1516                            &wr_failed);
1517         if (ret) {
1518                 pr_err("ib_post_send failed with %d\n", ret);
1519                 atomic_dec(&isert_conn->post_send_buf_count);
1520                 return ret;
1521         }
1522         return ret;
1523 }
1524
1525 static int
1526 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1527 {
1528         struct isert_cmd *isert_cmd = container_of(cmd,
1529                                         struct isert_cmd, iscsi_cmd);
1530         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1531         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1532         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1533                                 &isert_cmd->tx_desc.iscsi_header;
1534
1535         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1536         iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1537         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1538         /*
1539          * Attach SENSE DATA payload to iSCSI Response PDU
1540          */
1541         if (cmd->se_cmd.sense_buffer &&
1542             ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1543             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1544                 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1545                 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1546                 u32 padding, sense_len;
1547
1548                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1549                                    cmd->sense_buffer);
1550                 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1551
1552                 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1553                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1554                 sense_len = cmd->se_cmd.scsi_sense_length + padding;
1555
1556                 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1557                                 (void *)cmd->sense_buffer, sense_len,
1558                                 DMA_TO_DEVICE);
1559
1560                 isert_cmd->sense_buf_len = sense_len;
1561                 tx_dsg->addr    = isert_cmd->sense_buf_dma;
1562                 tx_dsg->length  = sense_len;
1563                 tx_dsg->lkey    = isert_conn->conn_mr->lkey;
1564                 isert_cmd->tx_desc.num_sge = 2;
1565         }
1566
1567         isert_init_send_wr(isert_cmd, send_wr);
1568
1569         pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1570
1571         return isert_post_response(isert_conn, isert_cmd);
1572 }
1573
1574 static int
1575 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1576                 bool nopout_response)
1577 {
1578         struct isert_cmd *isert_cmd = container_of(cmd,
1579                                 struct isert_cmd, iscsi_cmd);
1580         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1581         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1582
1583         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1584         iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1585                                &isert_cmd->tx_desc.iscsi_header,
1586                                nopout_response);
1587         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1588         isert_init_send_wr(isert_cmd, send_wr);
1589
1590         pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1591
1592         return isert_post_response(isert_conn, isert_cmd);
1593 }
1594
1595 static int
1596 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1597 {
1598         struct isert_cmd *isert_cmd = container_of(cmd,
1599                                 struct isert_cmd, iscsi_cmd);
1600         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1601         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1602
1603         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1604         iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1605                                 &isert_cmd->tx_desc.iscsi_header);
1606         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1607         isert_init_send_wr(isert_cmd, send_wr);
1608
1609         pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1610
1611         return isert_post_response(isert_conn, isert_cmd);
1612 }
1613
1614 static int
1615 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1616 {
1617         struct isert_cmd *isert_cmd = container_of(cmd,
1618                                 struct isert_cmd, iscsi_cmd);
1619         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1620         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1621
1622         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1623         iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1624                                   &isert_cmd->tx_desc.iscsi_header);
1625         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1626         isert_init_send_wr(isert_cmd, send_wr);
1627
1628         pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1629
1630         return isert_post_response(isert_conn, isert_cmd);
1631 }
1632
1633 static int
1634 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1635 {
1636         struct isert_cmd *isert_cmd = container_of(cmd,
1637                                 struct isert_cmd, iscsi_cmd);
1638         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1639         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1640
1641         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1642         iscsit_build_reject(cmd, conn, (struct iscsi_reject *)
1643                                 &isert_cmd->tx_desc.iscsi_header);
1644         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1645         isert_init_send_wr(isert_cmd, send_wr);
1646
1647         pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1648
1649         return isert_post_response(isert_conn, isert_cmd);
1650 }
1651
1652 static int
1653 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1654                     struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
1655                     u32 data_left, u32 offset)
1656 {
1657         struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1658         struct scatterlist *sg_start, *tmp_sg;
1659         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1660         u32 sg_off, page_off;
1661         int i = 0, sg_nents;
1662
1663         sg_off = offset / PAGE_SIZE;
1664         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1665         sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
1666         page_off = offset % PAGE_SIZE;
1667
1668         send_wr->sg_list = ib_sge;
1669         send_wr->num_sge = sg_nents;
1670         send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
1671         /*
1672          * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
1673          */
1674         for_each_sg(sg_start, tmp_sg, sg_nents, i) {
1675                 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
1676                          (unsigned long long)tmp_sg->dma_address,
1677                          tmp_sg->length, page_off);
1678
1679                 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
1680                 ib_sge->length = min_t(u32, data_left,
1681                                 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
1682                 ib_sge->lkey = isert_conn->conn_mr->lkey;
1683
1684                 pr_debug("RDMA ib_sge: addr: 0x%16llx  length: %u\n",
1685                          ib_sge->addr, ib_sge->length);
1686                 page_off = 0;
1687                 data_left -= ib_sge->length;
1688                 ib_sge++;
1689                 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
1690         }
1691
1692         pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
1693                  send_wr->sg_list, send_wr->num_sge);
1694
1695         return sg_nents;
1696 }
1697
1698 static int
1699 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1700 {
1701         struct se_cmd *se_cmd = &cmd->se_cmd;
1702         struct isert_cmd *isert_cmd = container_of(cmd,
1703                                         struct isert_cmd, iscsi_cmd);
1704         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1705         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1706         struct ib_send_wr *wr_failed, *send_wr;
1707         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1708         struct ib_sge *ib_sge;
1709         struct scatterlist *sg;
1710         u32 offset = 0, data_len, data_left, rdma_write_max;
1711         int rc, ret = 0, count, sg_nents, i, ib_sge_cnt;
1712
1713         pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length);
1714
1715         sg = &se_cmd->t_data_sg[0];
1716         sg_nents = se_cmd->t_data_nents;
1717
1718         count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1719         if (unlikely(!count)) {
1720                 pr_err("Unable to map put_datain SGs\n");
1721                 return -EINVAL;
1722         }
1723         wr->sge = sg;
1724         wr->num_sge = sg_nents;
1725         pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n",
1726                  count, sg, sg_nents);
1727
1728         ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1729         if (!ib_sge) {
1730                 pr_warn("Unable to allocate datain ib_sge\n");
1731                 ret = -ENOMEM;
1732                 goto unmap_sg;
1733         }
1734         isert_cmd->ib_sge = ib_sge;
1735
1736         pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n",
1737                  ib_sge, se_cmd->t_data_nents);
1738
1739         wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1740         wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1741                                 GFP_KERNEL);
1742         if (!wr->send_wr) {
1743                 pr_err("Unable to allocate wr->send_wr\n");
1744                 ret = -ENOMEM;
1745                 goto unmap_sg;
1746         }
1747         pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1748                  wr->send_wr, wr->send_wr_num);
1749
1750         iscsit_increment_maxcmdsn(cmd, conn->sess);
1751         cmd->stat_sn = conn->stat_sn++;
1752
1753         wr->isert_cmd = isert_cmd;
1754         rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1755         data_left = se_cmd->data_length;
1756
1757         for (i = 0; i < wr->send_wr_num; i++) {
1758                 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1759                 data_len = min(data_left, rdma_write_max);
1760
1761                 send_wr->opcode = IB_WR_RDMA_WRITE;
1762                 send_wr->send_flags = 0;
1763                 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
1764                 send_wr->wr.rdma.rkey = isert_cmd->read_stag;
1765
1766                 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1767                                         send_wr, data_len, offset);
1768                 ib_sge += ib_sge_cnt;
1769
1770                 if (i + 1 == wr->send_wr_num)
1771                         send_wr->next = &isert_cmd->tx_desc.send_wr;
1772                 else
1773                         send_wr->next = &wr->send_wr[i + 1];
1774
1775                 offset += data_len;
1776                 data_left -= data_len;
1777         }
1778         /*
1779          * Build isert_conn->tx_desc for iSCSI response PDU and attach
1780          */
1781         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1782         iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
1783                              &isert_cmd->tx_desc.iscsi_header);
1784         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1785         isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr);
1786
1787         atomic_inc(&isert_conn->post_send_buf_count);
1788
1789         rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1790         if (rc) {
1791                 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
1792                 atomic_dec(&isert_conn->post_send_buf_count);
1793         }
1794         pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n");
1795         return 1;
1796
1797 unmap_sg:
1798         ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1799         return ret;
1800 }
1801
1802 static int
1803 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
1804 {
1805         struct se_cmd *se_cmd = &cmd->se_cmd;
1806         struct isert_cmd *isert_cmd = container_of(cmd,
1807                                         struct isert_cmd, iscsi_cmd);
1808         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1809         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1810         struct ib_send_wr *wr_failed, *send_wr;
1811         struct ib_sge *ib_sge;
1812         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1813         struct scatterlist *sg_start;
1814         u32 sg_off, sg_nents, page_off, va_offset = 0;
1815         u32 offset = 0, data_len, data_left, rdma_write_max;
1816         int rc, ret = 0, count, i, ib_sge_cnt;
1817
1818         pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n",
1819                  se_cmd->data_length, cmd->write_data_done);
1820
1821         sg_off = cmd->write_data_done / PAGE_SIZE;
1822         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1823         page_off = cmd->write_data_done % PAGE_SIZE;
1824
1825         pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n",
1826                  sg_off, sg_start, page_off);
1827
1828         data_left = se_cmd->data_length - cmd->write_data_done;
1829         sg_nents = se_cmd->t_data_nents - sg_off;
1830
1831         pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n",
1832                  data_left, sg_nents);
1833
1834         count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1835         if (unlikely(!count)) {
1836                 pr_err("Unable to map get_dataout SGs\n");
1837                 return -EINVAL;
1838         }
1839         wr->sge = sg_start;
1840         wr->num_sge = sg_nents;
1841         pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n",
1842                  count, sg_start, sg_nents);
1843
1844         ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1845         if (!ib_sge) {
1846                 pr_warn("Unable to allocate dataout ib_sge\n");
1847                 ret = -ENOMEM;
1848                 goto unmap_sg;
1849         }
1850         isert_cmd->ib_sge = ib_sge;
1851
1852         pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n",
1853                  ib_sge, sg_nents);
1854
1855         wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1856         wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1857                                 GFP_KERNEL);
1858         if (!wr->send_wr) {
1859                 pr_debug("Unable to allocate wr->send_wr\n");
1860                 ret = -ENOMEM;
1861                 goto unmap_sg;
1862         }
1863         pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1864                  wr->send_wr, wr->send_wr_num);
1865
1866         isert_cmd->tx_desc.isert_cmd = isert_cmd;
1867
1868         wr->iser_ib_op = ISER_IB_RDMA_READ;
1869         wr->isert_cmd = isert_cmd;
1870         rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1871         offset = cmd->write_data_done;
1872
1873         for (i = 0; i < wr->send_wr_num; i++) {
1874                 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1875                 data_len = min(data_left, rdma_write_max);
1876
1877                 send_wr->opcode = IB_WR_RDMA_READ;
1878                 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
1879                 send_wr->wr.rdma.rkey = isert_cmd->write_stag;
1880
1881                 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1882                                         send_wr, data_len, offset);
1883                 ib_sge += ib_sge_cnt;
1884
1885                 if (i + 1 == wr->send_wr_num)
1886                         send_wr->send_flags = IB_SEND_SIGNALED;
1887                 else
1888                         send_wr->next = &wr->send_wr[i + 1];
1889
1890                 offset += data_len;
1891                 va_offset += data_len;
1892                 data_left -= data_len;
1893         }
1894
1895         atomic_inc(&isert_conn->post_send_buf_count);
1896
1897         rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1898         if (rc) {
1899                 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
1900                 atomic_dec(&isert_conn->post_send_buf_count);
1901         }
1902         pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n");
1903         return 0;
1904
1905 unmap_sg:
1906         ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1907         return ret;
1908 }
1909
1910 static int
1911 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1912 {
1913         int ret;
1914
1915         switch (state) {
1916         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
1917                 ret = isert_put_nopin(cmd, conn, false);
1918                 break;
1919         default:
1920                 pr_err("Unknown immediate state: 0x%02x\n", state);
1921                 ret = -EINVAL;
1922                 break;
1923         }
1924
1925         return ret;
1926 }
1927
1928 static int
1929 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1930 {
1931         int ret;
1932
1933         switch (state) {
1934         case ISTATE_SEND_LOGOUTRSP:
1935                 ret = isert_put_logout_rsp(cmd, conn);
1936                 if (!ret) {
1937                         pr_debug("Returning iSER Logout -EAGAIN\n");
1938                         ret = -EAGAIN;
1939                 }
1940                 break;
1941         case ISTATE_SEND_NOPIN:
1942                 ret = isert_put_nopin(cmd, conn, true);
1943                 break;
1944         case ISTATE_SEND_TASKMGTRSP:
1945                 ret = isert_put_tm_rsp(cmd, conn);
1946                 break;
1947         case ISTATE_SEND_REJECT:
1948                 ret = isert_put_reject(cmd, conn);
1949                 break;
1950         case ISTATE_SEND_STATUS:
1951                 /*
1952                  * Special case for sending non GOOD SCSI status from TX thread
1953                  * context during pre se_cmd excecution failure.
1954                  */
1955                 ret = isert_put_response(conn, cmd);
1956                 break;
1957         default:
1958                 pr_err("Unknown response state: 0x%02x\n", state);
1959                 ret = -EINVAL;
1960                 break;
1961         }
1962
1963         return ret;
1964 }
1965
1966 static int
1967 isert_setup_np(struct iscsi_np *np,
1968                struct __kernel_sockaddr_storage *ksockaddr)
1969 {
1970         struct isert_np *isert_np;
1971         struct rdma_cm_id *isert_lid;
1972         struct sockaddr *sa;
1973         int ret;
1974
1975         isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
1976         if (!isert_np) {
1977                 pr_err("Unable to allocate struct isert_np\n");
1978                 return -ENOMEM;
1979         }
1980         init_waitqueue_head(&isert_np->np_accept_wq);
1981         mutex_init(&isert_np->np_accept_mutex);
1982         INIT_LIST_HEAD(&isert_np->np_accept_list);
1983         init_completion(&isert_np->np_login_comp);
1984
1985         sa = (struct sockaddr *)ksockaddr;
1986         pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa);
1987         /*
1988          * Setup the np->np_sockaddr from the passed sockaddr setup
1989          * in iscsi_target_configfs.c code..
1990          */
1991         memcpy(&np->np_sockaddr, ksockaddr,
1992                sizeof(struct __kernel_sockaddr_storage));
1993
1994         isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP,
1995                                 IB_QPT_RC);
1996         if (IS_ERR(isert_lid)) {
1997                 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
1998                        PTR_ERR(isert_lid));
1999                 ret = PTR_ERR(isert_lid);
2000                 goto out;
2001         }
2002
2003         ret = rdma_bind_addr(isert_lid, sa);
2004         if (ret) {
2005                 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret);
2006                 goto out_lid;
2007         }
2008
2009         ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG);
2010         if (ret) {
2011                 pr_err("rdma_listen() for isert_lid failed: %d\n", ret);
2012                 goto out_lid;
2013         }
2014
2015         isert_np->np_cm_id = isert_lid;
2016         np->np_context = isert_np;
2017         pr_debug("Setup isert_lid->context: %p\n", isert_lid->context);
2018
2019         return 0;
2020
2021 out_lid:
2022         rdma_destroy_id(isert_lid);
2023 out:
2024         kfree(isert_np);
2025         return ret;
2026 }
2027
2028 static int
2029 isert_check_accept_queue(struct isert_np *isert_np)
2030 {
2031         int empty;
2032
2033         mutex_lock(&isert_np->np_accept_mutex);
2034         empty = list_empty(&isert_np->np_accept_list);
2035         mutex_unlock(&isert_np->np_accept_mutex);
2036
2037         return empty;
2038 }
2039
2040 static int
2041 isert_rdma_accept(struct isert_conn *isert_conn)
2042 {
2043         struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2044         struct rdma_conn_param cp;
2045         int ret;
2046
2047         memset(&cp, 0, sizeof(struct rdma_conn_param));
2048         cp.responder_resources = isert_conn->responder_resources;
2049         cp.initiator_depth = isert_conn->initiator_depth;
2050         cp.retry_count = 7;
2051         cp.rnr_retry_count = 7;
2052
2053         pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
2054
2055         ret = rdma_accept(cm_id, &cp);
2056         if (ret) {
2057                 pr_err("rdma_accept() failed with: %d\n", ret);
2058                 return ret;
2059         }
2060
2061         pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
2062
2063         return 0;
2064 }
2065
2066 static int
2067 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2068 {
2069         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2070         int ret;
2071
2072         pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn);
2073
2074         ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
2075         if (ret)
2076                 return ret;
2077
2078         pr_debug("isert_get_login_rx processing login->req: %p\n", login->req);
2079         return 0;
2080 }
2081
2082 static void
2083 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2084                     struct isert_conn *isert_conn)
2085 {
2086         struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2087         struct rdma_route *cm_route = &cm_id->route;
2088         struct sockaddr_in *sock_in;
2089         struct sockaddr_in6 *sock_in6;
2090
2091         conn->login_family = np->np_sockaddr.ss_family;
2092
2093         if (np->np_sockaddr.ss_family == AF_INET6) {
2094                 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
2095                 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
2096                          &sock_in6->sin6_addr.in6_u);
2097                 conn->login_port = ntohs(sock_in6->sin6_port);
2098
2099                 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
2100                 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
2101                          &sock_in6->sin6_addr.in6_u);
2102                 conn->local_port = ntohs(sock_in6->sin6_port);
2103         } else {
2104                 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
2105                 sprintf(conn->login_ip, "%pI4",
2106                         &sock_in->sin_addr.s_addr);
2107                 conn->login_port = ntohs(sock_in->sin_port);
2108
2109                 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
2110                 sprintf(conn->local_ip, "%pI4",
2111                         &sock_in->sin_addr.s_addr);
2112                 conn->local_port = ntohs(sock_in->sin_port);
2113         }
2114 }
2115
2116 static int
2117 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2118 {
2119         struct isert_np *isert_np = (struct isert_np *)np->np_context;
2120         struct isert_conn *isert_conn;
2121         int max_accept = 0, ret;
2122
2123 accept_wait:
2124         ret = wait_event_interruptible(isert_np->np_accept_wq,
2125                         !isert_check_accept_queue(isert_np) ||
2126                         np->np_thread_state == ISCSI_NP_THREAD_RESET);
2127         if (max_accept > 5)
2128                 return -ENODEV;
2129
2130         spin_lock_bh(&np->np_thread_lock);
2131         if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
2132                 spin_unlock_bh(&np->np_thread_lock);
2133                 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n");
2134                 return -ENODEV;
2135         }
2136         spin_unlock_bh(&np->np_thread_lock);
2137
2138         mutex_lock(&isert_np->np_accept_mutex);
2139         if (list_empty(&isert_np->np_accept_list)) {
2140                 mutex_unlock(&isert_np->np_accept_mutex);
2141                 max_accept++;
2142                 goto accept_wait;
2143         }
2144         isert_conn = list_first_entry(&isert_np->np_accept_list,
2145                         struct isert_conn, conn_accept_node);
2146         list_del_init(&isert_conn->conn_accept_node);
2147         mutex_unlock(&isert_np->np_accept_mutex);
2148
2149         conn->context = isert_conn;
2150         isert_conn->conn = conn;
2151         max_accept = 0;
2152
2153         ret = isert_rdma_post_recvl(isert_conn);
2154         if (ret)
2155                 return ret;
2156
2157         ret = isert_rdma_accept(isert_conn);
2158         if (ret)
2159                 return ret;
2160
2161         isert_set_conn_info(np, conn, isert_conn);
2162
2163         pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn);
2164         return 0;
2165 }
2166
2167 static void
2168 isert_free_np(struct iscsi_np *np)
2169 {
2170         struct isert_np *isert_np = (struct isert_np *)np->np_context;
2171
2172         rdma_destroy_id(isert_np->np_cm_id);
2173
2174         np->np_context = NULL;
2175         kfree(isert_np);
2176 }
2177
2178 static void isert_free_conn(struct iscsi_conn *conn)
2179 {
2180         struct isert_conn *isert_conn = conn->context;
2181
2182         pr_debug("isert_free_conn: Starting \n");
2183         /*
2184          * Decrement post_send_buf_count for special case when called
2185          * from isert_do_control_comp() -> iscsit_logout_post_handler()
2186          */
2187         if (isert_conn->logout_posted)
2188                 atomic_dec(&isert_conn->post_send_buf_count);
2189
2190         if (isert_conn->conn_cm_id)
2191                 rdma_disconnect(isert_conn->conn_cm_id);
2192         /*
2193          * Only wait for conn_wait_comp_err if the isert_conn made it
2194          * into full feature phase..
2195          */
2196         if (isert_conn->state > ISER_CONN_INIT) {
2197                 pr_debug("isert_free_conn: Before wait_event comp_err %d\n",
2198                          isert_conn->state);
2199                 wait_event(isert_conn->conn_wait_comp_err,
2200                            isert_conn->state == ISER_CONN_TERMINATING);
2201                 pr_debug("isert_free_conn: After wait_event #1 >>>>>>>>>>>>\n");
2202         }
2203
2204         pr_debug("isert_free_conn: wait_event conn_wait %d\n", isert_conn->state);
2205         wait_event(isert_conn->conn_wait, isert_conn->state == ISER_CONN_DOWN);
2206         pr_debug("isert_free_conn: After wait_event #2 >>>>>>>>>>>>>>>>>>>>\n");
2207
2208         isert_put_conn(isert_conn);
2209 }
2210
2211 static struct iscsit_transport iser_target_transport = {
2212         .name                   = "IB/iSER",
2213         .transport_type         = ISCSI_INFINIBAND,
2214         .owner                  = THIS_MODULE,
2215         .iscsit_setup_np        = isert_setup_np,
2216         .iscsit_accept_np       = isert_accept_np,
2217         .iscsit_free_np         = isert_free_np,
2218         .iscsit_free_conn       = isert_free_conn,
2219         .iscsit_alloc_cmd       = isert_alloc_cmd,
2220         .iscsit_get_login_rx    = isert_get_login_rx,
2221         .iscsit_put_login_tx    = isert_put_login_tx,
2222         .iscsit_immediate_queue = isert_immediate_queue,
2223         .iscsit_response_queue  = isert_response_queue,
2224         .iscsit_get_dataout     = isert_get_dataout,
2225         .iscsit_queue_data_in   = isert_put_datain,
2226         .iscsit_queue_status    = isert_put_response,
2227 };
2228
2229 static int __init isert_init(void)
2230 {
2231         int ret;
2232
2233         isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0);
2234         if (!isert_rx_wq) {
2235                 pr_err("Unable to allocate isert_rx_wq\n");
2236                 return -ENOMEM;
2237         }
2238
2239         isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
2240         if (!isert_comp_wq) {
2241                 pr_err("Unable to allocate isert_comp_wq\n");
2242                 ret = -ENOMEM;
2243                 goto destroy_rx_wq;
2244         }
2245
2246         isert_cmd_cache = kmem_cache_create("isert_cmd_cache",
2247                         sizeof(struct isert_cmd), __alignof__(struct isert_cmd),
2248                         0, NULL);
2249         if (!isert_cmd_cache) {
2250                 pr_err("Unable to create isert_cmd_cache\n");
2251                 ret = -ENOMEM;
2252                 goto destroy_tx_cq;
2253         }
2254
2255         iscsit_register_transport(&iser_target_transport);
2256         pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
2257         return 0;
2258
2259 destroy_tx_cq:
2260         destroy_workqueue(isert_comp_wq);
2261 destroy_rx_wq:
2262         destroy_workqueue(isert_rx_wq);
2263         return ret;
2264 }
2265
2266 static void __exit isert_exit(void)
2267 {
2268         kmem_cache_destroy(isert_cmd_cache);
2269         destroy_workqueue(isert_comp_wq);
2270         destroy_workqueue(isert_rx_wq);
2271         iscsit_unregister_transport(&iser_target_transport);
2272         pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
2273 }
2274
2275 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2276 MODULE_VERSION("0.1");
2277 MODULE_AUTHOR("nab@Linux-iSCSI.org");
2278 MODULE_LICENSE("GPL");
2279
2280 module_init(isert_init);
2281 module_exit(isert_exit);