drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2011 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45
  46 static unsigned int be_iopoll_budget = 10;
  47 static unsigned int be_max_phys_size = 64;
  48 static unsigned int enable_msix = 1;
  49 static unsigned int gcrashmode = 0;
  50 static unsigned int num_hba = 0;
  51
  52 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  53 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  54 MODULE_VERSION(BUILD_STR);
  55 MODULE_AUTHOR("Emulex Corporation");
  56 MODULE_LICENSE("GPL");
  57 module_param(be_iopoll_budget, int, 0);
  58 module_param(enable_msix, int, 0);
  59 module_param(be_max_phys_size, uint, S_IRUGO);
  60 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  61                                    "contiguous memory that can be allocated."
  62                                    "Range is 16 - 128");
  63
  64 static int beiscsi_slave_configure(struct scsi_device *sdev)
  65 {
  66         blk_queue_max_segment_size(sdev->request_queue, 65536);
  67         return 0;
  68 }
  69
  70 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
  71 {
  72         struct iscsi_cls_session *cls_session;
  73         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
  74         struct beiscsi_io_task *aborted_io_task;
  75         struct iscsi_conn *conn;
  76         struct beiscsi_conn *beiscsi_conn;
  77         struct beiscsi_hba *phba;
  78         struct iscsi_session *session;
  79         struct invalidate_command_table *inv_tbl;
  80         struct be_dma_mem nonemb_cmd;
  81         unsigned int cid, tag, num_invalidate;
  82
  83         cls_session = starget_to_session(scsi_target(sc->device));
  84         session = cls_session->dd_data;
  85
  86         spin_lock_bh(&session->lock);
  87         if (!aborted_task || !aborted_task->sc) {
  88                 /* we raced */
  89                 spin_unlock_bh(&session->lock);
  90                 return SUCCESS;
  91         }
  92
  93         aborted_io_task = aborted_task->dd_data;
  94         if (!aborted_io_task->scsi_cmnd) {
  95                 /* raced or invalid command */
  96                 spin_unlock_bh(&session->lock);
  97                 return SUCCESS;
  98         }
  99         spin_unlock_bh(&session->lock);
 100         conn = aborted_task->conn;
 101         beiscsi_conn = conn->dd_data;
 102         phba = beiscsi_conn->phba;
 103
 104         /* invalidate iocb */
 105         cid = beiscsi_conn->beiscsi_conn_cid;
 106         inv_tbl = phba->inv_tbl;
 107         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 108         inv_tbl->cid = cid;
 109         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 110         num_invalidate = 1;
 111         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 112                                 sizeof(struct invalidate_commands_params_in),
 113                                 &nonemb_cmd.dma);
 114         if (nonemb_cmd.va == NULL) {
 115                 SE_DEBUG(DBG_LVL_1,
 116                          "Failed to allocate memory for"
 117                          "mgmt_invalidate_icds\n");
 118                 return FAILED;
 119         }
 120         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 121
 122         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 123                                    cid, &nonemb_cmd);
 124         if (!tag) {
 125                 shost_printk(KERN_WARNING, phba->shost,
 126                              "mgmt_invalidate_icds could not be"
 127                              " submitted\n");
 128                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 129                                     nonemb_cmd.va, nonemb_cmd.dma);
 130
 131                 return FAILED;
 132         } else {
 133                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 134                                          phba->ctrl.mcc_numtag[tag]);
 135                 free_mcc_tag(&phba->ctrl, tag);
 136         }
 137         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 138                             nonemb_cmd.va, nonemb_cmd.dma);
 139         return iscsi_eh_abort(sc);
 140 }
 141
 142 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 143 {
 144         struct iscsi_task *abrt_task;
 145         struct beiscsi_io_task *abrt_io_task;
 146         struct iscsi_conn *conn;
 147         struct beiscsi_conn *beiscsi_conn;
 148         struct beiscsi_hba *phba;
 149         struct iscsi_session *session;
 150         struct iscsi_cls_session *cls_session;
 151         struct invalidate_command_table *inv_tbl;
 152         struct be_dma_mem nonemb_cmd;
 153         unsigned int cid, tag, i, num_invalidate;
 154
 155         /* invalidate iocbs */
 156         cls_session = starget_to_session(scsi_target(sc->device));
 157         session = cls_session->dd_data;
 158         spin_lock_bh(&session->lock);
 159         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 160                 spin_unlock_bh(&session->lock);
 161                 return FAILED;
 162         }
 163         conn = session->leadconn;
 164         beiscsi_conn = conn->dd_data;
 165         phba = beiscsi_conn->phba;
 166         cid = beiscsi_conn->beiscsi_conn_cid;
 167         inv_tbl = phba->inv_tbl;
 168         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 169         num_invalidate = 0;
 170         for (i = 0; i < conn->session->cmds_max; i++) {
 171                 abrt_task = conn->session->cmds[i];
 172                 abrt_io_task = abrt_task->dd_data;
 173                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 174                         continue;
 175
 176                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 177                         continue;
 178
 179                 inv_tbl->cid = cid;
 180                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 181                 num_invalidate++;
 182                 inv_tbl++;
 183         }
 184         spin_unlock_bh(&session->lock);
 185         inv_tbl = phba->inv_tbl;
 186
 187         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 188                                 sizeof(struct invalidate_commands_params_in),
 189                                 &nonemb_cmd.dma);
 190         if (nonemb_cmd.va == NULL) {
 191                 SE_DEBUG(DBG_LVL_1,
 192                          "Failed to allocate memory for"
 193                          "mgmt_invalidate_icds\n");
 194                 return FAILED;
 195         }
 196         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 197         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 198         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 199                                    cid, &nonemb_cmd);
 200         if (!tag) {
 201                 shost_printk(KERN_WARNING, phba->shost,
 202                              "mgmt_invalidate_icds could not be"
 203                              " submitted\n");
 204                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 205                                     nonemb_cmd.va, nonemb_cmd.dma);
 206                 return FAILED;
 207         } else {
 208                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 209                                          phba->ctrl.mcc_numtag[tag]);
 210                 free_mcc_tag(&phba->ctrl, tag);
 211         }
 212         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 213                             nonemb_cmd.va, nonemb_cmd.dma);
 214         return iscsi_eh_device_reset(sc);
 215 }
 216
 217 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 218 {
 219         struct beiscsi_hba *phba = data;
 220         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 221         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 222         char *str = buf;
 223         int rc;
 224
 225         switch (type) {
 226         case ISCSI_BOOT_TGT_NAME:
 227                 rc = sprintf(buf, "%.*s\n",
 228                             (int)strlen(boot_sess->target_name),
 229                             (char *)&boot_sess->target_name);
 230                 break;
 231         case ISCSI_BOOT_TGT_IP_ADDR:
 232                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 233                         rc = sprintf(buf, "%pI4\n",
 234                                 (char *)&boot_conn->dest_ipaddr.addr);
 235                 else
 236                         rc = sprintf(str, "%pI6\n",
 237                                 (char *)&boot_conn->dest_ipaddr.addr);
 238                 break;
 239         case ISCSI_BOOT_TGT_PORT:
 240                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 241                 break;
 242
 243         case ISCSI_BOOT_TGT_CHAP_NAME:
 244                 rc = sprintf(str,  "%.*s\n",
 245                              boot_conn->negotiated_login_options.auth_data.chap.
 246                              target_chap_name_length,
 247                              (char *)&boot_conn->negotiated_login_options.
 248                              auth_data.chap.target_chap_name);
 249                 break;
 250         case ISCSI_BOOT_TGT_CHAP_SECRET:
 251                 rc = sprintf(str,  "%.*s\n",
 252                              boot_conn->negotiated_login_options.auth_data.chap.
 253                              target_secret_length,
 254                              (char *)&boot_conn->negotiated_login_options.
 255                              auth_data.chap.target_secret);
 256                 break;
 257         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 258                 rc = sprintf(str,  "%.*s\n",
 259                              boot_conn->negotiated_login_options.auth_data.chap.
 260                              intr_chap_name_length,
 261                              (char *)&boot_conn->negotiated_login_options.
 262                              auth_data.chap.intr_chap_name);
 263                 break;
 264         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 265                 rc = sprintf(str,  "%.*s\n",
 266                              boot_conn->negotiated_login_options.auth_data.chap.
 267                              intr_secret_length,
 268                              (char *)&boot_conn->negotiated_login_options.
 269                              auth_data.chap.intr_secret);
 270                 break;
 271         case ISCSI_BOOT_TGT_FLAGS:
 272                 rc = sprintf(str, "2\n");
 273                 break;
 274         case ISCSI_BOOT_TGT_NIC_ASSOC:
 275                 rc = sprintf(str, "0\n");
 276                 break;
 277         default:
 278                 rc = -ENOSYS;
 279                 break;
 280         }
 281         return rc;
 282 }
 283
 284 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 285 {
 286         struct beiscsi_hba *phba = data;
 287         char *str = buf;
 288         int rc;
 289
 290         switch (type) {
 291         case ISCSI_BOOT_INI_INITIATOR_NAME:
 292                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 293                 break;
 294         default:
 295                 rc = -ENOSYS;
 296                 break;
 297         }
 298         return rc;
 299 }
 300
 301 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 302 {
 303         struct beiscsi_hba *phba = data;
 304         char *str = buf;
 305         int rc;
 306
 307         switch (type) {
 308         case ISCSI_BOOT_ETH_FLAGS:
 309                 rc = sprintf(str, "2\n");
 310                 break;
 311         case ISCSI_BOOT_ETH_INDEX:
 312                 rc = sprintf(str, "0\n");
 313                 break;
 314         case ISCSI_BOOT_ETH_MAC:
 315                 rc  = beiscsi_get_macaddr(str, phba);
 316                 break;
 317         default:
 318                 rc = -ENOSYS;
 319                 break;
 320         }
 321         return rc;
 322 }
 323
 324
 325 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 326 {
 327         umode_t rc;
 328
 329         switch (type) {
 330         case ISCSI_BOOT_TGT_NAME:
 331         case ISCSI_BOOT_TGT_IP_ADDR:
 332         case ISCSI_BOOT_TGT_PORT:
 333         case ISCSI_BOOT_TGT_CHAP_NAME:
 334         case ISCSI_BOOT_TGT_CHAP_SECRET:
 335         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 336         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 337         case ISCSI_BOOT_TGT_NIC_ASSOC:
 338         case ISCSI_BOOT_TGT_FLAGS:
 339                 rc = S_IRUGO;
 340                 break;
 341         default:
 342                 rc = 0;
 343                 break;
 344         }
 345         return rc;
 346 }
 347
 348 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 349 {
 350         umode_t rc;
 351
 352         switch (type) {
 353         case ISCSI_BOOT_INI_INITIATOR_NAME:
 354                 rc = S_IRUGO;
 355                 break;
 356         default:
 357                 rc = 0;
 358                 break;
 359         }
 360         return rc;
 361 }
 362
 363
 364 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 365 {
 366         umode_t rc;
 367
 368         switch (type) {
 369         case ISCSI_BOOT_ETH_FLAGS:
 370         case ISCSI_BOOT_ETH_MAC:
 371         case ISCSI_BOOT_ETH_INDEX:
 372                 rc = S_IRUGO;
 373                 break;
 374         default:
 375                 rc = 0;
 376                 break;
 377         }
 378         return rc;
 379 }
 380
 381 /*------------------- PCI Driver operations and data ----------------- */
 382 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 383         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 384         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 385         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 386         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 387         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 388         { 0 }
 389 };
 390 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 391
 392 static struct scsi_host_template beiscsi_sht = {
 393         .module = THIS_MODULE,
 394         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 395         .proc_name = DRV_NAME,
 396         .queuecommand = iscsi_queuecommand,
 397         .change_queue_depth = iscsi_change_queue_depth,
 398         .slave_configure = beiscsi_slave_configure,
 399         .target_alloc = iscsi_target_alloc,
 400         .eh_abort_handler = beiscsi_eh_abort,
 401         .eh_device_reset_handler = beiscsi_eh_device_reset,
 402         .eh_target_reset_handler = iscsi_eh_session_reset,
 403         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 404         .can_queue = BE2_IO_DEPTH,
 405         .this_id = -1,
 406         .max_sectors = BEISCSI_MAX_SECTORS,
 407         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 408         .use_clustering = ENABLE_CLUSTERING,
 409         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 410
 411 };
 412
 413 static struct scsi_transport_template *beiscsi_scsi_transport;
 414
 415 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 416 {
 417         struct beiscsi_hba *phba;
 418         struct Scsi_Host *shost;
 419
 420         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 421         if (!shost) {
 422                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
 423                         "iscsi_host_alloc failed\n");
 424                 return NULL;
 425         }
 426         shost->dma_boundary = pcidev->dma_mask;
 427         shost->max_id = BE2_MAX_SESSIONS;
 428         shost->max_channel = 0;
 429         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 430         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 431         shost->transportt = beiscsi_scsi_transport;
 432         phba = iscsi_host_priv(shost);
 433         memset(phba, 0, sizeof(*phba));
 434         phba->shost = shost;
 435         phba->pcidev = pci_dev_get(pcidev);
 436         pci_set_drvdata(pcidev, phba);
 437         phba->interface_handle = 0xFFFFFFFF;
 438
 439         if (iscsi_host_add(shost, &phba->pcidev->dev))
 440                 goto free_devices;
 441
 442         return phba;
 443
 444 free_devices:
 445         pci_dev_put(phba->pcidev);
 446         iscsi_host_free(phba->shost);
 447         return NULL;
 448 }
 449
 450 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 451 {
 452         if (phba->csr_va) {
 453                 iounmap(phba->csr_va);
 454                 phba->csr_va = NULL;
 455         }
 456         if (phba->db_va) {
 457                 iounmap(phba->db_va);
 458                 phba->db_va = NULL;
 459         }
 460         if (phba->pci_va) {
 461                 iounmap(phba->pci_va);
 462                 phba->pci_va = NULL;
 463         }
 464 }
 465
 466 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 467                                 struct pci_dev *pcidev)
 468 {
 469         u8 __iomem *addr;
 470         int pcicfg_reg;
 471
 472         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 473                                pci_resource_len(pcidev, 2));
 474         if (addr == NULL)
 475                 return -ENOMEM;
 476         phba->ctrl.csr = addr;
 477         phba->csr_va = addr;
 478         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 479
 480         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 481         if (addr == NULL)
 482                 goto pci_map_err;
 483         phba->ctrl.db = addr;
 484         phba->db_va = addr;
 485         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 486
 487         if (phba->generation == BE_GEN2)
 488                 pcicfg_reg = 1;
 489         else
 490                 pcicfg_reg = 0;
 491
 492         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 493                                pci_resource_len(pcidev, pcicfg_reg));
 494
 495         if (addr == NULL)
 496                 goto pci_map_err;
 497         phba->ctrl.pcicfg = addr;
 498         phba->pci_va = addr;
 499         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 500         return 0;
 501
 502 pci_map_err:
 503         beiscsi_unmap_pci_function(phba);
 504         return -ENOMEM;
 505 }
 506
 507 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 508 {
 509         int ret;
 510
 511         ret = pci_enable_device(pcidev);
 512         if (ret) {
 513                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 514                         "failed. Returning -ENODEV\n");
 515                 return ret;
 516         }
 517
 518         pci_set_master(pcidev);
 519         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 520                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 521                 if (ret) {
 522                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 523                         pci_disable_device(pcidev);
 524                         return ret;
 525                 }
 526         }
 527         return 0;
 528 }
 529
 530 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 531 {
 532         struct be_ctrl_info *ctrl = &phba->ctrl;
 533         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 534         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 535         int status = 0;
 536
 537         ctrl->pdev = pdev;
 538         status = beiscsi_map_pci_bars(phba, pdev);
 539         if (status)
 540                 return status;
 541         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 542         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 543                                                   mbox_mem_alloc->size,
 544                                                   &mbox_mem_alloc->dma);
 545         if (!mbox_mem_alloc->va) {
 546                 beiscsi_unmap_pci_function(phba);
 547                 return -ENOMEM;
 548         }
 549
 550         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 551         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 552         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 553         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 554         spin_lock_init(&ctrl->mbox_lock);
 555         spin_lock_init(&phba->ctrl.mcc_lock);
 556         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 557
 558         return status;
 559 }
 560
 561 static void beiscsi_get_params(struct beiscsi_hba *phba)
 562 {
 563         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 564                                     - (phba->fw_config.iscsi_cid_count
 565                                     + BE2_TMFS
 566                                     + BE2_NOPOUT_REQ));
 567         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 568         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 569         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 570         phba->params.num_sge_per_io = BE2_SGE;
 571         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 572         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 573         phba->params.eq_timer = 64;
 574         phba->params.num_eq_entries =
 575             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 576                                     + BE2_TMFS) / 512) + 1) * 512;
 577         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 578                                 ? 1024 : phba->params.num_eq_entries;
 579         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
 580                              phba->params.num_eq_entries);
 581         phba->params.num_cq_entries =
 582             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 583                                     + BE2_TMFS) / 512) + 1) * 512;
 584         phba->params.wrbs_per_cxn = 256;
 585 }
 586
 587 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 588                            unsigned int id, unsigned int clr_interrupt,
 589                            unsigned int num_processed,
 590                            unsigned char rearm, unsigned char event)
 591 {
 592         u32 val = 0;
 593         val |= id & DB_EQ_RING_ID_MASK;
 594         if (rearm)
 595                 val |= 1 << DB_EQ_REARM_SHIFT;
 596         if (clr_interrupt)
 597                 val |= 1 << DB_EQ_CLR_SHIFT;
 598         if (event)
 599                 val |= 1 << DB_EQ_EVNT_SHIFT;
 600         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 601         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 602 }
 603
 604 /**
 605  * be_isr_mcc - The isr routine of the driver.
 606  * @irq: Not used
 607  * @dev_id: Pointer to host adapter structure
 608  */
 609 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 610 {
 611         struct beiscsi_hba *phba;
 612         struct be_eq_entry *eqe = NULL;
 613         struct be_queue_info *eq;
 614         struct be_queue_info *mcc;
 615         unsigned int num_eq_processed;
 616         struct be_eq_obj *pbe_eq;
 617         unsigned long flags;
 618
 619         pbe_eq = dev_id;
 620         eq = &pbe_eq->q;
 621         phba =  pbe_eq->phba;
 622         mcc = &phba->ctrl.mcc_obj.cq;
 623         eqe = queue_tail_node(eq);
 624         if (!eqe)
 625                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 626
 627         num_eq_processed = 0;
 628
 629         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 630                                 & EQE_VALID_MASK) {
 631                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 632                      resource_id) / 32] &
 633                      EQE_RESID_MASK) >> 16) == mcc->id) {
 634                         spin_lock_irqsave(&phba->isr_lock, flags);
 635                         phba->todo_mcc_cq = 1;
 636                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 637                 }
 638                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 639                 queue_tail_inc(eq);
 640                 eqe = queue_tail_node(eq);
 641                 num_eq_processed++;
 642         }
 643         if (phba->todo_mcc_cq)
 644                 queue_work(phba->wq, &phba->work_cqs);
 645         if (num_eq_processed)
 646                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 647
 648         return IRQ_HANDLED;
 649 }
 650
 651 /**
 652  * be_isr_msix - The isr routine of the driver.
 653  * @irq: Not used
 654  * @dev_id: Pointer to host adapter structure
 655  */
 656 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 657 {
 658         struct beiscsi_hba *phba;
 659         struct be_eq_entry *eqe = NULL;
 660         struct be_queue_info *eq;
 661         struct be_queue_info *cq;
 662         unsigned int num_eq_processed;
 663         struct be_eq_obj *pbe_eq;
 664         unsigned long flags;
 665
 666         pbe_eq = dev_id;
 667         eq = &pbe_eq->q;
 668         cq = pbe_eq->cq;
 669         eqe = queue_tail_node(eq);
 670         if (!eqe)
 671                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 672
 673         phba = pbe_eq->phba;
 674         num_eq_processed = 0;
 675         if (blk_iopoll_enabled) {
 676                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 677                                         & EQE_VALID_MASK) {
 678                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 679                                 blk_iopoll_sched(&pbe_eq->iopoll);
 680
 681                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 682                         queue_tail_inc(eq);
 683                         eqe = queue_tail_node(eq);
 684                         num_eq_processed++;
 685                 }
 686                 if (num_eq_processed)
 687                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 688
 689                 return IRQ_HANDLED;
 690         } else {
 691                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 692                                                 & EQE_VALID_MASK) {
 693                         spin_lock_irqsave(&phba->isr_lock, flags);
 694                         phba->todo_cq = 1;
 695                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 696                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 697                         queue_tail_inc(eq);
 698                         eqe = queue_tail_node(eq);
 699                         num_eq_processed++;
 700                 }
 701                 if (phba->todo_cq)
 702                         queue_work(phba->wq, &phba->work_cqs);
 703
 704                 if (num_eq_processed)
 705                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 706
 707                 return IRQ_HANDLED;
 708         }
 709 }
 710
 711 /**
 712  * be_isr - The isr routine of the driver.
 713  * @irq: Not used
 714  * @dev_id: Pointer to host adapter structure
 715  */
 716 static irqreturn_t be_isr(int irq, void *dev_id)
 717 {
 718         struct beiscsi_hba *phba;
 719         struct hwi_controller *phwi_ctrlr;
 720         struct hwi_context_memory *phwi_context;
 721         struct be_eq_entry *eqe = NULL;
 722         struct be_queue_info *eq;
 723         struct be_queue_info *cq;
 724         struct be_queue_info *mcc;
 725         unsigned long flags, index;
 726         unsigned int num_mcceq_processed, num_ioeq_processed;
 727         struct be_ctrl_info *ctrl;
 728         struct be_eq_obj *pbe_eq;
 729         int isr;
 730
 731         phba = dev_id;
 732         ctrl = &phba->ctrl;
 733         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 734                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 735         if (!isr)
 736                 return IRQ_NONE;
 737
 738         phwi_ctrlr = phba->phwi_ctrlr;
 739         phwi_context = phwi_ctrlr->phwi_ctxt;
 740         pbe_eq = &phwi_context->be_eq[0];
 741
 742         eq = &phwi_context->be_eq[0].q;
 743         mcc = &phba->ctrl.mcc_obj.cq;
 744         index = 0;
 745         eqe = queue_tail_node(eq);
 746         if (!eqe)
 747                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 748
 749         num_ioeq_processed = 0;
 750         num_mcceq_processed = 0;
 751         if (blk_iopoll_enabled) {
 752                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 753                                         & EQE_VALID_MASK) {
 754                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 755                              resource_id) / 32] &
 756                              EQE_RESID_MASK) >> 16) == mcc->id) {
 757                                 spin_lock_irqsave(&phba->isr_lock, flags);
 758                                 phba->todo_mcc_cq = 1;
 759                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 760                                 num_mcceq_processed++;
 761                         } else {
 762                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 763                                         blk_iopoll_sched(&pbe_eq->iopoll);
 764                                 num_ioeq_processed++;
 765                         }
 766                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 767                         queue_tail_inc(eq);
 768                         eqe = queue_tail_node(eq);
 769                 }
 770                 if (num_ioeq_processed || num_mcceq_processed) {
 771                         if (phba->todo_mcc_cq)
 772                                 queue_work(phba->wq, &phba->work_cqs);
 773
 774                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 775                                 hwi_ring_eq_db(phba, eq->id, 0,
 776                                               (num_ioeq_processed +
 777                                                num_mcceq_processed) , 1, 1);
 778                         else
 779                                 hwi_ring_eq_db(phba, eq->id, 0,
 780                                                (num_ioeq_processed +
 781                                                 num_mcceq_processed), 0, 1);
 782
 783                         return IRQ_HANDLED;
 784                 } else
 785                         return IRQ_NONE;
 786         } else {
 787                 cq = &phwi_context->be_cq[0];
 788                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 789                                                 & EQE_VALID_MASK) {
 790
 791                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 792                              resource_id) / 32] &
 793                              EQE_RESID_MASK) >> 16) != cq->id) {
 794                                 spin_lock_irqsave(&phba->isr_lock, flags);
 795                                 phba->todo_mcc_cq = 1;
 796                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 797                         } else {
 798                                 spin_lock_irqsave(&phba->isr_lock, flags);
 799                                 phba->todo_cq = 1;
 800                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 801                         }
 802                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 803                         queue_tail_inc(eq);
 804                         eqe = queue_tail_node(eq);
 805                         num_ioeq_processed++;
 806                 }
 807                 if (phba->todo_cq || phba->todo_mcc_cq)
 808                         queue_work(phba->wq, &phba->work_cqs);
 809
 810                 if (num_ioeq_processed) {
 811                         hwi_ring_eq_db(phba, eq->id, 0,
 812                                        num_ioeq_processed, 1, 1);
 813                         return IRQ_HANDLED;
 814                 } else
 815                         return IRQ_NONE;
 816         }
 817 }
 818
 819 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 820 {
 821         struct pci_dev *pcidev = phba->pcidev;
 822         struct hwi_controller *phwi_ctrlr;
 823         struct hwi_context_memory *phwi_context;
 824         int ret, msix_vec, i, j;
 825
 826         phwi_ctrlr = phba->phwi_ctrlr;
 827         phwi_context = phwi_ctrlr->phwi_ctxt;
 828
 829         if (phba->msix_enabled) {
 830                 for (i = 0; i < phba->num_cpus; i++) {
 831                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 832                                                     GFP_KERNEL);
 833                         if (!phba->msi_name[i]) {
 834                                 ret = -ENOMEM;
 835                                 goto free_msix_irqs;
 836                         }
 837
 838                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 839                                 phba->shost->host_no, i);
 840                         msix_vec = phba->msix_entries[i].vector;
 841                         ret = request_irq(msix_vec, be_isr_msix, 0,
 842                                           phba->msi_name[i],
 843                                           &phwi_context->be_eq[i]);
 844                         if (ret) {
 845                                 shost_printk(KERN_ERR, phba->shost,
 846                                              "beiscsi_init_irqs-Failed to"
 847                                              "register msix for i = %d\n", i);
 848                                 kfree(phba->msi_name[i]);
 849                                 goto free_msix_irqs;
 850                         }
 851                 }
 852                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 853                 if (!phba->msi_name[i]) {
 854                         ret = -ENOMEM;
 855                         goto free_msix_irqs;
 856                 }
 857                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 858                         phba->shost->host_no);
 859                 msix_vec = phba->msix_entries[i].vector;
 860                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 861                                   &phwi_context->be_eq[i]);
 862                 if (ret) {
 863                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 864                                      "Failed to register beiscsi_msix_mcc\n");
 865                         kfree(phba->msi_name[i]);
 866                         goto free_msix_irqs;
 867                 }
 868
 869         } else {
 870                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 871                                   "beiscsi", phba);
 872                 if (ret) {
 873                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 874                                      "Failed to register irq\\n");
 875                         return ret;
 876                 }
 877         }
 878         return 0;
 879 free_msix_irqs:
 880         for (j = i - 1; j >= 0; j--) {
 881                 kfree(phba->msi_name[j]);
 882                 msix_vec = phba->msix_entries[j].vector;
 883                 free_irq(msix_vec, &phwi_context->be_eq[j]);
 884         }
 885         return ret;
 886 }
 887
 888 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 889                            unsigned int id, unsigned int num_processed,
 890                            unsigned char rearm, unsigned char event)
 891 {
 892         u32 val = 0;
 893         val |= id & DB_CQ_RING_ID_MASK;
 894         if (rearm)
 895                 val |= 1 << DB_CQ_REARM_SHIFT;
 896         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 897         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 898 }
 899
 900 static unsigned int
 901 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 902                           struct beiscsi_hba *phba,
 903                           unsigned short cid,
 904                           struct pdu_base *ppdu,
 905                           unsigned long pdu_len,
 906                           void *pbuffer, unsigned long buf_len)
 907 {
 908         struct iscsi_conn *conn = beiscsi_conn->conn;
 909         struct iscsi_session *session = conn->session;
 910         struct iscsi_task *task;
 911         struct beiscsi_io_task *io_task;
 912         struct iscsi_hdr *login_hdr;
 913
 914         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 915                                                 PDUBASE_OPCODE_MASK) {
 916         case ISCSI_OP_NOOP_IN:
 917                 pbuffer = NULL;
 918                 buf_len = 0;
 919                 break;
 920         case ISCSI_OP_ASYNC_EVENT:
 921                 break;
 922         case ISCSI_OP_REJECT:
 923                 WARN_ON(!pbuffer);
 924                 WARN_ON(!(buf_len == 48));
 925                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 926                 break;
 927         case ISCSI_OP_LOGIN_RSP:
 928         case ISCSI_OP_TEXT_RSP:
 929                 task = conn->login_task;
 930                 io_task = task->dd_data;
 931                 login_hdr = (struct iscsi_hdr *)ppdu;
 932                 login_hdr->itt = io_task->libiscsi_itt;
 933                 break;
 934         default:
 935                 shost_printk(KERN_WARNING, phba->shost,
 936                              "Unrecognized opcode 0x%x in async msg\n",
 937                              (ppdu->
 938                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 939                                                 & PDUBASE_OPCODE_MASK));
 940                 return 1;
 941         }
 942
 943         spin_lock_bh(&session->lock);
 944         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 945         spin_unlock_bh(&session->lock);
 946         return 0;
 947 }
 948
 949 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 950 {
 951         struct sgl_handle *psgl_handle;
 952
 953         if (phba->io_sgl_hndl_avbl) {
 954                 SE_DEBUG(DBG_LVL_8,
 955                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
 956                          phba->io_sgl_alloc_index);
 957                 psgl_handle = phba->io_sgl_hndl_base[phba->
 958                                                 io_sgl_alloc_index];
 959                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 960                 phba->io_sgl_hndl_avbl--;
 961                 if (phba->io_sgl_alloc_index == (phba->params.
 962                                                  ios_per_ctrl - 1))
 963                         phba->io_sgl_alloc_index = 0;
 964                 else
 965                         phba->io_sgl_alloc_index++;
 966         } else
 967                 psgl_handle = NULL;
 968         return psgl_handle;
 969 }
 970
 971 static void
 972 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 973 {
 974         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
 975                  phba->io_sgl_free_index);
 976         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 977                 /*
 978                  * this can happen if clean_task is called on a task that
 979                  * failed in xmit_task or alloc_pdu.
 980                  */
 981                  SE_DEBUG(DBG_LVL_8,
 982                          "Double Free in IO SGL io_sgl_free_index=%d,"
 983                          "value there=%p\n", phba->io_sgl_free_index,
 984                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 985                 return;
 986         }
 987         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 988         phba->io_sgl_hndl_avbl++;
 989         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 990                 phba->io_sgl_free_index = 0;
 991         else
 992                 phba->io_sgl_free_index++;
 993 }
 994
 995 /**
 996  * alloc_wrb_handle - To allocate a wrb handle
 997  * @phba: The hba pointer
 998  * @cid: The cid to use for allocation
 999  *
1000  * This happens under session_lock until submission to chip
1001  */
1002 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1003 {
1004         struct hwi_wrb_context *pwrb_context;
1005         struct hwi_controller *phwi_ctrlr;
1006         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1007
1008         phwi_ctrlr = phba->phwi_ctrlr;
1009         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1010         if (pwrb_context->wrb_handles_available >= 2) {
1011                 pwrb_handle = pwrb_context->pwrb_handle_base[
1012                                             pwrb_context->alloc_index];
1013                 pwrb_context->wrb_handles_available--;
1014                 if (pwrb_context->alloc_index ==
1015                                                 (phba->params.wrbs_per_cxn - 1))
1016                         pwrb_context->alloc_index = 0;
1017                 else
1018                         pwrb_context->alloc_index++;
1019                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1020                                                 pwrb_context->alloc_index];
1021                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1022         } else
1023                 pwrb_handle = NULL;
1024         return pwrb_handle;
1025 }
1026
1027 /**
1028  * free_wrb_handle - To free the wrb handle back to pool
1029  * @phba: The hba pointer
1030  * @pwrb_context: The context to free from
1031  * @pwrb_handle: The wrb_handle to free
1032  *
1033  * This happens under session_lock until submission to chip
1034  */
1035 static void
1036 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1037                 struct wrb_handle *pwrb_handle)
1038 {
1039         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1040         pwrb_context->wrb_handles_available++;
1041         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1042                 pwrb_context->free_index = 0;
1043         else
1044                 pwrb_context->free_index++;
1045
1046         SE_DEBUG(DBG_LVL_8,
1047                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
1048                  "wrb_handles_available=%d\n",
1049                  pwrb_handle, pwrb_context->free_index,
1050                  pwrb_context->wrb_handles_available);
1051 }
1052
1053 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1054 {
1055         struct sgl_handle *psgl_handle;
1056
1057         if (phba->eh_sgl_hndl_avbl) {
1058                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1059                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1060                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
1061                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1062                 phba->eh_sgl_hndl_avbl--;
1063                 if (phba->eh_sgl_alloc_index ==
1064                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1065                      1))
1066                         phba->eh_sgl_alloc_index = 0;
1067                 else
1068                         phba->eh_sgl_alloc_index++;
1069         } else
1070                 psgl_handle = NULL;
1071         return psgl_handle;
1072 }
1073
1074 void
1075 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1076 {
1077
1078         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1079                              phba->eh_sgl_free_index);
1080         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1081                 /*
1082                  * this can happen if clean_task is called on a task that
1083                  * failed in xmit_task or alloc_pdu.
1084                  */
1085                 SE_DEBUG(DBG_LVL_8,
1086                          "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1087                          phba->eh_sgl_free_index);
1088                 return;
1089         }
1090         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1091         phba->eh_sgl_hndl_avbl++;
1092         if (phba->eh_sgl_free_index ==
1093             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1094                 phba->eh_sgl_free_index = 0;
1095         else
1096                 phba->eh_sgl_free_index++;
1097 }
1098
1099 static void
1100 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1101                struct iscsi_task *task, struct sol_cqe *psol)
1102 {
1103         struct beiscsi_io_task *io_task = task->dd_data;
1104         struct be_status_bhs *sts_bhs =
1105                                 (struct be_status_bhs *)io_task->cmd_bhs;
1106         struct iscsi_conn *conn = beiscsi_conn->conn;
1107         unsigned char *sense;
1108         u32 resid = 0, exp_cmdsn, max_cmdsn;
1109         u8 rsp, status, flags;
1110
1111         exp_cmdsn = (psol->
1112                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1113                         & SOL_EXP_CMD_SN_MASK);
1114         max_cmdsn = ((psol->
1115                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1116                         & SOL_EXP_CMD_SN_MASK) +
1117                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1118                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1119         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1120                                                 & SOL_RESP_MASK) >> 16);
1121         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1122                                                 & SOL_STS_MASK) >> 8);
1123         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1124                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1125         if (!task->sc) {
1126                 if (io_task->scsi_cmnd)
1127                         scsi_dma_unmap(io_task->scsi_cmnd);
1128
1129                 return;
1130         }
1131         task->sc->result = (DID_OK << 16) | status;
1132         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1133                 task->sc->result = DID_ERROR << 16;
1134                 goto unmap;
1135         }
1136
1137         /* bidi not initially supported */
1138         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1139                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1140                                 32] & SOL_RES_CNT_MASK);
1141
1142                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1143                         task->sc->result = DID_ERROR << 16;
1144
1145                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1146                         scsi_set_resid(task->sc, resid);
1147                         if (!status && (scsi_bufflen(task->sc) - resid <
1148                             task->sc->underflow))
1149                                 task->sc->result = DID_ERROR << 16;
1150                 }
1151         }
1152
1153         if (status == SAM_STAT_CHECK_CONDITION) {
1154                 u16 sense_len;
1155                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1156
1157                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1158                 sense_len = be16_to_cpu(*slen);
1159                 memcpy(task->sc->sense_buffer, sense,
1160                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1161         }
1162
1163         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1164                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1165                                                         & SOL_RES_CNT_MASK)
1166                          conn->rxdata_octets += (psol->
1167                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1168                              & SOL_RES_CNT_MASK);
1169         }
1170 unmap:
1171         scsi_dma_unmap(io_task->scsi_cmnd);
1172         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1173 }
1174
1175 static void
1176 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1177                    struct iscsi_task *task, struct sol_cqe *psol)
1178 {
1179         struct iscsi_logout_rsp *hdr;
1180         struct beiscsi_io_task *io_task = task->dd_data;
1181         struct iscsi_conn *conn = beiscsi_conn->conn;
1182
1183         hdr = (struct iscsi_logout_rsp *)task->hdr;
1184         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1185         hdr->t2wait = 5;
1186         hdr->t2retain = 0;
1187         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1188                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1189         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1190                                         32] & SOL_RESP_MASK);
1191         hdr->exp_cmdsn = cpu_to_be32(psol->
1192                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1193                                         & SOL_EXP_CMD_SN_MASK);
1194         hdr->max_cmdsn = be32_to_cpu((psol->
1195                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1196                                         & SOL_EXP_CMD_SN_MASK) +
1197                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1198                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1199         hdr->dlength[0] = 0;
1200         hdr->dlength[1] = 0;
1201         hdr->dlength[2] = 0;
1202         hdr->hlength = 0;
1203         hdr->itt = io_task->libiscsi_itt;
1204         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1205 }
1206
1207 static void
1208 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1209                 struct iscsi_task *task, struct sol_cqe *psol)
1210 {
1211         struct iscsi_tm_rsp *hdr;
1212         struct iscsi_conn *conn = beiscsi_conn->conn;
1213         struct beiscsi_io_task *io_task = task->dd_data;
1214
1215         hdr = (struct iscsi_tm_rsp *)task->hdr;
1216         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1217         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1218                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1219         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1220                                         32] & SOL_RESP_MASK);
1221         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1222                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1223         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1224                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1225                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1226                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1227         hdr->itt = io_task->libiscsi_itt;
1228         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1229 }
1230
1231 static void
1232 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1233                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1234 {
1235         struct hwi_wrb_context *pwrb_context;
1236         struct wrb_handle *pwrb_handle = NULL;
1237         struct hwi_controller *phwi_ctrlr;
1238         struct iscsi_task *task;
1239         struct beiscsi_io_task *io_task;
1240         struct iscsi_conn *conn = beiscsi_conn->conn;
1241         struct iscsi_session *session = conn->session;
1242
1243         phwi_ctrlr = phba->phwi_ctrlr;
1244         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1245                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1246                                 SOL_CID_MASK) >> 6) -
1247                                 phba->fw_config.iscsi_cid_start];
1248         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1249                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1250                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1251         task = pwrb_handle->pio_handle;
1252
1253         io_task = task->dd_data;
1254         spin_lock_bh(&phba->mgmt_sgl_lock);
1255         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1256         spin_unlock_bh(&phba->mgmt_sgl_lock);
1257         spin_lock_bh(&session->lock);
1258         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1259         spin_unlock_bh(&session->lock);
1260 }
1261
1262 static void
1263 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1264                        struct iscsi_task *task, struct sol_cqe *psol)
1265 {
1266         struct iscsi_nopin *hdr;
1267         struct iscsi_conn *conn = beiscsi_conn->conn;
1268         struct beiscsi_io_task *io_task = task->dd_data;
1269
1270         hdr = (struct iscsi_nopin *)task->hdr;
1271         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1272                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1273         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1274                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1275         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1276                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1277                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1278                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1279         hdr->opcode = ISCSI_OP_NOOP_IN;
1280         hdr->itt = io_task->libiscsi_itt;
1281         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1282 }
1283
1284 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1285                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1286 {
1287         struct hwi_wrb_context *pwrb_context;
1288         struct wrb_handle *pwrb_handle;
1289         struct iscsi_wrb *pwrb = NULL;
1290         struct hwi_controller *phwi_ctrlr;
1291         struct iscsi_task *task;
1292         unsigned int type;
1293         struct iscsi_conn *conn = beiscsi_conn->conn;
1294         struct iscsi_session *session = conn->session;
1295
1296         phwi_ctrlr = phba->phwi_ctrlr;
1297         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1298                                 (struct amap_sol_cqe, cid) / 32]
1299                                 & SOL_CID_MASK) >> 6) -
1300                                 phba->fw_config.iscsi_cid_start];
1301         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1302                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1303                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1304         task = pwrb_handle->pio_handle;
1305         pwrb = pwrb_handle->pwrb;
1306         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1307                                  WRB_TYPE_MASK) >> 28;
1308
1309         spin_lock_bh(&session->lock);
1310         switch (type) {
1311         case HWH_TYPE_IO:
1312         case HWH_TYPE_IO_RD:
1313                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1314                      ISCSI_OP_NOOP_OUT)
1315                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1316                 else
1317                         be_complete_io(beiscsi_conn, task, psol);
1318                 break;
1319
1320         case HWH_TYPE_LOGOUT:
1321                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1322                         be_complete_logout(beiscsi_conn, task, psol);
1323                 else
1324                         be_complete_tmf(beiscsi_conn, task, psol);
1325
1326                 break;
1327
1328         case HWH_TYPE_LOGIN:
1329                 SE_DEBUG(DBG_LVL_1,
1330                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1331                          "- Solicited path\n");
1332                 break;
1333
1334         case HWH_TYPE_NOP:
1335                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1336                 break;
1337
1338         default:
1339                 shost_printk(KERN_WARNING, phba->shost,
1340                                 "In hwi_complete_cmd, unknown type = %d"
1341                                 "wrb_index 0x%x CID 0x%x\n", type,
1342                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1343                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1344                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1345                                 cid) / 32] & SOL_CID_MASK) >> 6));
1346                 break;
1347         }
1348
1349         spin_unlock_bh(&session->lock);
1350 }
1351
1352 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1353                                           *pasync_ctx, unsigned int is_header,
1354                                           unsigned int host_write_ptr)
1355 {
1356         if (is_header)
1357                 return &pasync_ctx->async_entry[host_write_ptr].
1358                     header_busy_list;
1359         else
1360                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1361 }
1362
1363 static struct async_pdu_handle *
1364 hwi_get_async_handle(struct beiscsi_hba *phba,
1365                      struct beiscsi_conn *beiscsi_conn,
1366                      struct hwi_async_pdu_context *pasync_ctx,
1367                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1368 {
1369         struct be_bus_address phys_addr;
1370         struct list_head *pbusy_list;
1371         struct async_pdu_handle *pasync_handle = NULL;
1372         unsigned char is_header = 0;
1373
1374         phys_addr.u.a32.address_lo =
1375             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1376             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1377                                                 & PDUCQE_DPL_MASK) >> 16);
1378         phys_addr.u.a32.address_hi =
1379             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1380
1381         phys_addr.u.a64.address =
1382                         *((unsigned long long *)(&phys_addr.u.a64.address));
1383
1384         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1385                         & PDUCQE_CODE_MASK) {
1386         case UNSOL_HDR_NOTIFY:
1387                 is_header = 1;
1388
1389                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1390                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1391                         index) / 32] & PDUCQE_INDEX_MASK));
1392                 break;
1393         case UNSOL_DATA_NOTIFY:
1394                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1395                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1396                                         index) / 32] & PDUCQE_INDEX_MASK));
1397                 break;
1398         default:
1399                 pbusy_list = NULL;
1400                 shost_printk(KERN_WARNING, phba->shost,
1401                         "Unexpected code=%d\n",
1402                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1403                                         code) / 32] & PDUCQE_CODE_MASK);
1404                 return NULL;
1405         }
1406
1407         WARN_ON(list_empty(pbusy_list));
1408         list_for_each_entry(pasync_handle, pbusy_list, link) {
1409                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1410                         break;
1411         }
1412
1413         WARN_ON(!pasync_handle);
1414
1415         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1416                                              phba->fw_config.iscsi_cid_start;
1417         pasync_handle->is_header = is_header;
1418         pasync_handle->buffer_len = ((pdpdu_cqe->
1419                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1420                         & PDUCQE_DPL_MASK) >> 16);
1421
1422         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1423                         index) / 32] & PDUCQE_INDEX_MASK);
1424         return pasync_handle;
1425 }
1426
1427 static unsigned int
1428 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1429                            unsigned int is_header, unsigned int cq_index)
1430 {
1431         struct list_head *pbusy_list;
1432         struct async_pdu_handle *pasync_handle;
1433         unsigned int num_entries, writables = 0;
1434         unsigned int *pep_read_ptr, *pwritables;
1435
1436         num_entries = pasync_ctx->num_entries;
1437         if (is_header) {
1438                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1439                 pwritables = &pasync_ctx->async_header.writables;
1440         } else {
1441                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1442                 pwritables = &pasync_ctx->async_data.writables;
1443         }
1444
1445         while ((*pep_read_ptr) != cq_index) {
1446                 (*pep_read_ptr)++;
1447                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1448
1449                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1450                                                      *pep_read_ptr);
1451                 if (writables == 0)
1452                         WARN_ON(list_empty(pbusy_list));
1453
1454                 if (!list_empty(pbusy_list)) {
1455                         pasync_handle = list_entry(pbusy_list->next,
1456                                                    struct async_pdu_handle,
1457                                                    link);
1458                         WARN_ON(!pasync_handle);
1459                         pasync_handle->consumed = 1;
1460                 }
1461
1462                 writables++;
1463         }
1464
1465         if (!writables) {
1466                 SE_DEBUG(DBG_LVL_1,
1467                          "Duplicate notification received - index 0x%x!!\n",
1468                          cq_index);
1469                 WARN_ON(1);
1470         }
1471
1472         *pwritables = *pwritables + writables;
1473         return 0;
1474 }
1475
1476 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1477                                        unsigned int cri)
1478 {
1479         struct hwi_controller *phwi_ctrlr;
1480         struct hwi_async_pdu_context *pasync_ctx;
1481         struct async_pdu_handle *pasync_handle, *tmp_handle;
1482         struct list_head *plist;
1483
1484         phwi_ctrlr = phba->phwi_ctrlr;
1485         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1486
1487         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1488
1489         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1490                 list_del(&pasync_handle->link);
1491
1492                 if (pasync_handle->is_header) {
1493                         list_add_tail(&pasync_handle->link,
1494                                       &pasync_ctx->async_header.free_list);
1495                         pasync_ctx->async_header.free_entries++;
1496                 } else {
1497                         list_add_tail(&pasync_handle->link,
1498                                       &pasync_ctx->async_data.free_list);
1499                         pasync_ctx->async_data.free_entries++;
1500                 }
1501         }
1502
1503         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1504         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1505         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1506 }
1507
1508 static struct phys_addr *
1509 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1510                      unsigned int is_header, unsigned int host_write_ptr)
1511 {
1512         struct phys_addr *pasync_sge = NULL;
1513
1514         if (is_header)
1515                 pasync_sge = pasync_ctx->async_header.ring_base;
1516         else
1517                 pasync_sge = pasync_ctx->async_data.ring_base;
1518
1519         return pasync_sge + host_write_ptr;
1520 }
1521
1522 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1523                                    unsigned int is_header)
1524 {
1525         struct hwi_controller *phwi_ctrlr;
1526         struct hwi_async_pdu_context *pasync_ctx;
1527         struct async_pdu_handle *pasync_handle;
1528         struct list_head *pfree_link, *pbusy_list;
1529         struct phys_addr *pasync_sge;
1530         unsigned int ring_id, num_entries;
1531         unsigned int host_write_num;
1532         unsigned int writables;
1533         unsigned int i = 0;
1534         u32 doorbell = 0;
1535
1536         phwi_ctrlr = phba->phwi_ctrlr;
1537         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1538         num_entries = pasync_ctx->num_entries;
1539
1540         if (is_header) {
1541                 writables = min(pasync_ctx->async_header.writables,
1542                                 pasync_ctx->async_header.free_entries);
1543                 pfree_link = pasync_ctx->async_header.free_list.next;
1544                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1545                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1546         } else {
1547                 writables = min(pasync_ctx->async_data.writables,
1548                                 pasync_ctx->async_data.free_entries);
1549                 pfree_link = pasync_ctx->async_data.free_list.next;
1550                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1551                 ring_id = phwi_ctrlr->default_pdu_data.id;
1552         }
1553
1554         writables = (writables / 8) * 8;
1555         if (writables) {
1556                 for (i = 0; i < writables; i++) {
1557                         pbusy_list =
1558                             hwi_get_async_busy_list(pasync_ctx, is_header,
1559                                                     host_write_num);
1560                         pasync_handle =
1561                             list_entry(pfree_link, struct async_pdu_handle,
1562                                                                 link);
1563                         WARN_ON(!pasync_handle);
1564                         pasync_handle->consumed = 0;
1565
1566                         pfree_link = pfree_link->next;
1567
1568                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1569                                                 is_header, host_write_num);
1570
1571                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1572                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1573
1574                         list_move(&pasync_handle->link, pbusy_list);
1575
1576                         host_write_num++;
1577                         host_write_num = host_write_num % num_entries;
1578                 }
1579
1580                 if (is_header) {
1581                         pasync_ctx->async_header.host_write_ptr =
1582                                                         host_write_num;
1583                         pasync_ctx->async_header.free_entries -= writables;
1584                         pasync_ctx->async_header.writables -= writables;
1585                         pasync_ctx->async_header.busy_entries += writables;
1586                 } else {
1587                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1588                         pasync_ctx->async_data.free_entries -= writables;
1589                         pasync_ctx->async_data.writables -= writables;
1590                         pasync_ctx->async_data.busy_entries += writables;
1591                 }
1592
1593                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1594                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1595                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1596                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1597                                         << DB_DEF_PDU_CQPROC_SHIFT;
1598
1599                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1600         }
1601 }
1602
1603 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1604                                          struct beiscsi_conn *beiscsi_conn,
1605                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1606 {
1607         struct hwi_controller *phwi_ctrlr;
1608         struct hwi_async_pdu_context *pasync_ctx;
1609         struct async_pdu_handle *pasync_handle = NULL;
1610         unsigned int cq_index = -1;
1611
1612         phwi_ctrlr = phba->phwi_ctrlr;
1613         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1614
1615         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1616                                              pdpdu_cqe, &cq_index);
1617         BUG_ON(pasync_handle->is_header != 0);
1618         if (pasync_handle->consumed == 0)
1619                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1620                                            cq_index);
1621
1622         hwi_free_async_msg(phba, pasync_handle->cri);
1623         hwi_post_async_buffers(phba, pasync_handle->is_header);
1624 }
1625
1626 static unsigned int
1627 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1628                   struct beiscsi_hba *phba,
1629                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1630 {
1631         struct list_head *plist;
1632         struct async_pdu_handle *pasync_handle;
1633         void *phdr = NULL;
1634         unsigned int hdr_len = 0, buf_len = 0;
1635         unsigned int status, index = 0, offset = 0;
1636         void *pfirst_buffer = NULL;
1637         unsigned int num_buf = 0;
1638
1639         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1640
1641         list_for_each_entry(pasync_handle, plist, link) {
1642                 if (index == 0) {
1643                         phdr = pasync_handle->pbuffer;
1644                         hdr_len = pasync_handle->buffer_len;
1645                 } else {
1646                         buf_len = pasync_handle->buffer_len;
1647                         if (!num_buf) {
1648                                 pfirst_buffer = pasync_handle->pbuffer;
1649                                 num_buf++;
1650                         }
1651                         memcpy(pfirst_buffer + offset,
1652                                pasync_handle->pbuffer, buf_len);
1653                         offset += buf_len;
1654                 }
1655                 index++;
1656         }
1657
1658         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1659                                            (beiscsi_conn->beiscsi_conn_cid -
1660                                             phba->fw_config.iscsi_cid_start),
1661                                             phdr, hdr_len, pfirst_buffer,
1662                                             offset);
1663
1664         hwi_free_async_msg(phba, cri);
1665         return 0;
1666 }
1667
1668 static unsigned int
1669 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1670                      struct beiscsi_hba *phba,
1671                      struct async_pdu_handle *pasync_handle)
1672 {
1673         struct hwi_async_pdu_context *pasync_ctx;
1674         struct hwi_controller *phwi_ctrlr;
1675         unsigned int bytes_needed = 0, status = 0;
1676         unsigned short cri = pasync_handle->cri;
1677         struct pdu_base *ppdu;
1678
1679         phwi_ctrlr = phba->phwi_ctrlr;
1680         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1681
1682         list_del(&pasync_handle->link);
1683         if (pasync_handle->is_header) {
1684                 pasync_ctx->async_header.busy_entries--;
1685                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1686                         hwi_free_async_msg(phba, cri);
1687                         BUG();
1688                 }
1689
1690                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1691                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1692                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1693                                 (unsigned short)pasync_handle->buffer_len;
1694                 list_add_tail(&pasync_handle->link,
1695                               &pasync_ctx->async_entry[cri].wait_queue.list);
1696
1697                 ppdu = pasync_handle->pbuffer;
1698                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1699                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1700                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1701                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1702                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1703
1704                 if (status == 0) {
1705                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1706                             bytes_needed;
1707
1708                         if (bytes_needed == 0)
1709                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1710                                                            pasync_ctx, cri);
1711                 }
1712         } else {
1713                 pasync_ctx->async_data.busy_entries--;
1714                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1715                         list_add_tail(&pasync_handle->link,
1716                                       &pasync_ctx->async_entry[cri].wait_queue.
1717                                       list);
1718                         pasync_ctx->async_entry[cri].wait_queue.
1719                                 bytes_received +=
1720                                 (unsigned short)pasync_handle->buffer_len;
1721
1722                         if (pasync_ctx->async_entry[cri].wait_queue.
1723                             bytes_received >=
1724                             pasync_ctx->async_entry[cri].wait_queue.
1725                             bytes_needed)
1726                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1727                                                            pasync_ctx, cri);
1728                 }
1729         }
1730         return status;
1731 }
1732
1733 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1734                                          struct beiscsi_hba *phba,
1735                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1736 {
1737         struct hwi_controller *phwi_ctrlr;
1738         struct hwi_async_pdu_context *pasync_ctx;
1739         struct async_pdu_handle *pasync_handle = NULL;
1740         unsigned int cq_index = -1;
1741
1742         phwi_ctrlr = phba->phwi_ctrlr;
1743         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1744         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1745                                              pdpdu_cqe, &cq_index);
1746
1747         if (pasync_handle->consumed == 0)
1748                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1749                                            cq_index);
1750         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1751         hwi_post_async_buffers(phba, pasync_handle->is_header);
1752 }
1753
1754 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1755 {
1756         struct be_queue_info *mcc_cq;
1757         struct  be_mcc_compl *mcc_compl;
1758         unsigned int num_processed = 0;
1759
1760         mcc_cq = &phba->ctrl.mcc_obj.cq;
1761         mcc_compl = queue_tail_node(mcc_cq);
1762         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1763         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1764
1765                 if (num_processed >= 32) {
1766                         hwi_ring_cq_db(phba, mcc_cq->id,
1767                                         num_processed, 0, 0);
1768                         num_processed = 0;
1769                 }
1770                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1771                         /* Interpret flags as an async trailer */
1772                         if (is_link_state_evt(mcc_compl->flags))
1773                                 /* Interpret compl as a async link evt */
1774                                 beiscsi_async_link_state_process(phba,
1775                                 (struct be_async_event_link_state *) mcc_compl);
1776                         else
1777                                 SE_DEBUG(DBG_LVL_1,
1778                                         " Unsupported Async Event, flags"
1779                                         " = 0x%08x\n", mcc_compl->flags);
1780                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1781                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1782                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1783                 }
1784
1785                 mcc_compl->flags = 0;
1786                 queue_tail_inc(mcc_cq);
1787                 mcc_compl = queue_tail_node(mcc_cq);
1788                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1789                 num_processed++;
1790         }
1791
1792         if (num_processed > 0)
1793                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1794
1795 }
1796
1797 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1798 {
1799         struct be_queue_info *cq;
1800         struct sol_cqe *sol;
1801         struct dmsg_cqe *dmsg;
1802         unsigned int num_processed = 0;
1803         unsigned int tot_nump = 0;
1804         struct beiscsi_conn *beiscsi_conn;
1805         struct beiscsi_endpoint *beiscsi_ep;
1806         struct iscsi_endpoint *ep;
1807         struct beiscsi_hba *phba;
1808
1809         cq = pbe_eq->cq;
1810         sol = queue_tail_node(cq);
1811         phba = pbe_eq->phba;
1812
1813         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1814                CQE_VALID_MASK) {
1815                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1816
1817                 ep = phba->ep_array[(u32) ((sol->
1818                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1819                                    SOL_CID_MASK) >> 6) -
1820                                    phba->fw_config.iscsi_cid_start];
1821
1822                 beiscsi_ep = ep->dd_data;
1823                 beiscsi_conn = beiscsi_ep->conn;
1824
1825                 if (num_processed >= 32) {
1826                         hwi_ring_cq_db(phba, cq->id,
1827                                         num_processed, 0, 0);
1828                         tot_nump += num_processed;
1829                         num_processed = 0;
1830                 }
1831
1832                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1833                         32] & CQE_CODE_MASK) {
1834                 case SOL_CMD_COMPLETE:
1835                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1836                         break;
1837                 case DRIVERMSG_NOTIFY:
1838                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
1839                         dmsg = (struct dmsg_cqe *)sol;
1840                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1841                         break;
1842                 case UNSOL_HDR_NOTIFY:
1843                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1844                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1845                                              (struct i_t_dpdu_cqe *)sol);
1846                         break;
1847                 case UNSOL_DATA_NOTIFY:
1848                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1849                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1850                                              (struct i_t_dpdu_cqe *)sol);
1851                         break;
1852                 case CXN_INVALIDATE_INDEX_NOTIFY:
1853                 case CMD_INVALIDATED_NOTIFY:
1854                 case CXN_INVALIDATE_NOTIFY:
1855                         SE_DEBUG(DBG_LVL_1,
1856                                  "Ignoring CQ Error notification for cmd/cxn"
1857                                  "invalidate\n");
1858                         break;
1859                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1860                 case CMD_KILLED_INVALID_STATSN_RCVD:
1861                 case CMD_KILLED_INVALID_R2T_RCVD:
1862                 case CMD_CXN_KILLED_LUN_INVALID:
1863                 case CMD_CXN_KILLED_ICD_INVALID:
1864                 case CMD_CXN_KILLED_ITT_INVALID:
1865                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1866                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1867                         SE_DEBUG(DBG_LVL_1,
1868                                  "CQ Error notification for cmd.. "
1869                                  "code %d cid 0x%x\n",
1870                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1871                                  32] & CQE_CODE_MASK,
1872                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1873                                  32] & SOL_CID_MASK));
1874                         break;
1875                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1876                         SE_DEBUG(DBG_LVL_1,
1877                                  "Digest error on def pdu ring, dropping..\n");
1878                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1879                                              (struct i_t_dpdu_cqe *) sol);
1880                         break;
1881                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1882                 case CXN_KILLED_BURST_LEN_MISMATCH:
1883                 case CXN_KILLED_AHS_RCVD:
1884                 case CXN_KILLED_HDR_DIGEST_ERR:
1885                 case CXN_KILLED_UNKNOWN_HDR:
1886                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1887                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1888                 case CXN_KILLED_TIMED_OUT:
1889                 case CXN_KILLED_FIN_RCVD:
1890                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1891                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1892                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1893                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1894                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1895                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1896                                  "0x%x...\n",
1897                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1898                                  32] & CQE_CODE_MASK,
1899                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1900                                  32] & CQE_CID_MASK));
1901                         iscsi_conn_failure(beiscsi_conn->conn,
1902                                            ISCSI_ERR_CONN_FAILED);
1903                         break;
1904                 case CXN_KILLED_RST_SENT:
1905                 case CXN_KILLED_RST_RCVD:
1906                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1907                                 "received/sent on CID 0x%x...\n",
1908                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1909                                  32] & CQE_CODE_MASK,
1910                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1911                                  32] & CQE_CID_MASK));
1912                         iscsi_conn_failure(beiscsi_conn->conn,
1913                                            ISCSI_ERR_CONN_FAILED);
1914                         break;
1915                 default:
1916                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1917                                  "received on CID 0x%x...\n",
1918                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1919                                  32] & CQE_CODE_MASK,
1920                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1921                                  32] & CQE_CID_MASK));
1922                         break;
1923                 }
1924
1925                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1926                 queue_tail_inc(cq);
1927                 sol = queue_tail_node(cq);
1928                 num_processed++;
1929         }
1930
1931         if (num_processed > 0) {
1932                 tot_nump += num_processed;
1933                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1934         }
1935         return tot_nump;
1936 }
1937
1938 void beiscsi_process_all_cqs(struct work_struct *work)
1939 {
1940         unsigned long flags;
1941         struct hwi_controller *phwi_ctrlr;
1942         struct hwi_context_memory *phwi_context;
1943         struct be_eq_obj *pbe_eq;
1944         struct beiscsi_hba *phba =
1945             container_of(work, struct beiscsi_hba, work_cqs);
1946
1947         phwi_ctrlr = phba->phwi_ctrlr;
1948         phwi_context = phwi_ctrlr->phwi_ctxt;
1949         if (phba->msix_enabled)
1950                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1951         else
1952                 pbe_eq = &phwi_context->be_eq[0];
1953
1954         if (phba->todo_mcc_cq) {
1955                 spin_lock_irqsave(&phba->isr_lock, flags);
1956                 phba->todo_mcc_cq = 0;
1957                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1958                 beiscsi_process_mcc_isr(phba);
1959         }
1960
1961         if (phba->todo_cq) {
1962                 spin_lock_irqsave(&phba->isr_lock, flags);
1963                 phba->todo_cq = 0;
1964                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1965                 beiscsi_process_cq(pbe_eq);
1966         }
1967 }
1968
1969 static int be_iopoll(struct blk_iopoll *iop, int budget)
1970 {
1971         static unsigned int ret;
1972         struct beiscsi_hba *phba;
1973         struct be_eq_obj *pbe_eq;
1974
1975         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1976         ret = beiscsi_process_cq(pbe_eq);
1977         if (ret < budget) {
1978                 phba = pbe_eq->phba;
1979                 blk_iopoll_complete(iop);
1980                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1981                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1982         }
1983         return ret;
1984 }
1985
1986 static void
1987 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1988               unsigned int num_sg, struct beiscsi_io_task *io_task)
1989 {
1990         struct iscsi_sge *psgl;
1991         unsigned int sg_len, index;
1992         unsigned int sge_len = 0;
1993         unsigned long long addr;
1994         struct scatterlist *l_sg;
1995         unsigned int offset;
1996
1997         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1998                                       io_task->bhs_pa.u.a32.address_lo);
1999         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2000                                       io_task->bhs_pa.u.a32.address_hi);
2001
2002         l_sg = sg;
2003         for (index = 0; (index < num_sg) && (index < 2); index++,
2004                                                          sg = sg_next(sg)) {
2005                 if (index == 0) {
2006                         sg_len = sg_dma_len(sg);
2007                         addr = (u64) sg_dma_address(sg);
2008                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2009                                                 ((u32)(addr & 0xFFFFFFFF)));
2010                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2011                                                         ((u32)(addr >> 32)));
2012                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2013                                                         sg_len);
2014                         sge_len = sg_len;
2015                 } else {
2016                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2017                                                         pwrb, sge_len);
2018                         sg_len = sg_dma_len(sg);
2019                         addr = (u64) sg_dma_address(sg);
2020                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2021                                                 ((u32)(addr & 0xFFFFFFFF)));
2022                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2023                                                         ((u32)(addr >> 32)));
2024                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2025                                                         sg_len);
2026                 }
2027         }
2028         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2029         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2030
2031         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2032
2033         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2034                         io_task->bhs_pa.u.a32.address_hi);
2035         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2036                         io_task->bhs_pa.u.a32.address_lo);
2037
2038         if (num_sg == 1) {
2039                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2040                                                                 1);
2041                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2042                                                                 0);
2043         } else if (num_sg == 2) {
2044                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2045                                                                 0);
2046                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2047                                                                 1);
2048         } else {
2049                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2050                                                                 0);
2051                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2052                                                                 0);
2053         }
2054         sg = l_sg;
2055         psgl++;
2056         psgl++;
2057         offset = 0;
2058         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2059                 sg_len = sg_dma_len(sg);
2060                 addr = (u64) sg_dma_address(sg);
2061                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2062                                                 (addr & 0xFFFFFFFF));
2063                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2064                                                 (addr >> 32));
2065                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2066                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2067                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2068                 offset += sg_len;
2069         }
2070         psgl--;
2071         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2072 }
2073
2074 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2075 {
2076         struct iscsi_sge *psgl;
2077         unsigned long long addr;
2078         struct beiscsi_io_task *io_task = task->dd_data;
2079         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2080         struct beiscsi_hba *phba = beiscsi_conn->phba;
2081
2082         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2083         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2084                                 io_task->bhs_pa.u.a32.address_lo);
2085         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2086                                 io_task->bhs_pa.u.a32.address_hi);
2087
2088         if (task->data) {
2089                 if (task->data_count) {
2090                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2091                         addr = (u64) pci_map_single(phba->pcidev,
2092                                                     task->data,
2093                                                     task->data_count, 1);
2094                 } else {
2095                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2096                         addr = 0;
2097                 }
2098                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2099                                                 ((u32)(addr & 0xFFFFFFFF)));
2100                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2101                                                 ((u32)(addr >> 32)));
2102                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2103                                                 task->data_count);
2104
2105                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2106         } else {
2107                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2108                 addr = 0;
2109         }
2110
2111         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2112
2113         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2114
2115         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2116                       io_task->bhs_pa.u.a32.address_hi);
2117         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2118                       io_task->bhs_pa.u.a32.address_lo);
2119         if (task->data) {
2120                 psgl++;
2121                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2122                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2123                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2124                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2125                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2126                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2127
2128                 psgl++;
2129                 if (task->data) {
2130                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2131                                                 ((u32)(addr & 0xFFFFFFFF)));
2132                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2133                                                 ((u32)(addr >> 32)));
2134                 }
2135                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2136         }
2137         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2138 }
2139
2140 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2141 {
2142         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2143         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2144         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2145
2146         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2147                                       sizeof(struct sol_cqe));
2148         num_async_pdu_buf_pages =
2149                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2150                                        phba->params.defpdu_hdr_sz);
2151         num_async_pdu_buf_sgl_pages =
2152                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2153                                        sizeof(struct phys_addr));
2154         num_async_pdu_data_pages =
2155                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2156                                        phba->params.defpdu_data_sz);
2157         num_async_pdu_data_sgl_pages =
2158                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2159                                        sizeof(struct phys_addr));
2160
2161         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2162
2163         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2164                                                  BE_ISCSI_PDU_HEADER_SIZE;
2165         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2166                                             sizeof(struct hwi_context_memory);
2167
2168
2169         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2170             * (phba->params.wrbs_per_cxn)
2171             * phba->params.cxns_per_ctrl;
2172         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2173                                  (phba->params.wrbs_per_cxn);
2174         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2175                                 phba->params.cxns_per_ctrl);
2176
2177         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2178                 phba->params.icds_per_ctrl;
2179         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2180                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2181
2182         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2183                 num_async_pdu_buf_pages * PAGE_SIZE;
2184         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2185                 num_async_pdu_data_pages * PAGE_SIZE;
2186         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2187                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2188         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2189                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2190         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2191                 phba->params.asyncpdus_per_ctrl *
2192                 sizeof(struct async_pdu_handle);
2193         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2194                 phba->params.asyncpdus_per_ctrl *
2195                 sizeof(struct async_pdu_handle);
2196         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2197                 sizeof(struct hwi_async_pdu_context) +
2198                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2199 }
2200
2201 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2202 {
2203         struct be_mem_descriptor *mem_descr;
2204         dma_addr_t bus_add;
2205         struct mem_array *mem_arr, *mem_arr_orig;
2206         unsigned int i, j, alloc_size, curr_alloc_size;
2207
2208         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2209         if (!phba->phwi_ctrlr)
2210                 return -ENOMEM;
2211
2212         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2213                                  GFP_KERNEL);
2214         if (!phba->init_mem) {
2215                 kfree(phba->phwi_ctrlr);
2216                 return -ENOMEM;
2217         }
2218
2219         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2220                                GFP_KERNEL);
2221         if (!mem_arr_orig) {
2222                 kfree(phba->init_mem);
2223                 kfree(phba->phwi_ctrlr);
2224                 return -ENOMEM;
2225         }
2226
2227         mem_descr = phba->init_mem;
2228         for (i = 0; i < SE_MEM_MAX; i++) {
2229                 j = 0;
2230                 mem_arr = mem_arr_orig;
2231                 alloc_size = phba->mem_req[i];
2232                 memset(mem_arr, 0, sizeof(struct mem_array) *
2233                        BEISCSI_MAX_FRAGS_INIT);
2234                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2235                 do {
2236                         mem_arr->virtual_address = pci_alloc_consistent(
2237                                                         phba->pcidev,
2238                                                         curr_alloc_size,
2239                                                         &bus_add);
2240                         if (!mem_arr->virtual_address) {
2241                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2242                                         goto free_mem;
2243                                 if (curr_alloc_size -
2244                                         rounddown_pow_of_two(curr_alloc_size))
2245                                         curr_alloc_size = rounddown_pow_of_two
2246                                                              (curr_alloc_size);
2247                                 else
2248                                         curr_alloc_size = curr_alloc_size / 2;
2249                         } else {
2250                                 mem_arr->bus_address.u.
2251                                     a64.address = (__u64) bus_add;
2252                                 mem_arr->size = curr_alloc_size;
2253                                 alloc_size -= curr_alloc_size;
2254                                 curr_alloc_size = min(be_max_phys_size *
2255                                                       1024, alloc_size);
2256                                 j++;
2257                                 mem_arr++;
2258                         }
2259                 } while (alloc_size);
2260                 mem_descr->num_elements = j;
2261                 mem_descr->size_in_bytes = phba->mem_req[i];
2262                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2263                                                GFP_KERNEL);
2264                 if (!mem_descr->mem_array)
2265                         goto free_mem;
2266
2267                 memcpy(mem_descr->mem_array, mem_arr_orig,
2268                        sizeof(struct mem_array) * j);
2269                 mem_descr++;
2270         }
2271         kfree(mem_arr_orig);
2272         return 0;
2273 free_mem:
2274         mem_descr->num_elements = j;
2275         while ((i) || (j)) {
2276                 for (j = mem_descr->num_elements; j > 0; j--) {
2277                         pci_free_consistent(phba->pcidev,
2278                                             mem_descr->mem_array[j - 1].size,
2279                                             mem_descr->mem_array[j - 1].
2280                                             virtual_address,
2281                                             (unsigned long)mem_descr->
2282                                             mem_array[j - 1].
2283                                             bus_address.u.a64.address);
2284                 }
2285                 if (i) {
2286                         i--;
2287                         kfree(mem_descr->mem_array);
2288                         mem_descr--;
2289                 }
2290         }
2291         kfree(mem_arr_orig);
2292         kfree(phba->init_mem);
2293         kfree(phba->phwi_ctrlr);
2294         return -ENOMEM;
2295 }
2296
2297 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2298 {
2299         beiscsi_find_mem_req(phba);
2300         return beiscsi_alloc_mem(phba);
2301 }
2302
2303 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2304 {
2305         struct pdu_data_out *pdata_out;
2306         struct pdu_nop_out *pnop_out;
2307         struct be_mem_descriptor *mem_descr;
2308
2309         mem_descr = phba->init_mem;
2310         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2311         pdata_out =
2312             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2313         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2314
2315         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2316                       IIOC_SCSI_DATA);
2317
2318         pnop_out =
2319             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2320                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2321
2322         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2323         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2324         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2325         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2326 }
2327
2328 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2329 {
2330         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2331         struct wrb_handle *pwrb_handle = NULL;
2332         struct hwi_controller *phwi_ctrlr;
2333         struct hwi_wrb_context *pwrb_context;
2334         struct iscsi_wrb *pwrb = NULL;
2335         unsigned int num_cxn_wrbh = 0;
2336         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2337
2338         mem_descr_wrbh = phba->init_mem;
2339         mem_descr_wrbh += HWI_MEM_WRBH;
2340
2341         mem_descr_wrb = phba->init_mem;
2342         mem_descr_wrb += HWI_MEM_WRB;
2343         phwi_ctrlr = phba->phwi_ctrlr;
2344
2345         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2346                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2347                 pwrb_context->pwrb_handle_base =
2348                                 kzalloc(sizeof(struct wrb_handle *) *
2349                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2350                 if (!pwrb_context->pwrb_handle_base) {
2351                         shost_printk(KERN_ERR, phba->shost,
2352                                         "Mem Alloc Failed. Failing to load\n");
2353                         goto init_wrb_hndl_failed;
2354                 }
2355                 pwrb_context->pwrb_handle_basestd =
2356                                 kzalloc(sizeof(struct wrb_handle *) *
2357                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2358                 if (!pwrb_context->pwrb_handle_basestd) {
2359                         shost_printk(KERN_ERR, phba->shost,
2360                                         "Mem Alloc Failed. Failing to load\n");
2361                         goto init_wrb_hndl_failed;
2362                 }
2363                 if (!num_cxn_wrbh) {
2364                         pwrb_handle =
2365                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2366                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2367                                         ((sizeof(struct wrb_handle)) *
2368                                          phba->params.wrbs_per_cxn));
2369                         idx++;
2370                 }
2371                 pwrb_context->alloc_index = 0;
2372                 pwrb_context->wrb_handles_available = 0;
2373                 pwrb_context->free_index = 0;
2374
2375                 if (num_cxn_wrbh) {
2376                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2377                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2378                                 pwrb_context->pwrb_handle_basestd[j] =
2379                                                                 pwrb_handle;
2380                                 pwrb_context->wrb_handles_available++;
2381                                 pwrb_handle->wrb_index = j;
2382                                 pwrb_handle++;
2383                         }
2384                         num_cxn_wrbh--;
2385                 }
2386         }
2387         idx = 0;
2388         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2389                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2390                 if (!num_cxn_wrb) {
2391                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2392                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2393                                 ((sizeof(struct iscsi_wrb) *
2394                                   phba->params.wrbs_per_cxn));
2395                         idx++;
2396                 }
2397
2398                 if (num_cxn_wrb) {
2399                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2400                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2401                                 pwrb_handle->pwrb = pwrb;
2402                                 pwrb++;
2403                         }
2404                         num_cxn_wrb--;
2405                 }
2406         }
2407         return 0;
2408 init_wrb_hndl_failed:
2409         for (j = index; j > 0; j--) {
2410                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2411                 kfree(pwrb_context->pwrb_handle_base);
2412                 kfree(pwrb_context->pwrb_handle_basestd);
2413         }
2414         return -ENOMEM;
2415 }
2416
2417 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2418 {
2419         struct hwi_controller *phwi_ctrlr;
2420         struct hba_parameters *p = &phba->params;
2421         struct hwi_async_pdu_context *pasync_ctx;
2422         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2423         unsigned int index, idx, num_per_mem, num_async_data;
2424         struct be_mem_descriptor *mem_descr;
2425
2426         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2427         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2428
2429         phwi_ctrlr = phba->phwi_ctrlr;
2430         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2431                                 mem_descr->mem_array[0].virtual_address;
2432         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2433         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2434
2435         pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2436         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2437
2438         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2439         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2440         if (mem_descr->mem_array[0].virtual_address) {
2441                 SE_DEBUG(DBG_LVL_8,
2442                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2443                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2444         } else
2445                 shost_printk(KERN_WARNING, phba->shost,
2446                              "No Virtual address\n");
2447
2448         pasync_ctx->async_header.va_base =
2449                         mem_descr->mem_array[0].virtual_address;
2450
2451         pasync_ctx->async_header.pa_base.u.a64.address =
2452                         mem_descr->mem_array[0].bus_address.u.a64.address;
2453
2454         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2455         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2456         if (mem_descr->mem_array[0].virtual_address) {
2457                 SE_DEBUG(DBG_LVL_8,
2458                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2459                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2460         } else
2461                 shost_printk(KERN_WARNING, phba->shost,
2462                             "No Virtual address\n");
2463         pasync_ctx->async_header.ring_base =
2464                         mem_descr->mem_array[0].virtual_address;
2465
2466         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2467         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2468         if (mem_descr->mem_array[0].virtual_address) {
2469                 SE_DEBUG(DBG_LVL_8,
2470                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2471                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2472         } else
2473                 shost_printk(KERN_WARNING, phba->shost,
2474                             "No Virtual address\n");
2475
2476         pasync_ctx->async_header.handle_base =
2477                         mem_descr->mem_array[0].virtual_address;
2478         pasync_ctx->async_header.writables = 0;
2479         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2480
2481
2482         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2483         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2484         if (mem_descr->mem_array[0].virtual_address) {
2485                 SE_DEBUG(DBG_LVL_8,
2486                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2487                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2488         } else
2489                 shost_printk(KERN_WARNING, phba->shost,
2490                              "No Virtual address\n");
2491
2492         pasync_ctx->async_data.ring_base =
2493                         mem_descr->mem_array[0].virtual_address;
2494
2495         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2496         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2497         if (!mem_descr->mem_array[0].virtual_address)
2498                 shost_printk(KERN_WARNING, phba->shost,
2499                             "No Virtual address\n");
2500
2501         pasync_ctx->async_data.handle_base =
2502                         mem_descr->mem_array[0].virtual_address;
2503         pasync_ctx->async_data.writables = 0;
2504         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2505
2506         pasync_header_h =
2507                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2508         pasync_data_h =
2509                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2510
2511         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2512         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2513         if (mem_descr->mem_array[0].virtual_address) {
2514                 SE_DEBUG(DBG_LVL_8,
2515                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2516                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2517         } else
2518                 shost_printk(KERN_WARNING, phba->shost,
2519                             "No Virtual address\n");
2520         idx = 0;
2521         pasync_ctx->async_data.va_base =
2522                         mem_descr->mem_array[idx].virtual_address;
2523         pasync_ctx->async_data.pa_base.u.a64.address =
2524                         mem_descr->mem_array[idx].bus_address.u.a64.address;
2525
2526         num_async_data = ((mem_descr->mem_array[idx].size) /
2527                                 phba->params.defpdu_data_sz);
2528         num_per_mem = 0;
2529
2530         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2531                 pasync_header_h->cri = -1;
2532                 pasync_header_h->index = (char)index;
2533                 INIT_LIST_HEAD(&pasync_header_h->link);
2534                 pasync_header_h->pbuffer =
2535                         (void *)((unsigned long)
2536                         (pasync_ctx->async_header.va_base) +
2537                         (p->defpdu_hdr_sz * index));
2538
2539                 pasync_header_h->pa.u.a64.address =
2540                         pasync_ctx->async_header.pa_base.u.a64.address +
2541                         (p->defpdu_hdr_sz * index);
2542
2543                 list_add_tail(&pasync_header_h->link,
2544                                 &pasync_ctx->async_header.free_list);
2545                 pasync_header_h++;
2546                 pasync_ctx->async_header.free_entries++;
2547                 pasync_ctx->async_header.writables++;
2548
2549                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2550                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2551                                header_busy_list);
2552                 pasync_data_h->cri = -1;
2553                 pasync_data_h->index = (char)index;
2554                 INIT_LIST_HEAD(&pasync_data_h->link);
2555
2556                 if (!num_async_data) {
2557                         num_per_mem = 0;
2558                         idx++;
2559                         pasync_ctx->async_data.va_base =
2560                                 mem_descr->mem_array[idx].virtual_address;
2561                         pasync_ctx->async_data.pa_base.u.a64.address =
2562                                 mem_descr->mem_array[idx].
2563                                 bus_address.u.a64.address;
2564
2565                         num_async_data = ((mem_descr->mem_array[idx].size) /
2566                                         phba->params.defpdu_data_sz);
2567                 }
2568                 pasync_data_h->pbuffer =
2569                         (void *)((unsigned long)
2570                         (pasync_ctx->async_data.va_base) +
2571                         (p->defpdu_data_sz * num_per_mem));
2572
2573                 pasync_data_h->pa.u.a64.address =
2574                     pasync_ctx->async_data.pa_base.u.a64.address +
2575                     (p->defpdu_data_sz * num_per_mem);
2576                 num_per_mem++;
2577                 num_async_data--;
2578
2579                 list_add_tail(&pasync_data_h->link,
2580                               &pasync_ctx->async_data.free_list);
2581                 pasync_data_h++;
2582                 pasync_ctx->async_data.free_entries++;
2583                 pasync_ctx->async_data.writables++;
2584
2585                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2586         }
2587
2588         pasync_ctx->async_header.host_write_ptr = 0;
2589         pasync_ctx->async_header.ep_read_ptr = -1;
2590         pasync_ctx->async_data.host_write_ptr = 0;
2591         pasync_ctx->async_data.ep_read_ptr = -1;
2592 }
2593
2594 static int
2595 be_sgl_create_contiguous(void *virtual_address,
2596                          u64 physical_address, u32 length,
2597                          struct be_dma_mem *sgl)
2598 {
2599         WARN_ON(!virtual_address);
2600         WARN_ON(!physical_address);
2601         WARN_ON(!length > 0);
2602         WARN_ON(!sgl);
2603
2604         sgl->va = virtual_address;
2605         sgl->dma = (unsigned long)physical_address;
2606         sgl->size = length;
2607
2608         return 0;
2609 }
2610
2611 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2612 {
2613         memset(sgl, 0, sizeof(*sgl));
2614 }
2615
2616 static void
2617 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2618                      struct mem_array *pmem, struct be_dma_mem *sgl)
2619 {
2620         if (sgl->va)
2621                 be_sgl_destroy_contiguous(sgl);
2622
2623         be_sgl_create_contiguous(pmem->virtual_address,
2624                                  pmem->bus_address.u.a64.address,
2625                                  pmem->size, sgl);
2626 }
2627
2628 static void
2629 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2630                            struct mem_array *pmem, struct be_dma_mem *sgl)
2631 {
2632         if (sgl->va)
2633                 be_sgl_destroy_contiguous(sgl);
2634
2635         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2636                                  pmem->bus_address.u.a64.address,
2637                                  pmem->size, sgl);
2638 }
2639
2640 static int be_fill_queue(struct be_queue_info *q,
2641                 u16 len, u16 entry_size, void *vaddress)
2642 {
2643         struct be_dma_mem *mem = &q->dma_mem;
2644
2645         memset(q, 0, sizeof(*q));
2646         q->len = len;
2647         q->entry_size = entry_size;
2648         mem->size = len * entry_size;
2649         mem->va = vaddress;
2650         if (!mem->va)
2651                 return -ENOMEM;
2652         memset(mem->va, 0, mem->size);
2653         return 0;
2654 }
2655
2656 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2657                              struct hwi_context_memory *phwi_context)
2658 {
2659         unsigned int i, num_eq_pages;
2660         int ret, eq_for_mcc;
2661         struct be_queue_info *eq;
2662         struct be_dma_mem *mem;
2663         void *eq_vaddress;
2664         dma_addr_t paddr;
2665
2666         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2667                                       sizeof(struct be_eq_entry));
2668
2669         if (phba->msix_enabled)
2670                 eq_for_mcc = 1;
2671         else
2672                 eq_for_mcc = 0;
2673         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2674                 eq = &phwi_context->be_eq[i].q;
2675                 mem = &eq->dma_mem;
2676                 phwi_context->be_eq[i].phba = phba;
2677                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2678                                                      num_eq_pages * PAGE_SIZE,
2679                                                      &paddr);
2680                 if (!eq_vaddress)
2681                         goto create_eq_error;
2682
2683                 mem->va = eq_vaddress;
2684                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2685                                     sizeof(struct be_eq_entry), eq_vaddress);
2686                 if (ret) {
2687                         shost_printk(KERN_ERR, phba->shost,
2688                                      "be_fill_queue Failed for EQ\n");
2689                         goto create_eq_error;
2690                 }
2691
2692                 mem->dma = paddr;
2693                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2694                                             phwi_context->cur_eqd);
2695                 if (ret) {
2696                         shost_printk(KERN_ERR, phba->shost,
2697                                      "beiscsi_cmd_eq_create"
2698                                      "Failedfor EQ\n");
2699                         goto create_eq_error;
2700                 }
2701                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2702         }
2703         return 0;
2704 create_eq_error:
2705         for (i = 0; i < (phba->num_cpus + 1); i++) {
2706                 eq = &phwi_context->be_eq[i].q;
2707                 mem = &eq->dma_mem;
2708                 if (mem->va)
2709                         pci_free_consistent(phba->pcidev, num_eq_pages
2710                                             * PAGE_SIZE,
2711                                             mem->va, mem->dma);
2712         }
2713         return ret;
2714 }
2715
2716 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2717                              struct hwi_context_memory *phwi_context)
2718 {
2719         unsigned int i, num_cq_pages;
2720         int ret;
2721         struct be_queue_info *cq, *eq;
2722         struct be_dma_mem *mem;
2723         struct be_eq_obj *pbe_eq;
2724         void *cq_vaddress;
2725         dma_addr_t paddr;
2726
2727         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2728                                       sizeof(struct sol_cqe));
2729
2730         for (i = 0; i < phba->num_cpus; i++) {
2731                 cq = &phwi_context->be_cq[i];
2732                 eq = &phwi_context->be_eq[i].q;
2733                 pbe_eq = &phwi_context->be_eq[i];
2734                 pbe_eq->cq = cq;
2735                 pbe_eq->phba = phba;
2736                 mem = &cq->dma_mem;
2737                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2738                                                      num_cq_pages * PAGE_SIZE,
2739                                                      &paddr);
2740                 if (!cq_vaddress)
2741                         goto create_cq_error;
2742                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2743                                     sizeof(struct sol_cqe), cq_vaddress);
2744                 if (ret) {
2745                         shost_printk(KERN_ERR, phba->shost,
2746                                      "be_fill_queue Failed for ISCSI CQ\n");
2747                         goto create_cq_error;
2748                 }
2749
2750                 mem->dma = paddr;
2751                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2752                                             false, 0);
2753                 if (ret) {
2754                         shost_printk(KERN_ERR, phba->shost,
2755                                      "beiscsi_cmd_eq_create"
2756                                      "Failed for ISCSI CQ\n");
2757                         goto create_cq_error;
2758                 }
2759                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2760                                                  cq->id, eq->id);
2761                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2762         }
2763         return 0;
2764
2765 create_cq_error:
2766         for (i = 0; i < phba->num_cpus; i++) {
2767                 cq = &phwi_context->be_cq[i];
2768                 mem = &cq->dma_mem;
2769                 if (mem->va)
2770                         pci_free_consistent(phba->pcidev, num_cq_pages
2771                                             * PAGE_SIZE,
2772                                             mem->va, mem->dma);
2773         }
2774         return ret;
2775
2776 }
2777
2778 static int
2779 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2780                        struct hwi_context_memory *phwi_context,
2781                        struct hwi_controller *phwi_ctrlr,
2782                        unsigned int def_pdu_ring_sz)
2783 {
2784         unsigned int idx;
2785         int ret;
2786         struct be_queue_info *dq, *cq;
2787         struct be_dma_mem *mem;
2788         struct be_mem_descriptor *mem_descr;
2789         void *dq_vaddress;
2790
2791         idx = 0;
2792         dq = &phwi_context->be_def_hdrq;
2793         cq = &phwi_context->be_cq[0];
2794         mem = &dq->dma_mem;
2795         mem_descr = phba->init_mem;
2796         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2797         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2798         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2799                             sizeof(struct phys_addr),
2800                             sizeof(struct phys_addr), dq_vaddress);
2801         if (ret) {
2802                 shost_printk(KERN_ERR, phba->shost,
2803                              "be_fill_queue Failed for DEF PDU HDR\n");
2804                 return ret;
2805         }
2806         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2807                                   bus_address.u.a64.address;
2808         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2809                                               def_pdu_ring_sz,
2810                                               phba->params.defpdu_hdr_sz);
2811         if (ret) {
2812                 shost_printk(KERN_ERR, phba->shost,
2813                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2814                 return ret;
2815         }
2816         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2817         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2818                  phwi_context->be_def_hdrq.id);
2819         hwi_post_async_buffers(phba, 1);
2820         return 0;
2821 }
2822
2823 static int
2824 beiscsi_create_def_data(struct beiscsi_hba *phba,
2825                         struct hwi_context_memory *phwi_context,
2826                         struct hwi_controller *phwi_ctrlr,
2827                         unsigned int def_pdu_ring_sz)
2828 {
2829         unsigned int idx;
2830         int ret;
2831         struct be_queue_info *dataq, *cq;
2832         struct be_dma_mem *mem;
2833         struct be_mem_descriptor *mem_descr;
2834         void *dq_vaddress;
2835
2836         idx = 0;
2837         dataq = &phwi_context->be_def_dataq;
2838         cq = &phwi_context->be_cq[0];
2839         mem = &dataq->dma_mem;
2840         mem_descr = phba->init_mem;
2841         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2842         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2843         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2844                             sizeof(struct phys_addr),
2845                             sizeof(struct phys_addr), dq_vaddress);
2846         if (ret) {
2847                 shost_printk(KERN_ERR, phba->shost,
2848                              "be_fill_queue Failed for DEF PDU DATA\n");
2849                 return ret;
2850         }
2851         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2852                                   bus_address.u.a64.address;
2853         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2854                                               def_pdu_ring_sz,
2855                                               phba->params.defpdu_data_sz);
2856         if (ret) {
2857                 shost_printk(KERN_ERR, phba->shost,
2858                              "be_cmd_create_default_pdu_queue Failed"
2859                              " for DEF PDU DATA\n");
2860                 return ret;
2861         }
2862         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2863         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2864                  phwi_context->be_def_dataq.id);
2865         hwi_post_async_buffers(phba, 0);
2866         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
2867         return 0;
2868 }
2869
2870 static int
2871 beiscsi_post_pages(struct beiscsi_hba *phba)
2872 {
2873         struct be_mem_descriptor *mem_descr;
2874         struct mem_array *pm_arr;
2875         unsigned int page_offset, i;
2876         struct be_dma_mem sgl;
2877         int status;
2878
2879         mem_descr = phba->init_mem;
2880         mem_descr += HWI_MEM_SGE;
2881         pm_arr = mem_descr->mem_array;
2882
2883         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2884                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2885         for (i = 0; i < mem_descr->num_elements; i++) {
2886                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2887                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2888                                                 page_offset,
2889                                                 (pm_arr->size / PAGE_SIZE));
2890                 page_offset += pm_arr->size / PAGE_SIZE;
2891                 if (status != 0) {
2892                         shost_printk(KERN_ERR, phba->shost,
2893                                      "post sgl failed.\n");
2894                         return status;
2895                 }
2896                 pm_arr++;
2897         }
2898         SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
2899         return 0;
2900 }
2901
2902 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2903 {
2904         struct be_dma_mem *mem = &q->dma_mem;
2905         if (mem->va) {
2906                 pci_free_consistent(phba->pcidev, mem->size,
2907                         mem->va, mem->dma);
2908                 mem->va = NULL;
2909         }
2910 }
2911
2912 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2913                 u16 len, u16 entry_size)
2914 {
2915         struct be_dma_mem *mem = &q->dma_mem;
2916
2917         memset(q, 0, sizeof(*q));
2918         q->len = len;
2919         q->entry_size = entry_size;
2920         mem->size = len * entry_size;
2921         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2922         if (!mem->va)
2923                 return -ENOMEM;
2924         memset(mem->va, 0, mem->size);
2925         return 0;
2926 }
2927
2928 static int
2929 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2930                          struct hwi_context_memory *phwi_context,
2931                          struct hwi_controller *phwi_ctrlr)
2932 {
2933         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2934         u64 pa_addr_lo;
2935         unsigned int idx, num, i;
2936         struct mem_array *pwrb_arr;
2937         void *wrb_vaddr;
2938         struct be_dma_mem sgl;
2939         struct be_mem_descriptor *mem_descr;
2940         int status;
2941
2942         idx = 0;
2943         mem_descr = phba->init_mem;
2944         mem_descr += HWI_MEM_WRB;
2945         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2946                            GFP_KERNEL);
2947         if (!pwrb_arr) {
2948                 shost_printk(KERN_ERR, phba->shost,
2949                              "Memory alloc failed in create wrb ring.\n");
2950                 return -ENOMEM;
2951         }
2952         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2953         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2954         num_wrb_rings = mem_descr->mem_array[idx].size /
2955                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2956
2957         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2958                 if (num_wrb_rings) {
2959                         pwrb_arr[num].virtual_address = wrb_vaddr;
2960                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2961                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2962                                             sizeof(struct iscsi_wrb);
2963                         wrb_vaddr += pwrb_arr[num].size;
2964                         pa_addr_lo += pwrb_arr[num].size;
2965                         num_wrb_rings--;
2966                 } else {
2967                         idx++;
2968                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2969                         pa_addr_lo = mem_descr->mem_array[idx].\
2970                                         bus_address.u.a64.address;
2971                         num_wrb_rings = mem_descr->mem_array[idx].size /
2972                                         (phba->params.wrbs_per_cxn *
2973                                         sizeof(struct iscsi_wrb));
2974                         pwrb_arr[num].virtual_address = wrb_vaddr;
2975                         pwrb_arr[num].bus_address.u.a64.address\
2976                                                 = pa_addr_lo;
2977                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2978                                                  sizeof(struct iscsi_wrb);
2979                         wrb_vaddr += pwrb_arr[num].size;
2980                         pa_addr_lo   += pwrb_arr[num].size;
2981                         num_wrb_rings--;
2982                 }
2983         }
2984         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2985                 wrb_mem_index = 0;
2986                 offset = 0;
2987                 size = 0;
2988
2989                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2990                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2991                                             &phwi_context->be_wrbq[i]);
2992                 if (status != 0) {
2993                         shost_printk(KERN_ERR, phba->shost,
2994                                      "wrbq create failed.");
2995                         kfree(pwrb_arr);
2996                         return status;
2997                 }
2998                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2999                                                                    id;
3000         }
3001         kfree(pwrb_arr);
3002         return 0;
3003 }
3004
3005 static void free_wrb_handles(struct beiscsi_hba *phba)
3006 {
3007         unsigned int index;
3008         struct hwi_controller *phwi_ctrlr;
3009         struct hwi_wrb_context *pwrb_context;
3010
3011         phwi_ctrlr = phba->phwi_ctrlr;
3012         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3013                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3014                 kfree(pwrb_context->pwrb_handle_base);
3015                 kfree(pwrb_context->pwrb_handle_basestd);
3016         }
3017 }
3018
3019 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3020 {
3021         struct be_queue_info *q;
3022         struct be_ctrl_info *ctrl = &phba->ctrl;
3023
3024         q = &phba->ctrl.mcc_obj.q;
3025         if (q->created)
3026                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3027         be_queue_free(phba, q);
3028
3029         q = &phba->ctrl.mcc_obj.cq;
3030         if (q->created)
3031                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3032         be_queue_free(phba, q);
3033 }
3034
3035 static void hwi_cleanup(struct beiscsi_hba *phba)
3036 {
3037         struct be_queue_info *q;
3038         struct be_ctrl_info *ctrl = &phba->ctrl;
3039         struct hwi_controller *phwi_ctrlr;
3040         struct hwi_context_memory *phwi_context;
3041         int i, eq_num;
3042
3043         phwi_ctrlr = phba->phwi_ctrlr;
3044         phwi_context = phwi_ctrlr->phwi_ctxt;
3045         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3046                 q = &phwi_context->be_wrbq[i];
3047                 if (q->created)
3048                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3049         }
3050         free_wrb_handles(phba);
3051
3052         q = &phwi_context->be_def_hdrq;
3053         if (q->created)
3054                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3055
3056         q = &phwi_context->be_def_dataq;
3057         if (q->created)
3058                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3059
3060         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3061
3062         for (i = 0; i < (phba->num_cpus); i++) {
3063                 q = &phwi_context->be_cq[i];
3064                 if (q->created)
3065                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3066         }
3067         if (phba->msix_enabled)
3068                 eq_num = 1;
3069         else
3070                 eq_num = 0;
3071         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3072                 q = &phwi_context->be_eq[i].q;
3073                 if (q->created)
3074                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3075         }
3076         be_mcc_queues_destroy(phba);
3077 }
3078
3079 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3080                                 struct hwi_context_memory *phwi_context)
3081 {
3082         struct be_queue_info *q, *cq;
3083         struct be_ctrl_info *ctrl = &phba->ctrl;
3084
3085         /* Alloc MCC compl queue */
3086         cq = &phba->ctrl.mcc_obj.cq;
3087         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3088                         sizeof(struct be_mcc_compl)))
3089                 goto err;
3090         /* Ask BE to create MCC compl queue; */
3091         if (phba->msix_enabled) {
3092                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3093                                          [phba->num_cpus].q, false, true, 0))
3094                 goto mcc_cq_free;
3095         } else {
3096                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3097                                           false, true, 0))
3098                 goto mcc_cq_free;
3099         }
3100
3101         /* Alloc MCC queue */
3102         q = &phba->ctrl.mcc_obj.q;
3103         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3104                 goto mcc_cq_destroy;
3105
3106         /* Ask BE to create MCC queue */
3107         if (beiscsi_cmd_mccq_create(phba, q, cq))
3108                 goto mcc_q_free;
3109
3110         return 0;
3111
3112 mcc_q_free:
3113         be_queue_free(phba, q);
3114 mcc_cq_destroy:
3115         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3116 mcc_cq_free:
3117         be_queue_free(phba, cq);
3118 err:
3119         return -ENOMEM;
3120 }
3121
3122 static int find_num_cpus(void)
3123 {
3124         int  num_cpus = 0;
3125
3126         num_cpus = num_online_cpus();
3127         if (num_cpus >= MAX_CPUS)
3128                 num_cpus = MAX_CPUS - 1;
3129
3130         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
3131         return num_cpus;
3132 }
3133
3134 static int hwi_init_port(struct beiscsi_hba *phba)
3135 {
3136         struct hwi_controller *phwi_ctrlr;
3137         struct hwi_context_memory *phwi_context;
3138         unsigned int def_pdu_ring_sz;
3139         struct be_ctrl_info *ctrl = &phba->ctrl;
3140         int status;
3141
3142         def_pdu_ring_sz =
3143                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3144         phwi_ctrlr = phba->phwi_ctrlr;
3145         phwi_context = phwi_ctrlr->phwi_ctxt;
3146         phwi_context->max_eqd = 0;
3147         phwi_context->min_eqd = 0;
3148         phwi_context->cur_eqd = 64;
3149         be_cmd_fw_initialize(&phba->ctrl);
3150
3151         status = beiscsi_create_eqs(phba, phwi_context);
3152         if (status != 0) {
3153                 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
3154                 goto error;
3155         }
3156
3157         status = be_mcc_queues_create(phba, phwi_context);
3158         if (status != 0)
3159                 goto error;
3160
3161         status = mgmt_check_supported_fw(ctrl, phba);
3162         if (status != 0) {
3163                 shost_printk(KERN_ERR, phba->shost,
3164                              "Unsupported fw version\n");
3165                 goto error;
3166         }
3167
3168         status = beiscsi_create_cqs(phba, phwi_context);
3169         if (status != 0) {
3170                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3171                 goto error;
3172         }
3173
3174         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3175                                         def_pdu_ring_sz);
3176         if (status != 0) {
3177                 shost_printk(KERN_ERR, phba->shost,
3178                              "Default Header not created\n");
3179                 goto error;
3180         }
3181
3182         status = beiscsi_create_def_data(phba, phwi_context,
3183                                          phwi_ctrlr, def_pdu_ring_sz);
3184         if (status != 0) {
3185                 shost_printk(KERN_ERR, phba->shost,
3186                              "Default Data not created\n");
3187                 goto error;
3188         }
3189
3190         status = beiscsi_post_pages(phba);
3191         if (status != 0) {
3192                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3193                 goto error;
3194         }
3195
3196         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3197         if (status != 0) {
3198                 shost_printk(KERN_ERR, phba->shost,
3199                              "WRB Rings not created\n");
3200                 goto error;
3201         }
3202
3203         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3204         return 0;
3205
3206 error:
3207         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3208         hwi_cleanup(phba);
3209         return status;
3210 }
3211
3212 static int hwi_init_controller(struct beiscsi_hba *phba)
3213 {
3214         struct hwi_controller *phwi_ctrlr;
3215
3216         phwi_ctrlr = phba->phwi_ctrlr;
3217         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3218                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3219                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3220                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
3221                          phwi_ctrlr->phwi_ctxt);
3222         } else {
3223                 shost_printk(KERN_ERR, phba->shost,
3224                              "HWI_MEM_ADDN_CONTEXT is more than one element."
3225                              "Failing to load\n");
3226                 return -ENOMEM;
3227         }
3228
3229         iscsi_init_global_templates(phba);
3230         if (beiscsi_init_wrb_handle(phba))
3231                 return -ENOMEM;
3232
3233         hwi_init_async_pdu_ctx(phba);
3234         if (hwi_init_port(phba) != 0) {
3235                 shost_printk(KERN_ERR, phba->shost,
3236                              "hwi_init_controller failed\n");
3237                 return -ENOMEM;
3238         }
3239         return 0;
3240 }
3241
3242 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3243 {
3244         struct be_mem_descriptor *mem_descr;
3245         int i, j;
3246
3247         mem_descr = phba->init_mem;
3248         i = 0;
3249         j = 0;
3250         for (i = 0; i < SE_MEM_MAX; i++) {
3251                 for (j = mem_descr->num_elements; j > 0; j--) {
3252                         pci_free_consistent(phba->pcidev,
3253                           mem_descr->mem_array[j - 1].size,
3254                           mem_descr->mem_array[j - 1].virtual_address,
3255                           (unsigned long)mem_descr->mem_array[j - 1].
3256                           bus_address.u.a64.address);
3257                 }
3258                 kfree(mem_descr->mem_array);
3259                 mem_descr++;
3260         }
3261         kfree(phba->init_mem);
3262         kfree(phba->phwi_ctrlr);
3263 }
3264
3265 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3266 {
3267         int ret = -ENOMEM;
3268
3269         ret = beiscsi_get_memory(phba);
3270         if (ret < 0) {
3271                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3272                              "Failed in beiscsi_alloc_memory\n");
3273                 return ret;
3274         }
3275
3276         ret = hwi_init_controller(phba);
3277         if (ret)
3278                 goto free_init;
3279         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3280         return 0;
3281
3282 free_init:
3283         beiscsi_free_mem(phba);
3284         return ret;
3285 }
3286
3287 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3288 {
3289         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3290         struct sgl_handle *psgl_handle;
3291         struct iscsi_sge *pfrag;
3292         unsigned int arr_index, i, idx;
3293
3294         phba->io_sgl_hndl_avbl = 0;
3295         phba->eh_sgl_hndl_avbl = 0;
3296
3297         mem_descr_sglh = phba->init_mem;
3298         mem_descr_sglh += HWI_MEM_SGLH;
3299         if (1 == mem_descr_sglh->num_elements) {
3300                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3301                                                  phba->params.ios_per_ctrl,
3302                                                  GFP_KERNEL);
3303                 if (!phba->io_sgl_hndl_base) {
3304                         shost_printk(KERN_ERR, phba->shost,
3305                                      "Mem Alloc Failed. Failing to load\n");
3306                         return -ENOMEM;
3307                 }
3308                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3309                                                  (phba->params.icds_per_ctrl -
3310                                                  phba->params.ios_per_ctrl),
3311                                                  GFP_KERNEL);
3312                 if (!phba->eh_sgl_hndl_base) {
3313                         kfree(phba->io_sgl_hndl_base);
3314                         shost_printk(KERN_ERR, phba->shost,
3315                                      "Mem Alloc Failed. Failing to load\n");
3316                         return -ENOMEM;
3317                 }
3318         } else {
3319                 shost_printk(KERN_ERR, phba->shost,
3320                              "HWI_MEM_SGLH is more than one element."
3321                              "Failing to load\n");
3322                 return -ENOMEM;
3323         }
3324
3325         arr_index = 0;
3326         idx = 0;
3327         while (idx < mem_descr_sglh->num_elements) {
3328                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3329
3330                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3331                       sizeof(struct sgl_handle)); i++) {
3332                         if (arr_index < phba->params.ios_per_ctrl) {
3333                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3334                                 phba->io_sgl_hndl_avbl++;
3335                                 arr_index++;
3336                         } else {
3337                                 phba->eh_sgl_hndl_base[arr_index -
3338                                         phba->params.ios_per_ctrl] =
3339                                                                 psgl_handle;
3340                                 arr_index++;
3341                                 phba->eh_sgl_hndl_avbl++;
3342                         }
3343                         psgl_handle++;
3344                 }
3345                 idx++;
3346         }
3347         SE_DEBUG(DBG_LVL_8,
3348                  "phba->io_sgl_hndl_avbl=%d"
3349                  "phba->eh_sgl_hndl_avbl=%d\n",
3350                  phba->io_sgl_hndl_avbl,
3351                  phba->eh_sgl_hndl_avbl);
3352         mem_descr_sg = phba->init_mem;
3353         mem_descr_sg += HWI_MEM_SGE;
3354         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
3355                  mem_descr_sg->num_elements);
3356         arr_index = 0;
3357         idx = 0;
3358         while (idx < mem_descr_sg->num_elements) {
3359                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3360
3361                 for (i = 0;
3362                      i < (mem_descr_sg->mem_array[idx].size) /
3363                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3364                      i++) {
3365                         if (arr_index < phba->params.ios_per_ctrl)
3366                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3367                         else
3368                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3369                                                 phba->params.ios_per_ctrl];
3370                         psgl_handle->pfrag = pfrag;
3371                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3372                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3373                         pfrag += phba->params.num_sge_per_io;
3374                         psgl_handle->sgl_index =
3375                                 phba->fw_config.iscsi_icd_start + arr_index++;
3376                 }
3377                 idx++;
3378         }
3379         phba->io_sgl_free_index = 0;
3380         phba->io_sgl_alloc_index = 0;
3381         phba->eh_sgl_free_index = 0;
3382         phba->eh_sgl_alloc_index = 0;
3383         return 0;
3384 }
3385
3386 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3387 {
3388         int i, new_cid;
3389
3390         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3391                                   GFP_KERNEL);
3392         if (!phba->cid_array) {
3393                 shost_printk(KERN_ERR, phba->shost,
3394                              "Failed to allocate memory in "
3395                              "hba_setup_cid_tbls\n");
3396                 return -ENOMEM;
3397         }
3398         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3399                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3400         if (!phba->ep_array) {
3401                 shost_printk(KERN_ERR, phba->shost,
3402                              "Failed to allocate memory in "
3403                              "hba_setup_cid_tbls\n");
3404                 kfree(phba->cid_array);
3405                 return -ENOMEM;
3406         }
3407         new_cid = phba->fw_config.iscsi_cid_start;
3408         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3409                 phba->cid_array[i] = new_cid;
3410                 new_cid += 2;
3411         }
3412         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3413         return 0;
3414 }
3415
3416 static void hwi_enable_intr(struct beiscsi_hba *phba)
3417 {
3418         struct be_ctrl_info *ctrl = &phba->ctrl;
3419         struct hwi_controller *phwi_ctrlr;
3420         struct hwi_context_memory *phwi_context;
3421         struct be_queue_info *eq;
3422         u8 __iomem *addr;
3423         u32 reg, i;
3424         u32 enabled;
3425
3426         phwi_ctrlr = phba->phwi_ctrlr;
3427         phwi_context = phwi_ctrlr->phwi_ctxt;
3428
3429         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3430                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3431         reg = ioread32(addr);
3432
3433         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3434         if (!enabled) {
3435                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3436                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
3437                 iowrite32(reg, addr);
3438         }
3439
3440         if (!phba->msix_enabled) {
3441                 eq = &phwi_context->be_eq[0].q;
3442                 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3443                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3444         } else {
3445                 for (i = 0; i <= phba->num_cpus; i++) {
3446                         eq = &phwi_context->be_eq[i].q;
3447                         SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3448                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3449                 }
3450         }
3451 }
3452
3453 static void hwi_disable_intr(struct beiscsi_hba *phba)
3454 {
3455         struct be_ctrl_info *ctrl = &phba->ctrl;
3456
3457         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3458         u32 reg = ioread32(addr);
3459
3460         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3461         if (enabled) {
3462                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3463                 iowrite32(reg, addr);
3464         } else
3465                 shost_printk(KERN_WARNING, phba->shost,
3466                              "In hwi_disable_intr, Already Disabled\n");
3467 }
3468
3469 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3470 {
3471         struct be_cmd_get_boot_target_resp *boot_resp;
3472         struct be_cmd_get_session_resp *session_resp;
3473         struct be_mcc_wrb *wrb;
3474         struct be_dma_mem nonemb_cmd;
3475         unsigned int tag, wrb_num;
3476         unsigned short status, extd_status;
3477         struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3478         int ret = -ENOMEM;
3479
3480         tag = mgmt_get_boot_target(phba);
3481         if (!tag) {
3482                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed\n");
3483                 return -EAGAIN;
3484         } else
3485                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3486                                          phba->ctrl.mcc_numtag[tag]);
3487
3488         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3489         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3490         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3491         if (status || extd_status) {
3492                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed"
3493                                     " status = %d extd_status = %d\n",
3494                                     status, extd_status);
3495                 free_mcc_tag(&phba->ctrl, tag);
3496                 return -EBUSY;
3497         }
3498         wrb = queue_get_wrb(mccq, wrb_num);
3499         free_mcc_tag(&phba->ctrl, tag);
3500         boot_resp = embedded_payload(wrb);
3501
3502         if (boot_resp->boot_session_handle < 0) {
3503                 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
3504                 return -ENXIO;
3505         }
3506
3507         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3508                                 sizeof(*session_resp),
3509                                 &nonemb_cmd.dma);
3510         if (nonemb_cmd.va == NULL) {
3511                 SE_DEBUG(DBG_LVL_1,
3512                          "Failed to allocate memory for"
3513                          "beiscsi_get_session_info\n");
3514                 return -ENOMEM;
3515         }
3516
3517         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3518         tag = mgmt_get_session_info(phba, boot_resp->boot_session_handle,
3519                                     &nonemb_cmd);
3520         if (!tag) {
3521                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3522                         " Failed\n");
3523                 goto boot_freemem;
3524         } else
3525                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3526                                          phba->ctrl.mcc_numtag[tag]);
3527
3528         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3529         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3530         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3531         if (status || extd_status) {
3532                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3533                                     " status = %d extd_status = %d\n",
3534                                     status, extd_status);
3535                 free_mcc_tag(&phba->ctrl, tag);
3536                 goto boot_freemem;
3537         }
3538         wrb = queue_get_wrb(mccq, wrb_num);
3539         free_mcc_tag(&phba->ctrl, tag);
3540         session_resp = nonemb_cmd.va ;
3541
3542         memcpy(&phba->boot_sess, &session_resp->session_info,
3543                sizeof(struct mgmt_session_info));
3544         ret = 0;
3545
3546 boot_freemem:
3547         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3548                     nonemb_cmd.va, nonemb_cmd.dma);
3549         return ret;
3550 }
3551
3552 static void beiscsi_boot_release(void *data)
3553 {
3554         struct beiscsi_hba *phba = data;
3555
3556         scsi_host_put(phba->shost);
3557 }
3558
3559 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3560 {
3561         struct iscsi_boot_kobj *boot_kobj;
3562
3563         /* get boot info using mgmt cmd */
3564         if (beiscsi_get_boot_info(phba))
3565                 /* Try to see if we can carry on without this */
3566                 return 0;
3567
3568         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3569         if (!phba->boot_kset)
3570                 return -ENOMEM;
3571
3572         /* get a ref because the show function will ref the phba */
3573         if (!scsi_host_get(phba->shost))
3574                 goto free_kset;
3575         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3576                                              beiscsi_show_boot_tgt_info,
3577                                              beiscsi_tgt_get_attr_visibility,
3578                                              beiscsi_boot_release);
3579         if (!boot_kobj)
3580                 goto put_shost;
3581
3582         if (!scsi_host_get(phba->shost))
3583                 goto free_kset;
3584         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3585                                                 beiscsi_show_boot_ini_info,
3586                                                 beiscsi_ini_get_attr_visibility,
3587                                                 beiscsi_boot_release);
3588         if (!boot_kobj)
3589                 goto put_shost;
3590
3591         if (!scsi_host_get(phba->shost))
3592                 goto free_kset;
3593         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3594                                                beiscsi_show_boot_eth_info,
3595                                                beiscsi_eth_get_attr_visibility,
3596                                                beiscsi_boot_release);
3597         if (!boot_kobj)
3598                 goto put_shost;
3599         return 0;
3600
3601 put_shost:
3602         scsi_host_put(phba->shost);
3603 free_kset:
3604         iscsi_boot_destroy_kset(phba->boot_kset);
3605         return -ENOMEM;
3606 }
3607
3608 static int beiscsi_init_port(struct beiscsi_hba *phba)
3609 {
3610         int ret;
3611
3612         ret = beiscsi_init_controller(phba);
3613         if (ret < 0) {
3614                 shost_printk(KERN_ERR, phba->shost,
3615                              "beiscsi_dev_probe - Failed in"
3616                              "beiscsi_init_controller\n");
3617                 return ret;
3618         }
3619         ret = beiscsi_init_sgl_handle(phba);
3620         if (ret < 0) {
3621                 shost_printk(KERN_ERR, phba->shost,
3622                              "beiscsi_dev_probe - Failed in"
3623                              "beiscsi_init_sgl_handle\n");
3624                 goto do_cleanup_ctrlr;
3625         }
3626
3627         if (hba_setup_cid_tbls(phba)) {
3628                 shost_printk(KERN_ERR, phba->shost,
3629                              "Failed in hba_setup_cid_tbls\n");
3630                 kfree(phba->io_sgl_hndl_base);
3631                 kfree(phba->eh_sgl_hndl_base);
3632                 goto do_cleanup_ctrlr;
3633         }
3634
3635         return ret;
3636
3637 do_cleanup_ctrlr:
3638         hwi_cleanup(phba);
3639         return ret;
3640 }
3641
3642 static void hwi_purge_eq(struct beiscsi_hba *phba)
3643 {
3644         struct hwi_controller *phwi_ctrlr;
3645         struct hwi_context_memory *phwi_context;
3646         struct be_queue_info *eq;
3647         struct be_eq_entry *eqe = NULL;
3648         int i, eq_msix;
3649         unsigned int num_processed;
3650
3651         phwi_ctrlr = phba->phwi_ctrlr;
3652         phwi_context = phwi_ctrlr->phwi_ctxt;
3653         if (phba->msix_enabled)
3654                 eq_msix = 1;
3655         else
3656                 eq_msix = 0;
3657
3658         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3659                 eq = &phwi_context->be_eq[i].q;
3660                 eqe = queue_tail_node(eq);
3661                 num_processed = 0;
3662                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3663                                         & EQE_VALID_MASK) {
3664                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3665                         queue_tail_inc(eq);
3666                         eqe = queue_tail_node(eq);
3667                         num_processed++;
3668                 }
3669
3670                 if (num_processed)
3671                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3672         }
3673 }
3674
3675 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3676 {
3677         int mgmt_status;
3678
3679         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3680         if (mgmt_status)
3681                 shost_printk(KERN_WARNING, phba->shost,
3682                              "mgmt_epfw_cleanup FAILED\n");
3683
3684         hwi_purge_eq(phba);
3685         hwi_cleanup(phba);
3686         kfree(phba->io_sgl_hndl_base);
3687         kfree(phba->eh_sgl_hndl_base);
3688         kfree(phba->cid_array);
3689         kfree(phba->ep_array);
3690 }
3691
3692 static void beiscsi_cleanup_task(struct iscsi_task *task)
3693 {
3694         struct beiscsi_io_task *io_task = task->dd_data;
3695         struct iscsi_conn *conn = task->conn;
3696         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3697         struct beiscsi_hba *phba = beiscsi_conn->phba;
3698         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3699         struct hwi_wrb_context *pwrb_context;
3700         struct hwi_controller *phwi_ctrlr;
3701
3702         phwi_ctrlr = phba->phwi_ctrlr;
3703         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3704                         - phba->fw_config.iscsi_cid_start];
3705
3706         if (io_task->cmd_bhs) {
3707                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3708                               io_task->bhs_pa.u.a64.address);
3709                 io_task->cmd_bhs = NULL;
3710         }
3711
3712         if (task->sc) {
3713                 if (io_task->pwrb_handle) {
3714                         free_wrb_handle(phba, pwrb_context,
3715                                         io_task->pwrb_handle);
3716                         io_task->pwrb_handle = NULL;
3717                 }
3718
3719                 if (io_task->psgl_handle) {
3720                         spin_lock(&phba->io_sgl_lock);
3721                         free_io_sgl_handle(phba, io_task->psgl_handle);
3722                         spin_unlock(&phba->io_sgl_lock);
3723                         io_task->psgl_handle = NULL;
3724                 }
3725         } else {
3726                 if (!beiscsi_conn->login_in_progress) {
3727                         if (io_task->pwrb_handle) {
3728                                 free_wrb_handle(phba, pwrb_context,
3729                                                 io_task->pwrb_handle);
3730                                 io_task->pwrb_handle = NULL;
3731                         }
3732                         if (io_task->psgl_handle) {
3733                                 spin_lock(&phba->mgmt_sgl_lock);
3734                                 free_mgmt_sgl_handle(phba,
3735                                                      io_task->psgl_handle);
3736                                 spin_unlock(&phba->mgmt_sgl_lock);
3737                                 io_task->psgl_handle = NULL;
3738                         }
3739                 }
3740         }
3741 }
3742
3743 void
3744 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3745                            struct beiscsi_offload_params *params)
3746 {
3747         struct wrb_handle *pwrb_handle;
3748         struct iscsi_target_context_update_wrb *pwrb = NULL;
3749         struct be_mem_descriptor *mem_descr;
3750         struct beiscsi_hba *phba = beiscsi_conn->phba;
3751         struct iscsi_task *task = beiscsi_conn->task;
3752         struct iscsi_session *session = task->conn->session;
3753         u32 doorbell = 0;
3754
3755         /*
3756          * We can always use 0 here because it is reserved by libiscsi for
3757          * login/startup related tasks.
3758          */
3759         beiscsi_conn->login_in_progress = 0;
3760         spin_lock_bh(&session->lock);
3761         beiscsi_cleanup_task(task);
3762         spin_unlock_bh(&session->lock);
3763
3764         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3765                                        phba->fw_config.iscsi_cid_start));
3766         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3767         memset(pwrb, 0, sizeof(*pwrb));
3768         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3769                       max_burst_length, pwrb, params->dw[offsetof
3770                       (struct amap_beiscsi_offload_params,
3771                       max_burst_length) / 32]);
3772         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3773                       max_send_data_segment_length, pwrb,
3774                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3775                       max_send_data_segment_length) / 32]);
3776         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3777                       first_burst_length,
3778                       pwrb,
3779                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3780                       first_burst_length) / 32]);
3781
3782         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3783                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3784                       erl) / 32] & OFFLD_PARAMS_ERL));
3785         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3786                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3787                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3788         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3789                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3790                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3791         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3792                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3793                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3794         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3795                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3796                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3797         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3798                       pwrb,
3799                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3800                       exp_statsn) / 32] + 1));
3801         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3802                       0x7);
3803         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3804                       pwrb, pwrb_handle->wrb_index);
3805         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3806                       pwrb, pwrb_handle->nxt_wrb_index);
3807         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3808                         session_state, pwrb, 0);
3809         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3810                       pwrb, 1);
3811         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3812                       pwrb, 0);
3813         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3814                       0);
3815
3816         mem_descr = phba->init_mem;
3817         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3818
3819         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3820                         pad_buffer_addr_hi, pwrb,
3821                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3822         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3823                         pad_buffer_addr_lo, pwrb,
3824                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3825
3826         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3827
3828         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3829         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3830                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3831         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3832
3833         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3834 }
3835
3836 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3837                               int *index, int *age)
3838 {
3839         *index = (int)itt;
3840         if (age)
3841                 *age = conn->session->age;
3842 }
3843
3844 /**
3845  * beiscsi_alloc_pdu - allocates pdu and related resources
3846  * @task: libiscsi task
3847  * @opcode: opcode of pdu for task
3848  *
3849  * This is called with the session lock held. It will allocate
3850  * the wrb and sgl if needed for the command. And it will prep
3851  * the pdu's itt. beiscsi_parse_pdu will later translate
3852  * the pdu itt to the libiscsi task itt.
3853  */
3854 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3855 {
3856         struct beiscsi_io_task *io_task = task->dd_data;
3857         struct iscsi_conn *conn = task->conn;
3858         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3859         struct beiscsi_hba *phba = beiscsi_conn->phba;
3860         struct hwi_wrb_context *pwrb_context;
3861         struct hwi_controller *phwi_ctrlr;
3862         itt_t itt;
3863         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3864         dma_addr_t paddr;
3865
3866         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3867                                           GFP_ATOMIC, &paddr);
3868         if (!io_task->cmd_bhs)
3869                 return -ENOMEM;
3870         io_task->bhs_pa.u.a64.address = paddr;
3871         io_task->libiscsi_itt = (itt_t)task->itt;
3872         io_task->conn = beiscsi_conn;
3873
3874         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3875         task->hdr_max = sizeof(struct be_cmd_bhs);
3876         io_task->psgl_handle = NULL;
3877         io_task->pwrb_handle = NULL;
3878
3879         if (task->sc) {
3880                 spin_lock(&phba->io_sgl_lock);
3881                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3882                 spin_unlock(&phba->io_sgl_lock);
3883                 if (!io_task->psgl_handle)
3884                         goto free_hndls;
3885                 io_task->pwrb_handle = alloc_wrb_handle(phba,
3886                                         beiscsi_conn->beiscsi_conn_cid -
3887                                         phba->fw_config.iscsi_cid_start);
3888                 if (!io_task->pwrb_handle)
3889                         goto free_io_hndls;
3890         } else {
3891                 io_task->scsi_cmnd = NULL;
3892                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3893                         if (!beiscsi_conn->login_in_progress) {
3894                                 spin_lock(&phba->mgmt_sgl_lock);
3895                                 io_task->psgl_handle = (struct sgl_handle *)
3896                                                 alloc_mgmt_sgl_handle(phba);
3897                                 spin_unlock(&phba->mgmt_sgl_lock);
3898                                 if (!io_task->psgl_handle)
3899                                         goto free_hndls;
3900
3901                                 beiscsi_conn->login_in_progress = 1;
3902                                 beiscsi_conn->plogin_sgl_handle =
3903                                                         io_task->psgl_handle;
3904                                 io_task->pwrb_handle =
3905                                         alloc_wrb_handle(phba,
3906                                         beiscsi_conn->beiscsi_conn_cid -
3907                                         phba->fw_config.iscsi_cid_start);
3908                                 if (!io_task->pwrb_handle)
3909                                         goto free_io_hndls;
3910                                 beiscsi_conn->plogin_wrb_handle =
3911                                                         io_task->pwrb_handle;
3912
3913                         } else {
3914                                 io_task->psgl_handle =
3915                                                 beiscsi_conn->plogin_sgl_handle;
3916                                 io_task->pwrb_handle =
3917                                                 beiscsi_conn->plogin_wrb_handle;
3918                         }
3919                         beiscsi_conn->task = task;
3920                 } else {
3921                         spin_lock(&phba->mgmt_sgl_lock);
3922                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3923                         spin_unlock(&phba->mgmt_sgl_lock);
3924                         if (!io_task->psgl_handle)
3925                                 goto free_hndls;
3926                         io_task->pwrb_handle =
3927                                         alloc_wrb_handle(phba,
3928                                         beiscsi_conn->beiscsi_conn_cid -
3929                                         phba->fw_config.iscsi_cid_start);
3930                         if (!io_task->pwrb_handle)
3931                                 goto free_mgmt_hndls;
3932
3933                 }
3934         }
3935         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3936                                  wrb_index << 16) | (unsigned int)
3937                                 (io_task->psgl_handle->sgl_index));
3938         io_task->pwrb_handle->pio_handle = task;
3939
3940         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3941         return 0;
3942
3943 free_io_hndls:
3944         spin_lock(&phba->io_sgl_lock);
3945         free_io_sgl_handle(phba, io_task->psgl_handle);
3946         spin_unlock(&phba->io_sgl_lock);
3947         goto free_hndls;
3948 free_mgmt_hndls:
3949         spin_lock(&phba->mgmt_sgl_lock);
3950         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3951         spin_unlock(&phba->mgmt_sgl_lock);
3952 free_hndls:
3953         phwi_ctrlr = phba->phwi_ctrlr;
3954         pwrb_context = &phwi_ctrlr->wrb_context[
3955                         beiscsi_conn->beiscsi_conn_cid -
3956                         phba->fw_config.iscsi_cid_start];
3957         if (io_task->pwrb_handle)
3958                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3959         io_task->pwrb_handle = NULL;
3960         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3961                       io_task->bhs_pa.u.a64.address);
3962         io_task->cmd_bhs = NULL;
3963         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
3964         return -ENOMEM;
3965 }
3966
3967 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3968                           unsigned int num_sg, unsigned int xferlen,
3969                           unsigned int writedir)
3970 {
3971
3972         struct beiscsi_io_task *io_task = task->dd_data;
3973         struct iscsi_conn *conn = task->conn;
3974         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3975         struct beiscsi_hba *phba = beiscsi_conn->phba;
3976         struct iscsi_wrb *pwrb = NULL;
3977         unsigned int doorbell = 0;
3978
3979         pwrb = io_task->pwrb_handle->pwrb;
3980         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3981         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3982
3983         if (writedir) {
3984                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3985                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3986                               &io_task->cmd_bhs->iscsi_data_pdu,
3987                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3988                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3989                               &io_task->cmd_bhs->iscsi_data_pdu,
3990                               ISCSI_OPCODE_SCSI_DATA_OUT);
3991                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3992                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3993                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3994                               INI_WR_CMD);
3995                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3996         } else {
3997                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3998                               INI_RD_CMD);
3999                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4000         }
4001         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
4002                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
4003                &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
4004
4005         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4006                       cpu_to_be16(*(unsigned short *)
4007                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4008         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4009         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4010                       io_task->pwrb_handle->wrb_index);
4011         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4012                       be32_to_cpu(task->cmdsn));
4013         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4014                       io_task->psgl_handle->sgl_index);
4015
4016         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4017
4018         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4019                       io_task->pwrb_handle->nxt_wrb_index);
4020         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4021
4022         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4023         doorbell |= (io_task->pwrb_handle->wrb_index &
4024                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4025         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4026
4027         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4028         return 0;
4029 }
4030
4031 static int beiscsi_mtask(struct iscsi_task *task)
4032 {
4033         struct beiscsi_io_task *io_task = task->dd_data;
4034         struct iscsi_conn *conn = task->conn;
4035         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4036         struct beiscsi_hba *phba = beiscsi_conn->phba;
4037         struct iscsi_wrb *pwrb = NULL;
4038         unsigned int doorbell = 0;
4039         unsigned int cid;
4040
4041         cid = beiscsi_conn->beiscsi_conn_cid;
4042         pwrb = io_task->pwrb_handle->pwrb;
4043         memset(pwrb, 0, sizeof(*pwrb));
4044         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4045                       be32_to_cpu(task->cmdsn));
4046         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4047                       io_task->pwrb_handle->wrb_index);
4048         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4049                       io_task->psgl_handle->sgl_index);
4050
4051         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4052         case ISCSI_OP_LOGIN:
4053                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4054                               TGT_DM_CMD);
4055                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4056                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4057                 hwi_write_buffer(pwrb, task);
4058                 break;
4059         case ISCSI_OP_NOOP_OUT:
4060                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4061                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4062                                       TGT_DM_CMD);
4063                         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4064                                       pwrb, 0);
4065                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
4066                 } else {
4067                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4068                                       INI_RD_CMD);
4069                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4070                 }
4071                 hwi_write_buffer(pwrb, task);
4072                 break;
4073         case ISCSI_OP_TEXT:
4074                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4075                               TGT_DM_CMD);
4076                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4077                 hwi_write_buffer(pwrb, task);
4078                 break;
4079         case ISCSI_OP_SCSI_TMFUNC:
4080                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4081                               INI_TMF_CMD);
4082                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4083                 hwi_write_buffer(pwrb, task);
4084                 break;
4085         case ISCSI_OP_LOGOUT:
4086                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4087                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4088                               HWH_TYPE_LOGOUT);
4089                 hwi_write_buffer(pwrb, task);
4090                 break;
4091
4092         default:
4093                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
4094                          task->hdr->opcode & ISCSI_OPCODE_MASK);
4095                 return -EINVAL;
4096         }
4097
4098         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4099                       task->data_count);
4100         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4101                       io_task->pwrb_handle->nxt_wrb_index);
4102         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4103
4104         doorbell |= cid & DB_WRB_POST_CID_MASK;
4105         doorbell |= (io_task->pwrb_handle->wrb_index &
4106                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4107         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4108         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4109         return 0;
4110 }
4111
4112 static int beiscsi_task_xmit(struct iscsi_task *task)
4113 {
4114         struct beiscsi_io_task *io_task = task->dd_data;
4115         struct scsi_cmnd *sc = task->sc;
4116         struct scatterlist *sg;
4117         int num_sg;
4118         unsigned int  writedir = 0, xferlen = 0;
4119
4120         if (!sc)
4121                 return beiscsi_mtask(task);
4122
4123         io_task->scsi_cmnd = sc;
4124         num_sg = scsi_dma_map(sc);
4125         if (num_sg < 0) {
4126                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4127                 return num_sg;
4128         }
4129         xferlen = scsi_bufflen(sc);
4130         sg = scsi_sglist(sc);
4131         if (sc->sc_data_direction == DMA_TO_DEVICE) {
4132                 writedir = 1;
4133                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
4134                          task->imm_count);
4135         } else
4136                 writedir = 0;
4137         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4138 }
4139
4140 /**
4141  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4142  * @job: job to handle
4143  */
4144 static int beiscsi_bsg_request(struct bsg_job *job)
4145 {
4146         struct Scsi_Host *shost;
4147         struct beiscsi_hba *phba;
4148         struct iscsi_bsg_request *bsg_req = job->request;
4149         int rc = -EINVAL;
4150         unsigned int tag;
4151         struct be_dma_mem nonemb_cmd;
4152         struct be_cmd_resp_hdr *resp;
4153         struct iscsi_bsg_reply *bsg_reply = job->reply;
4154         unsigned short status, extd_status;
4155
4156         shost = iscsi_job_to_shost(job);
4157         phba = iscsi_host_priv(shost);
4158
4159         switch (bsg_req->msgcode) {
4160         case ISCSI_BSG_HST_VENDOR:
4161                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4162                                         job->request_payload.payload_len,
4163                                         &nonemb_cmd.dma);
4164                 if (nonemb_cmd.va == NULL) {
4165                         SE_DEBUG(DBG_LVL_1, "Failed to allocate memory for "
4166                                  "beiscsi_bsg_request\n");
4167                         return -EIO;
4168                 }
4169                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4170                                                   &nonemb_cmd);
4171                 if (!tag) {
4172                         SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
4173                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4174                                             nonemb_cmd.va, nonemb_cmd.dma);
4175                         return -EAGAIN;
4176                 } else
4177                         wait_event_interruptible(phba->ctrl.mcc_wait[tag],
4178                                                  phba->ctrl.mcc_numtag[tag]);
4179                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4180                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4181                 free_mcc_tag(&phba->ctrl, tag);
4182                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4183                 sg_copy_from_buffer(job->reply_payload.sg_list,
4184                                     job->reply_payload.sg_cnt,
4185                                     nonemb_cmd.va, (resp->response_length
4186                                     + sizeof(*resp)));
4187                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4188                 bsg_reply->result = status;
4189                 bsg_job_done(job, bsg_reply->result,
4190                              bsg_reply->reply_payload_rcv_len);
4191                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4192                                     nonemb_cmd.va, nonemb_cmd.dma);
4193                 if (status || extd_status) {
4194                         SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
4195                                  " status = %d extd_status = %d\n",
4196                                  status, extd_status);
4197                         return -EIO;
4198                 }
4199                 break;
4200
4201         default:
4202                 SE_DEBUG(DBG_LVL_1, "Unsupported bsg command: 0x%x\n",
4203                          bsg_req->msgcode);
4204                 break;
4205         }
4206
4207         return rc;
4208 }
4209
4210 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4211 {
4212         struct hwi_controller *phwi_ctrlr;
4213         struct hwi_context_memory *phwi_context;
4214         struct be_eq_obj *pbe_eq;
4215         unsigned int i, msix_vec;
4216         u8 *real_offset = 0;
4217         u32 value = 0;
4218
4219         phwi_ctrlr = phba->phwi_ctrlr;
4220         phwi_context = phwi_ctrlr->phwi_ctxt;
4221         hwi_disable_intr(phba);
4222         if (phba->msix_enabled) {
4223                 for (i = 0; i <= phba->num_cpus; i++) {
4224                         msix_vec = phba->msix_entries[i].vector;
4225                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4226                         kfree(phba->msi_name[i]);
4227                 }
4228         } else
4229                 if (phba->pcidev->irq)
4230                         free_irq(phba->pcidev->irq, phba);
4231         pci_disable_msix(phba->pcidev);
4232         destroy_workqueue(phba->wq);
4233         if (blk_iopoll_enabled)
4234                 for (i = 0; i < phba->num_cpus; i++) {
4235                         pbe_eq = &phwi_context->be_eq[i];
4236                         blk_iopoll_disable(&pbe_eq->iopoll);
4237                 }
4238
4239         beiscsi_clean_port(phba);
4240         beiscsi_free_mem(phba);
4241         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4242
4243         value = readl((void *)real_offset);
4244
4245         if (value & 0x00010000) {
4246                 value &= 0xfffeffff;
4247                 writel(value, (void *)real_offset);
4248         }
4249         beiscsi_unmap_pci_function(phba);
4250         pci_free_consistent(phba->pcidev,
4251                             phba->ctrl.mbox_mem_alloced.size,
4252                             phba->ctrl.mbox_mem_alloced.va,
4253                             phba->ctrl.mbox_mem_alloced.dma);
4254 }
4255
4256 static void beiscsi_remove(struct pci_dev *pcidev)
4257 {
4258
4259         struct beiscsi_hba *phba = NULL;
4260
4261         phba = pci_get_drvdata(pcidev);
4262         if (!phba) {
4263                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4264                 return;
4265         }
4266
4267         beiscsi_destroy_def_ifaces(phba);
4268         beiscsi_quiesce(phba);
4269         iscsi_boot_destroy_kset(phba->boot_kset);
4270         iscsi_host_remove(phba->shost);
4271         pci_dev_put(phba->pcidev);
4272         iscsi_host_free(phba->shost);
4273         pci_disable_device(pcidev);
4274 }
4275
4276 static void beiscsi_shutdown(struct pci_dev *pcidev)
4277 {
4278
4279         struct beiscsi_hba *phba = NULL;
4280
4281         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4282         if (!phba) {
4283                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4284                 return;
4285         }
4286
4287         beiscsi_quiesce(phba);
4288         pci_disable_device(pcidev);
4289 }
4290
4291 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4292 {
4293         int i, status;
4294
4295         for (i = 0; i <= phba->num_cpus; i++)
4296                 phba->msix_entries[i].entry = i;
4297
4298         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4299                                  (phba->num_cpus + 1));
4300         if (!status)
4301                 phba->msix_enabled = true;
4302
4303         return;
4304 }
4305
4306 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4307                                 const struct pci_device_id *id)
4308 {
4309         struct beiscsi_hba *phba = NULL;
4310         struct hwi_controller *phwi_ctrlr;
4311         struct hwi_context_memory *phwi_context;
4312         struct be_eq_obj *pbe_eq;
4313         int ret, num_cpus, i;
4314         u8 *real_offset = 0;
4315         u32 value = 0;
4316
4317         ret = beiscsi_enable_pci(pcidev);
4318         if (ret < 0) {
4319                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4320                         " Failed to enable pci device\n");
4321                 return ret;
4322         }
4323
4324         phba = beiscsi_hba_alloc(pcidev);
4325         if (!phba) {
4326                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4327                         " Failed in beiscsi_hba_alloc\n");
4328                 goto disable_pci;
4329         }
4330
4331         switch (pcidev->device) {
4332         case BE_DEVICE_ID1:
4333         case OC_DEVICE_ID1:
4334         case OC_DEVICE_ID2:
4335                 phba->generation = BE_GEN2;
4336                 break;
4337         case BE_DEVICE_ID2:
4338         case OC_DEVICE_ID3:
4339                 phba->generation = BE_GEN3;
4340                 break;
4341         default:
4342                 phba->generation = 0;
4343         }
4344
4345         if (enable_msix)
4346                 num_cpus = find_num_cpus();
4347         else
4348                 num_cpus = 1;
4349         phba->num_cpus = num_cpus;
4350         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
4351
4352         if (enable_msix) {
4353                 beiscsi_msix_enable(phba);
4354                 if (!phba->msix_enabled)
4355                         phba->num_cpus = 1;
4356         }
4357         ret = be_ctrl_init(phba, pcidev);
4358         if (ret) {
4359                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4360                                 "Failed in be_ctrl_init\n");
4361                 goto hba_free;
4362         }
4363
4364         if (!num_hba) {
4365                 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4366                 value = readl((void *)real_offset);
4367                 if (value & 0x00010000) {
4368                         gcrashmode++;
4369                         shost_printk(KERN_ERR, phba->shost,
4370                                 "Loading Driver in crashdump mode\n");
4371                         ret = beiscsi_cmd_reset_function(phba);
4372                         if (ret) {
4373                                 shost_printk(KERN_ERR, phba->shost,
4374                                         "Reset Failed. Aborting Crashdump\n");
4375                                 goto hba_free;
4376                         }
4377                         ret = be_chk_reset_complete(phba);
4378                         if (ret) {
4379                                 shost_printk(KERN_ERR, phba->shost,
4380                                         "Failed to get out of reset."
4381                                         "Aborting Crashdump\n");
4382                                 goto hba_free;
4383                         }
4384                 } else {
4385                         value |= 0x00010000;
4386                         writel(value, (void *)real_offset);
4387                         num_hba++;
4388                 }
4389         }
4390
4391         spin_lock_init(&phba->io_sgl_lock);
4392         spin_lock_init(&phba->mgmt_sgl_lock);
4393         spin_lock_init(&phba->isr_lock);
4394         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4395         if (ret != 0) {
4396                 shost_printk(KERN_ERR, phba->shost,
4397                              "Error getting fw config\n");
4398                 goto free_port;
4399         }
4400         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4401         beiscsi_get_params(phba);
4402         phba->shost->can_queue = phba->params.ios_per_ctrl;
4403         ret = beiscsi_init_port(phba);
4404         if (ret < 0) {
4405                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4406                              "Failed in beiscsi_init_port\n");
4407                 goto free_port;
4408         }
4409
4410         for (i = 0; i < MAX_MCC_CMD ; i++) {
4411                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4412                 phba->ctrl.mcc_tag[i] = i + 1;
4413                 phba->ctrl.mcc_numtag[i + 1] = 0;
4414                 phba->ctrl.mcc_tag_available++;
4415         }
4416
4417         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4418
4419         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4420                  phba->shost->host_no);
4421         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4422         if (!phba->wq) {
4423                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4424                                 "Failed to allocate work queue\n");
4425                 goto free_twq;
4426         }
4427
4428         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4429
4430         phwi_ctrlr = phba->phwi_ctrlr;
4431         phwi_context = phwi_ctrlr->phwi_ctxt;
4432         if (blk_iopoll_enabled) {
4433                 for (i = 0; i < phba->num_cpus; i++) {
4434                         pbe_eq = &phwi_context->be_eq[i];
4435                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4436                                         be_iopoll);
4437                         blk_iopoll_enable(&pbe_eq->iopoll);
4438                 }
4439         }
4440         ret = beiscsi_init_irqs(phba);
4441         if (ret < 0) {
4442                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4443                              "Failed to beiscsi_init_irqs\n");
4444                 goto free_blkenbld;
4445         }
4446         hwi_enable_intr(phba);
4447
4448         if (beiscsi_setup_boot_info(phba))
4449                 /*
4450                  * log error but continue, because we may not be using
4451                  * iscsi boot.
4452                  */
4453                 shost_printk(KERN_ERR, phba->shost, "Could not set up "
4454                              "iSCSI boot info.\n");
4455
4456         beiscsi_create_def_ifaces(phba);
4457         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4458         return 0;
4459
4460 free_blkenbld:
4461         destroy_workqueue(phba->wq);
4462         if (blk_iopoll_enabled)
4463                 for (i = 0; i < phba->num_cpus; i++) {
4464                         pbe_eq = &phwi_context->be_eq[i];
4465                         blk_iopoll_disable(&pbe_eq->iopoll);
4466                 }
4467 free_twq:
4468         beiscsi_clean_port(phba);
4469         beiscsi_free_mem(phba);
4470 free_port:
4471         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4472
4473         value = readl((void *)real_offset);
4474
4475         if (value & 0x00010000) {
4476                 value &= 0xfffeffff;
4477                 writel(value, (void *)real_offset);
4478         }
4479
4480         pci_free_consistent(phba->pcidev,
4481                             phba->ctrl.mbox_mem_alloced.size,
4482                             phba->ctrl.mbox_mem_alloced.va,
4483                            phba->ctrl.mbox_mem_alloced.dma);
4484         beiscsi_unmap_pci_function(phba);
4485 hba_free:
4486         if (phba->msix_enabled)
4487                 pci_disable_msix(phba->pcidev);
4488         iscsi_host_remove(phba->shost);
4489         pci_dev_put(phba->pcidev);
4490         iscsi_host_free(phba->shost);
4491 disable_pci:
4492         pci_disable_device(pcidev);
4493         return ret;
4494 }
4495
4496 struct iscsi_transport beiscsi_iscsi_transport = {
4497         .owner = THIS_MODULE,
4498         .name = DRV_NAME,
4499         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4500                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4501         .create_session = beiscsi_session_create,
4502         .destroy_session = beiscsi_session_destroy,
4503         .create_conn = beiscsi_conn_create,
4504         .bind_conn = beiscsi_conn_bind,
4505         .destroy_conn = iscsi_conn_teardown,
4506         .attr_is_visible = be2iscsi_attr_is_visible,
4507         .set_iface_param = be2iscsi_iface_set_param,
4508         .get_iface_param = be2iscsi_iface_get_param,
4509         .set_param = beiscsi_set_param,
4510         .get_conn_param = iscsi_conn_get_param,
4511         .get_session_param = iscsi_session_get_param,
4512         .get_host_param = beiscsi_get_host_param,
4513         .start_conn = beiscsi_conn_start,
4514         .stop_conn = iscsi_conn_stop,
4515         .send_pdu = iscsi_conn_send_pdu,
4516         .xmit_task = beiscsi_task_xmit,
4517         .cleanup_task = beiscsi_cleanup_task,
4518         .alloc_pdu = beiscsi_alloc_pdu,
4519         .parse_pdu_itt = beiscsi_parse_pdu,
4520         .get_stats = beiscsi_conn_get_stats,
4521         .get_ep_param = beiscsi_ep_get_param,
4522         .ep_connect = beiscsi_ep_connect,
4523         .ep_poll = beiscsi_ep_poll,
4524         .ep_disconnect = beiscsi_ep_disconnect,
4525         .session_recovery_timedout = iscsi_session_recovery_timedout,
4526         .bsg_request = beiscsi_bsg_request,
4527 };
4528
4529 static struct pci_driver beiscsi_pci_driver = {
4530         .name = DRV_NAME,
4531         .probe = beiscsi_dev_probe,
4532         .remove = beiscsi_remove,
4533         .shutdown = beiscsi_shutdown,
4534         .id_table = beiscsi_pci_id_table
4535 };
4536
4537
4538 static int __init beiscsi_module_init(void)
4539 {
4540         int ret;
4541
4542         beiscsi_scsi_transport =
4543                         iscsi_register_transport(&beiscsi_iscsi_transport);
4544         if (!beiscsi_scsi_transport) {
4545                 SE_DEBUG(DBG_LVL_1,
4546                          "beiscsi_module_init - Unable to  register beiscsi"
4547                          "transport.\n");
4548                 return -ENOMEM;
4549         }
4550         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
4551                  &beiscsi_iscsi_transport);
4552
4553         ret = pci_register_driver(&beiscsi_pci_driver);
4554         if (ret) {
4555                 SE_DEBUG(DBG_LVL_1,
4556                          "beiscsi_module_init - Unable to  register"
4557                          "beiscsi pci driver.\n");
4558                 goto unregister_iscsi_transport;
4559         }
4560         return 0;
4561
4562 unregister_iscsi_transport:
4563         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4564         return ret;
4565 }
4566
4567 static void __exit beiscsi_module_exit(void)
4568 {
4569         pci_unregister_driver(&beiscsi_pci_driver);
4570         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4571 }
4572
4573 module_init(beiscsi_module_init);
4574 module_exit(beiscsi_module_exit);