drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77 static void nes_terminate_timeout(unsigned long context);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existant",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existant",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112
 113 /**
 114  * nes_nic_init_timer_defaults
 115  */
 116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 117 {
 118         unsigned long flags;
 119         struct nes_adapter *nesadapter = nesdev->nesadapter;
 120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 121
 122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 123
 124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 126         if (jumbomode) {
 127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 130         } else {
 131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 134         }
 135
 136         /* todo use netdev->mtu to set thresholds */
 137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 138 }
 139
 140
 141 /**
 142  * nes_nic_init_timer
 143  */
 144 static void  nes_nic_init_timer(struct nes_device *nesdev)
 145 {
 146         unsigned long flags;
 147         struct nes_adapter *nesadapter = nesdev->nesadapter;
 148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 149
 150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 151
 152         if (shared_timer->timer_in_use_old == 0) {
 153                 nesdev->deepcq_count = 0;
 154                 shared_timer->timer_direction_upward = 0;
 155                 shared_timer->timer_direction_downward = 0;
 156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 157                 shared_timer->timer_in_use_old = 0;
 158
 159         }
 160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 164         }
 165         /* todo use netdev->mtu to set thresholds */
 166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 167 }
 168
 169
 170 /**
 171  * nes_nic_tune_timer
 172  */
 173 static void nes_nic_tune_timer(struct nes_device *nesdev)
 174 {
 175         unsigned long flags;
 176         struct nes_adapter *nesadapter = nesdev->nesadapter;
 177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 178         u16 cq_count = nesdev->currcq_count;
 179
 180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 181
 182         if (shared_timer->cq_count_old <= cq_count)
 183                 shared_timer->cq_direction_downward = 0;
 184         else
 185                 shared_timer->cq_direction_downward++;
 186         shared_timer->cq_count_old = cq_count;
 187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 188                 if (cq_count <= shared_timer->threshold_low &&
 189                     shared_timer->threshold_low > 4) {
 190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 191                         shared_timer->cq_direction_downward=0;
 192                         nesdev->currcq_count = 0;
 193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 194                         return;
 195                 }
 196         }
 197
 198         if (cq_count > 1) {
 199                 nesdev->deepcq_count += cq_count;
 200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 201                         shared_timer->timer_direction_upward++;
 202                         shared_timer->timer_direction_downward = 0;
 203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward = 0;
 206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 207                         shared_timer->timer_direction_downward++;
 208                         shared_timer->timer_direction_upward = 0;
 209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 210                         shared_timer->timer_in_use -= 2;
 211                         shared_timer->timer_direction_upward = 0;
 212                         shared_timer->timer_direction_downward++;
 213                 } else {
 214                         shared_timer->timer_in_use -= 4;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward++;
 217                 }
 218
 219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 220                         shared_timer->timer_in_use += 3;
 221                         shared_timer->timer_direction_upward = 0;
 222                         shared_timer->timer_direction_downward = 0;
 223                 }
 224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 225                         shared_timer->timer_in_use -= 4 ;
 226                         shared_timer->timer_direction_downward = 0;
 227                         shared_timer->timer_direction_upward = 0;
 228                 }
 229         }
 230
 231         /* boundary checking */
 232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 233                 shared_timer->timer_in_use = shared_timer->threshold_high;
 234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 235                 shared_timer->timer_in_use = shared_timer->threshold_low;
 236
 237         nesdev->currcq_count = 0;
 238
 239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 240 }
 241
 242
 243 /**
 244  * nes_init_adapter - initialize adapter
 245  */
 246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 247         struct nes_adapter *nesadapter = NULL;
 248         unsigned long num_pds;
 249         u32 u32temp;
 250         u32 port_count;
 251         u16 max_rq_wrs;
 252         u16 max_sq_wrs;
 253         u32 max_mr;
 254         u32 max_256pbl;
 255         u32 max_4kpbl;
 256         u32 max_qp;
 257         u32 max_irrq;
 258         u32 max_cq;
 259         u32 hte_index_mask;
 260         u32 adapter_size;
 261         u32 arp_table_size;
 262         u16 vendor_id;
 263         u16 device_id;
 264         u8  OneG_Mode;
 265         u8  func_index;
 266
 267         /* search the list of existing adapters */
 268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 271                                 nesdev->pcidev->devfn,
 272                                 PCI_SLOT(nesadapter->devfn),
 273                                 nesadapter->bus_number,
 274                                 PCI_SLOT(nesdev->pcidev->devfn),
 275                                 nesdev->pcidev->bus->number );
 276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 278                         nesadapter->ref_count++;
 279                         return nesadapter;
 280                 }
 281         }
 282
 283         /* no adapter found */
 284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 287                                 hw_rev);
 288                 return NULL;
 289         }
 290
 291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 296
 297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 298
 299
 300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 301                 return NULL;
 302
 303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 305
 306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 309                                 max_qp, u32temp);
 310                 max_qp = (u32)1 << (u32temp & 0x001f);
 311         }
 312
 313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 315                         max_qp, hte_index_mask);
 316
 317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 318
 319         max_irrq = 1 << (u32temp & 0x001f);
 320
 321         if (max_qp > max_irrq) {
 322                 max_qp = max_irrq;
 323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 324                                 max_qp);
 325         }
 326
 327         /* there should be no reason to allocate more pds than qps */
 328         if (num_pds > max_qp)
 329                 num_pds = max_qp;
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 332         max_mr = (u32)8192 << (u32temp & 0x7);
 333
 334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 340         arp_table_size = 1 << u32temp;
 341
 342         adapter_size = (sizeof(struct nes_adapter) +
 343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 349         adapter_size += sizeof(struct nes_qp **) * max_qp;
 350
 351         /* allocate a new adapter struct */
 352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 353         if (nesadapter == NULL) {
 354                 return NULL;
 355         }
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs, MR and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445         set_bit(0, nesadapter->allocated_mrs);
 446
 447         for (u32temp = 0; u32temp < 20; u32temp++)
 448                 set_bit(u32temp, nesadapter->allocated_pds);
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 450
 451         max_rq_wrs = ((u32temp >> 8) & 3);
 452         switch (max_rq_wrs) {
 453                 case 0:
 454                         max_rq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_rq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_rq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_rq_wrs = 512;
 464                         break;
 465         }
 466
 467         max_sq_wrs = (u32temp & 3);
 468         switch (max_sq_wrs) {
 469                 case 0:
 470                         max_sq_wrs = 4;
 471                         break;
 472                 case 1:
 473                         max_sq_wrs = 16;
 474                         break;
 475                 case 2:
 476                         max_sq_wrs = 32;
 477                         break;
 478                 case 3:
 479                         max_sq_wrs = 512;
 480                         break;
 481         }
 482         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 483         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 484
 485         nesadapter->max_sge = 4;
 486         nesadapter->max_cqe = 32766;
 487
 488         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 489                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 490                 kfree(nesadapter);
 491                 return NULL;
 492         }
 493
 494         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 495         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 496                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 497
 498         /* setup port configuration */
 499         if (nesadapter->port_count == 1) {
 500                 nesadapter->log_port = 0x00000000;
 501                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 503                 else
 504                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 505         } else {
 506                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 507                         nesadapter->log_port = 0x000000D8;
 508                 } else {
 509                         if (nesadapter->port_count == 2)
 510                                 nesadapter->log_port = 0x00000044;
 511                         else
 512                                 nesadapter->log_port = 0x000000e4;
 513                 }
 514                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 515         }
 516
 517         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 518                                                 nesadapter->log_port);
 519         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 520                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 521
 522         spin_lock_init(&nesadapter->resource_lock);
 523         spin_lock_init(&nesadapter->phy_lock);
 524         spin_lock_init(&nesadapter->pbl_lock);
 525         spin_lock_init(&nesadapter->periodic_timer_lock);
 526
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 531
 532         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 533                 u32 pcs_control_status0, pcs_control_status1;
 534                 u32 reset_value;
 535                 u32 i = 0;
 536                 u32 int_cnt = 0;
 537                 u32 ext_cnt = 0;
 538                 unsigned long flags;
 539                 u32 j = 0;
 540
 541                 pcs_control_status0 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 543                 pcs_control_status1 = nes_read_indexed(nesdev,
 544                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 545
 546                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 547                         pcs_control_status0 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 549                         pcs_control_status1 = nes_read_indexed(nesdev,
 550                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 551                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 552                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 553                                 int_cnt++;
 554                         msleep(1);
 555                 }
 556                 if (int_cnt > 1) {
 557                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 558                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 559                         mh_detected++;
 560                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 561                         reset_value |= 0x0000003d;
 562                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 563
 564                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 565                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 566                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 567
 568                         pcs_control_status0 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 570                         pcs_control_status1 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 572
 573                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 574                                 pcs_control_status0 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 576                                 pcs_control_status1 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 578                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 579                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 580                                         if (++ext_cnt > int_cnt) {
 581                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 582                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 583                                                                 0x0000F088);
 584                                                 mh_detected++;
 585                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 586                                                 reset_value |= 0x0000003d;
 587                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 588
 589                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 590                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 591                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 592                                                 break;
 593                                         }
 594                                 }
 595                                 msleep(1);
 596                         }
 597                 }
 598         }
 599
 600         if (nesadapter->hw_rev == NE020_REV) {
 601                 init_timer(&nesadapter->mh_timer);
 602                 nesadapter->mh_timer.function = nes_mh_fix;
 603                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 604                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 605                 add_timer(&nesadapter->mh_timer);
 606         } else {
 607                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 608         }
 609
 610         init_timer(&nesadapter->lc_timer);
 611         nesadapter->lc_timer.function = nes_clc;
 612         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 613         nesadapter->lc_timer.data = (unsigned long)nesdev;
 614         add_timer(&nesadapter->lc_timer);
 615
 616         list_add_tail(&nesadapter->list, &nes_adapter_list);
 617
 618         for (func_index = 0; func_index < 8; func_index++) {
 619                 pci_bus_read_config_word(nesdev->pcidev->bus,
 620                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 621                                         func_index), 0, &vendor_id);
 622                 if (vendor_id == 0xffff)
 623                         break;
 624         }
 625         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 626                 func_index, pci_name(nesdev->pcidev));
 627         nesadapter->adapter_fcn_count = func_index;
 628
 629         return nesadapter;
 630 }
 631
 632
 633 /**
 634  * nes_reset_adapter_ne020
 635  */
 636 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 637 {
 638         u32 port_count;
 639         u32 u32temp;
 640         u32 i;
 641
 642         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 643         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 644         /* TODO: assuming that both SERDES are set the same for now */
 645         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 646         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 647                         u32temp, port_count);
 648         if (*OneG_Mode)
 649                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 650         u32temp &= 0xff00ffc0;
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0000;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0000;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000000;
 660                         break;
 661                 default:
 662                         return 0;
 663                         break;
 664         }
 665
 666         /* check and do full reset if needed */
 667         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 668                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 669                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671                 i = 0;
 672                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i > 10000) {
 675                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 676                         return 0;
 677                 }
 678
 679                 i = 0;
 680                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                         return 0;
 686                 }
 687         }
 688
 689         /* port reset */
 690         switch (port_count) {
 691                 case 1:
 692                         u32temp |= 0x00ee0010;
 693                         break;
 694                 case 2:
 695                         u32temp |= 0x00cc0030;
 696                         break;
 697                 case 4:
 698                         u32temp |= 0x00000030;
 699                         break;
 700         }
 701
 702         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 703         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 704
 705         i = 0;
 706         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 707                 mdelay(1);
 708         if (i > 10000) {
 709                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 710                 return 0;
 711         }
 712
 713         /* serdes 0 */
 714         i = 0;
 715         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                 mdelay(1);
 718         if (i > 5000) {
 719                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 720                 return 0;
 721         }
 722
 723         /* serdes 1 */
 724         if (port_count > 1) {
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i > 5000) {
 730                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 731                         return 0;
 732                 }
 733         }
 734
 735         return port_count;
 736 }
 737
 738
 739 /**
 740  * nes_init_serdes
 741  */
 742 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 743                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 744 {
 745         int i;
 746         u32 u32temp;
 747         u32 sds;
 748
 749         if (hw_rev != NE020_REV) {
 750                 /* init serdes 0 */
 751                 switch (nesadapter->phy_type[0]) {
 752                 case NES_PHY_TYPE_CX4:
 753                         if (wide_ppm_offset)
 754                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 755                         else
 756                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 757                         break;
 758                 case NES_PHY_TYPE_KR:
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 761                         break;
 762                 case NES_PHY_TYPE_PUMA_1G:
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 764                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 765                         sds |= 0x00000100;
 766                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 767                         break;
 768                 default:
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 770                         break;
 771                 }
 772
 773                 if (!OneG_Mode)
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 775
 776                 if (port_count < 2)
 777                         return 0;
 778
 779                 /* init serdes 1 */
 780                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 781                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 782
 783                 switch (nesadapter->phy_type[1]) {
 784                 case NES_PHY_TYPE_ARGUS:
 785                 case NES_PHY_TYPE_SFP_D:
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 788                         break;
 789                 case NES_PHY_TYPE_CX4:
 790                         if (wide_ppm_offset)
 791                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 792                         break;
 793                 case NES_PHY_TYPE_KR:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 795                         break;
 796                 case NES_PHY_TYPE_PUMA_1G:
 797                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 798                         sds |= 0x000000100;
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 800                 }
 801                 if (!OneG_Mode) {
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 803                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 804                         sds &= 0xFFFFFFBF;
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 806                 }
 807         } else {
 808                 /* init serdes 0 */
 809                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 810                 i = 0;
 811                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 812                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 813                         mdelay(1);
 814                 if (i > 5000) {
 815                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 816                         return 1;
 817                 }
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 824                 if (OneG_Mode)
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 826                 else
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 828
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 830                 if (port_count > 1) {
 831                         /* init serdes 1 */
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 833                         i = 0;
 834                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 835                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 836                                 mdelay(1);
 837                         if (i > 5000) {
 838                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 839                                 /* return 1; */
 840                         }
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 849                 }
 850         }
 851         return 0;
 852 }
 853
 854
 855 /**
 856  * nes_init_csr_ne020
 857  * Initialize registers for ne020 hardware
 858  */
 859 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 860 {
 861         u32 u32temp;
 862
 863         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 864
 865         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 866         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 867         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 868         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 869         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 870         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 871         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 872         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 873
 874         /* TODO: move these MAC register settings to NIC bringup */
 875         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 876         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 878         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 879         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 880         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 881         if (port_count > 1) {
 882                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 885                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 887                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 888                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 889         }
 890         if (port_count > 2) {
 891                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 898
 899                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 906         }
 907
 908         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 909         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 910         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 911                                                          0x00000001);
 912         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 913         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 917
 918         /* TODO: move this to code, get from EEPROM */
 919         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 920         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 921         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 922
 923         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 924         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 925
 926         if (hw_rev != NE020_REV) {
 927                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 928                 u32temp |= 0x80000000;
 929                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 930                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 931                 u32temp &= 0x7fffffff;
 932                 u32temp |= 0x7fff0010;
 933                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 934                 if (port_count > 1) {
 935                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 936                         u32temp &= 0x7fffffff;
 937                         u32temp |= 0x7fff0010;
 938                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 939                 }
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_destroy_adapter - destroy the adapter structure
 946  */
 947 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 948 {
 949         struct nes_adapter *tmp_adapter;
 950
 951         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 952                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 953                                 tmp_adapter);
 954         }
 955
 956         nesadapter->ref_count--;
 957         if (!nesadapter->ref_count) {
 958                 if (nesadapter->hw_rev == NE020_REV) {
 959                         del_timer(&nesadapter->mh_timer);
 960                 }
 961                 del_timer(&nesadapter->lc_timer);
 962
 963                 list_del(&nesadapter->list);
 964                 kfree(nesadapter);
 965         }
 966 }
 967
 968
 969 /**
 970  * nes_init_cqp
 971  */
 972 int nes_init_cqp(struct nes_device *nesdev)
 973 {
 974         struct nes_adapter *nesadapter = nesdev->nesadapter;
 975         struct nes_hw_cqp_qp_context *cqp_qp_context;
 976         struct nes_hw_cqp_wqe *cqp_wqe;
 977         struct nes_hw_ceq *ceq;
 978         struct nes_hw_ceq *nic_ceq;
 979         struct nes_hw_aeq *aeq;
 980         void *vmem;
 981         dma_addr_t pmem;
 982         u32 count=0;
 983         u32 cqp_head;
 984         u64 u64temp;
 985         u32 u32temp;
 986
 987         /* allocate CQP memory */
 988         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 989         /* SQ is 512 byte aligned, others are 256 byte aligned */
 990         nesdev->cqp_mem_size = 512 +
 991                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 992                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 993                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 995                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 996                         sizeof(struct nes_hw_cqp_qp_context);
 997
 998         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 999                         &nesdev->cqp_pbase);
1000         if (!nesdev->cqp_vbase) {
1001                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1002                 return -ENOMEM;
1003         }
1004         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1005
1006         /* Allocate a twice the number of CQP requests as the SQ size */
1007         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1008                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1009         if (nesdev->nes_cqp_requests == NULL) {
1010                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1011                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1012                                 nesdev->cqp.sq_pbase);
1013                 return -ENOMEM;
1014         }
1015
1016         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1017                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1018
1019         spin_lock_init(&nesdev->cqp.lock);
1020         init_waitqueue_head(&nesdev->cqp.waitq);
1021
1022         /* Setup Various Structures */
1023         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1024                         ~(unsigned long)(512 - 1));
1025         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1026                         ~(unsigned long long)(512 - 1));
1027
1028         nesdev->cqp.sq_vbase = vmem;
1029         nesdev->cqp.sq_pbase = pmem;
1030         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1031         nesdev->cqp.sq_head = 0;
1032         nesdev->cqp.sq_tail = 0;
1033         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1034
1035         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1036         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1037
1038         nesdev->ccq.cq_vbase = vmem;
1039         nesdev->ccq.cq_pbase = pmem;
1040         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1041         nesdev->ccq.cq_head = 0;
1042         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1043         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1044
1045         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1046         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1047
1048         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1049         ceq = &nesadapter->ceq[nesdev->ceq_index];
1050         ceq->ceq_vbase = vmem;
1051         ceq->ceq_pbase = pmem;
1052         ceq->ceq_size = NES_CCEQ_SIZE;
1053         ceq->ceq_head = 0;
1054
1055         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1056         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1057
1058         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1059         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1060         nic_ceq->ceq_vbase = vmem;
1061         nic_ceq->ceq_pbase = pmem;
1062         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1063         nic_ceq->ceq_head = 0;
1064
1065         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1066         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1067
1068         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1069         aeq->aeq_vbase = vmem;
1070         aeq->aeq_pbase = pmem;
1071         aeq->aeq_size = nesadapter->max_qp;
1072         aeq->aeq_head = 0;
1073
1074         /* Setup QP Context */
1075         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1076         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1077
1078         cqp_qp_context = vmem;
1079         cqp_qp_context->context_words[0] =
1080                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1081         cqp_qp_context->context_words[1] = 0;
1082         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1083         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1084
1085
1086         /* Write the address to Create CQP */
1087         if ((sizeof(dma_addr_t) > 4)) {
1088                 nes_write_indexed(nesdev,
1089                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1090                                 ((u64)pmem) >> 32);
1091         } else {
1092                 nes_write_indexed(nesdev,
1093                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1094         }
1095         nes_write_indexed(nesdev,
1096                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1097                         (u32)pmem);
1098
1099         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1100         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1101
1102         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1103                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1104                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1105         }
1106
1107         /* Write Create CCQ WQE */
1108         cqp_head = nesdev->cqp.sq_head++;
1109         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1110         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1111         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1112                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1113                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1114         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1115                             (nesdev->ccq.cq_number |
1116                              ((u32)nesdev->ceq_index << 16)));
1117         u64temp = (u64)nesdev->ccq.cq_pbase;
1118         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1119         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1120         u64temp = (unsigned long)&nesdev->ccq;
1121         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1122                         cpu_to_le32((u32)(u64temp >> 1));
1123         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1124                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1125         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1126
1127         /* Write Create CEQ WQE */
1128         cqp_head = nesdev->cqp.sq_head++;
1129         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1130         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1131         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1132                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1133         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1134         u64temp = (u64)ceq->ceq_pbase;
1135         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1136
1137         /* Write Create AEQ WQE */
1138         cqp_head = nesdev->cqp.sq_head++;
1139         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1140         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1141         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1142                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1143         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1144         u64temp = (u64)aeq->aeq_pbase;
1145         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1146
1147         /* Write Create NIC CEQ WQE */
1148         cqp_head = nesdev->cqp.sq_head++;
1149         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1150         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1151         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1152                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1153         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1154         u64temp = (u64)nic_ceq->ceq_pbase;
1155         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1156
1157         /* Poll until CCQP done */
1158         count = 0;
1159         do {
1160                 if (count++ > 1000) {
1161                         printk(KERN_ERR PFX "Error creating CQP\n");
1162                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1163                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1164                         return -1;
1165                 }
1166                 udelay(10);
1167         } while (!(nes_read_indexed(nesdev,
1168                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1169
1170         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1171                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1172
1173         u32temp = 0x04800000;
1174         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1175
1176         /* wait for the CCQ, CEQ, and AEQ to get created */
1177         count = 0;
1178         do {
1179                 if (count++ > 1000) {
1180                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1181                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1182                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1183                         return -1;
1184                 }
1185                 udelay(10);
1186         } while (((nes_read_indexed(nesdev,
1187                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1188
1189         /* dump the QP status value */
1190         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1191                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1192
1193         nesdev->cqp.sq_tail++;
1194
1195         return 0;
1196 }
1197
1198
1199 /**
1200  * nes_destroy_cqp
1201  */
1202 int nes_destroy_cqp(struct nes_device *nesdev)
1203 {
1204         struct nes_hw_cqp_wqe *cqp_wqe;
1205         u32 count = 0;
1206         u32 cqp_head;
1207         unsigned long flags;
1208
1209         do {
1210                 if (count++ > 1000)
1211                         break;
1212                 udelay(10);
1213         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1214
1215         /* Reset CCQ */
1216         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1217                         nesdev->ccq.cq_number);
1218
1219         /* Disable device interrupts */
1220         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1221
1222         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1223
1224         /* Destroy the AEQ */
1225         cqp_head = nesdev->cqp.sq_head++;
1226         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1227         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1228         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1229                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1230         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1231
1232         /* Destroy the NIC CEQ */
1233         cqp_head = nesdev->cqp.sq_head++;
1234         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1235         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1236         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1237                         ((u32)nesdev->nic_ceq_index << 8));
1238
1239         /* Destroy the CEQ */
1240         cqp_head = nesdev->cqp.sq_head++;
1241         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1242         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1243         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1244                         (nesdev->ceq_index << 8));
1245
1246         /* Destroy the CCQ */
1247         cqp_head = nesdev->cqp.sq_head++;
1248         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1249         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1250         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1251         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1252                         ((u32)nesdev->ceq_index << 16));
1253
1254         /* Destroy CQP */
1255         cqp_head = nesdev->cqp.sq_head++;
1256         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1257         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1258         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1259                         NES_CQP_QP_TYPE_CQP);
1260         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1261
1262         barrier();
1263         /* Ring doorbell (5 WQEs) */
1264         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1265
1266         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1267
1268         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1269         count = 0;
1270         do {
1271                 if (count++ > 1000) {
1272                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1273                                         PCI_FUNC(nesdev->pcidev->devfn));
1274                         break;
1275                 }
1276                 udelay(10);
1277         } while (((nes_read_indexed(nesdev,
1278                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1279
1280         /* dump the QP status value */
1281         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1282                         PCI_FUNC(nesdev->pcidev->devfn),
1283                         nes_read_indexed(nesdev,
1284                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1285
1286         kfree(nesdev->nes_cqp_requests);
1287
1288         /* Free the control structures */
1289         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1290                         nesdev->cqp.sq_pbase);
1291
1292         return 0;
1293 }
1294
1295
1296 /**
1297  * nes_init_1g_phy
1298  */
1299 int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1300 {
1301         u32 counter = 0;
1302         u16 phy_data;
1303         int ret = 0;
1304
1305         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1306         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1307
1308         /* Reset the PHY */
1309         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1310         udelay(100);
1311         counter = 0;
1312         do {
1313                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1314                 if (counter++ > 100) {
1315                         ret = -1;
1316                         break;
1317                 }
1318         } while (phy_data & 0x8000);
1319
1320         /* Setting no phy loopback */
1321         phy_data &= 0xbfff;
1322         phy_data |= 0x1140;
1323         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1324         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1326         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1327
1328         /* Setting the interrupt mask */
1329         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1330         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1331         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1332
1333         /* turning on flow control */
1334         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1335         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1336         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1337
1338         /* Clear Half duplex */
1339         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1340         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1341         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1342
1343         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1344         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1345
1346         return ret;
1347 }
1348
1349
1350 /**
1351  * nes_init_2025_phy
1352  */
1353 int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1354 {
1355         u32 temp_phy_data = 0;
1356         u32 temp_phy_data2 = 0;
1357         u32 counter = 0;
1358         u32 sds;
1359         u32 mac_index = nesdev->mac_index;
1360         int ret = 0;
1361         unsigned int first_attempt = 1;
1362
1363         /* Check firmware heartbeat */
1364         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1365         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1366         udelay(1500);
1367         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1368         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1369
1370         if (temp_phy_data != temp_phy_data2) {
1371                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1372                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1373                 if ((temp_phy_data & 0xff) > 0x20)
1374                         return 0;
1375                 printk(PFX "Reinitialize external PHY\n");
1376         }
1377
1378         /* no heartbeat, configure the PHY */
1379         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1382         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1383
1384         switch (phy_type) {
1385         case NES_PHY_TYPE_ARGUS:
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1392                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1393
1394                 /* setup LEDs */
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1398                 break;
1399
1400         case NES_PHY_TYPE_SFP_D:
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1407                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1408
1409                 /* setup LEDs */
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1413                 break;
1414
1415         case NES_PHY_TYPE_KR:
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1422                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1423
1424                 /* setup LEDs */
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1428
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1431                 break;
1432         }
1433
1434         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1435
1436         /* Bring PHY out of reset */
1437         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1438
1439         /* Check for heartbeat */
1440         counter = 0;
1441         mdelay(690);
1442         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1443         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1444         do {
1445                 if (counter++ > 150) {
1446                         printk(PFX "No PHY heartbeat\n");
1447                         break;
1448                 }
1449                 mdelay(1);
1450                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1451                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1452         } while ((temp_phy_data2 == temp_phy_data));
1453
1454         /* wait for tracking */
1455         counter = 0;
1456         do {
1457                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1458                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1459                 if (counter++ > 300) {
1460                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1461                                 first_attempt = 0;
1462                                 counter = 0;
1463                                 /* reset AMCC PHY and try again */
1464                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1465                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1466                                 continue;
1467                         } else {
1468                                 ret = 1;
1469                                 break;
1470                         }
1471                 }
1472                 mdelay(10);
1473         } while ((temp_phy_data & 0xff) < 0x30);
1474
1475         /* setup signal integrity */
1476         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1478         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1479         if (phy_type == NES_PHY_TYPE_KR) {
1480                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1481         } else {
1482                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1483                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1484         }
1485
1486         /* reset serdes */
1487         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1488         sds |= 0x1;
1489         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1490         sds &= 0xfffffffe;
1491         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1492
1493         counter = 0;
1494         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1495                         && (counter++ < 5000))
1496                 ;
1497
1498         return ret;
1499 }
1500
1501
1502 /**
1503  * nes_init_phy
1504  */
1505 int nes_init_phy(struct nes_device *nesdev)
1506 {
1507         struct nes_adapter *nesadapter = nesdev->nesadapter;
1508         u32 mac_index = nesdev->mac_index;
1509         u32 tx_config = 0;
1510         unsigned long flags;
1511         u8  phy_type = nesadapter->phy_type[mac_index];
1512         u8  phy_index = nesadapter->phy_index[mac_index];
1513         int ret = 0;
1514
1515         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1516         if (phy_type == NES_PHY_TYPE_1G) {
1517                 /* setup 1G MDIO operation */
1518                 tx_config &= 0xFFFFFFE3;
1519                 tx_config |= 0x04;
1520         } else {
1521                 /* setup 10G MDIO operation */
1522                 tx_config &= 0xFFFFFFE3;
1523                 tx_config |= 0x15;
1524         }
1525         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1526
1527         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1528
1529         switch (phy_type) {
1530         case NES_PHY_TYPE_1G:
1531                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1532                 break;
1533         case NES_PHY_TYPE_ARGUS:
1534         case NES_PHY_TYPE_SFP_D:
1535         case NES_PHY_TYPE_KR:
1536                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1537                 break;
1538         }
1539
1540         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1541
1542         return ret;
1543 }
1544
1545
1546 /**
1547  * nes_replenish_nic_rq
1548  */
1549 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1550 {
1551         unsigned long flags;
1552         dma_addr_t bus_address;
1553         struct sk_buff *skb;
1554         struct nes_hw_nic_rq_wqe *nic_rqe;
1555         struct nes_hw_nic *nesnic;
1556         struct nes_device *nesdev;
1557         u32 rx_wqes_posted = 0;
1558
1559         nesnic = &nesvnic->nic;
1560         nesdev = nesvnic->nesdev;
1561         spin_lock_irqsave(&nesnic->rq_lock, flags);
1562         if (nesnic->replenishing_rq !=0) {
1563                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1564                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1565                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1566                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1567                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1568                         add_timer(&nesvnic->rq_wqes_timer);
1569                 } else
1570                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1571                 return;
1572         }
1573         nesnic->replenishing_rq = 1;
1574         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1575         do {
1576                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1577                 if (skb) {
1578                         skb->dev = nesvnic->netdev;
1579
1580                         bus_address = pci_map_single(nesdev->pcidev,
1581                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1582
1583                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1584                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1585                                         cpu_to_le32(nesvnic->max_frame_size);
1586                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1588                                         cpu_to_le32((u32)bus_address);
1589                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1590                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1591                         nesnic->rx_skb[nesnic->rq_head] = skb;
1592                         nesnic->rq_head++;
1593                         nesnic->rq_head &= nesnic->rq_size - 1;
1594                         atomic_dec(&nesvnic->rx_skbs_needed);
1595                         barrier();
1596                         if (++rx_wqes_posted == 255) {
1597                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1598                                 rx_wqes_posted = 0;
1599                         }
1600                 } else {
1601                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1602                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1603                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1604                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1605                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1606                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1607                                 add_timer(&nesvnic->rq_wqes_timer);
1608                         } else
1609                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1610                         break;
1611                 }
1612         } while (atomic_read(&nesvnic->rx_skbs_needed));
1613         barrier();
1614         if (rx_wqes_posted)
1615                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1616         nesnic->replenishing_rq = 0;
1617 }
1618
1619
1620 /**
1621  * nes_rq_wqes_timeout
1622  */
1623 static void nes_rq_wqes_timeout(unsigned long parm)
1624 {
1625         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1626         printk("%s: Timer fired.\n", __func__);
1627         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1628         if (atomic_read(&nesvnic->rx_skbs_needed))
1629                 nes_replenish_nic_rq(nesvnic);
1630 }
1631
1632
1633 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1634                                void **tcph, u64 *hdr_flags, void *priv)
1635 {
1636         unsigned int ip_len;
1637         struct iphdr *iph;
1638         skb_reset_network_header(skb);
1639         iph = ip_hdr(skb);
1640         if (iph->protocol != IPPROTO_TCP)
1641                 return -1;
1642         ip_len = ip_hdrlen(skb);
1643         skb_set_transport_header(skb, ip_len);
1644         *tcph = tcp_hdr(skb);
1645
1646         *hdr_flags = LRO_IPV4 | LRO_TCP;
1647         *iphdr = iph;
1648         return 0;
1649 }
1650
1651
1652 /**
1653  * nes_init_nic_qp
1654  */
1655 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1656 {
1657         struct nes_hw_cqp_wqe *cqp_wqe;
1658         struct nes_hw_nic_sq_wqe *nic_sqe;
1659         struct nes_hw_nic_qp_context *nic_context;
1660         struct sk_buff *skb;
1661         struct nes_hw_nic_rq_wqe *nic_rqe;
1662         struct nes_vnic *nesvnic = netdev_priv(netdev);
1663         unsigned long flags;
1664         void *vmem;
1665         dma_addr_t pmem;
1666         u64 u64temp;
1667         int ret;
1668         u32 cqp_head;
1669         u32 counter;
1670         u32 wqe_count;
1671         u8 jumbomode=0;
1672
1673         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1674         nesvnic->nic_mem_size = 256 +
1675                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1676                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1677                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1678                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1679                         sizeof(struct nes_hw_nic_qp_context);
1680
1681         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1682                         &nesvnic->nic_pbase);
1683         if (!nesvnic->nic_vbase) {
1684                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1685                 return -ENOMEM;
1686         }
1687         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1688         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1689                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1690
1691         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1692                         ~(unsigned long)(256 - 1));
1693         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1694                         ~(unsigned long long)(256 - 1));
1695
1696         /* Setup the first Fragment buffers */
1697         nesvnic->nic.first_frag_vbase = vmem;
1698
1699         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1700                 nesvnic->nic.frag_paddr[counter] = pmem;
1701                 pmem += sizeof(struct nes_first_frag);
1702         }
1703
1704         /* setup the SQ */
1705         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1706
1707         nesvnic->nic.sq_vbase = (void *)vmem;
1708         nesvnic->nic.sq_pbase = pmem;
1709         nesvnic->nic.sq_head = 0;
1710         nesvnic->nic.sq_tail = 0;
1711         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1712         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1713                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1714                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1715                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1716                                 NES_NIC_SQ_WQE_COMPLETION);
1717                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1718                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1719                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1720                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1721                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1722                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1723         }
1724
1725         nesvnic->get_cqp_request = nes_get_cqp_request;
1726         nesvnic->post_cqp_request = nes_post_cqp_request;
1727         nesvnic->mcrq_mcast_filter = NULL;
1728
1729         spin_lock_init(&nesvnic->nic.rq_lock);
1730
1731         /* setup the RQ */
1732         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1733         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1734
1735
1736         nesvnic->nic.rq_vbase = vmem;
1737         nesvnic->nic.rq_pbase = pmem;
1738         nesvnic->nic.rq_head = 0;
1739         nesvnic->nic.rq_tail = 0;
1740         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1741
1742         /* setup the CQ */
1743         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1744         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1745
1746         if (nesdev->nesadapter->netdev_count > 2)
1747                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1748         else
1749                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1750
1751         nesvnic->nic_cq.cq_vbase = vmem;
1752         nesvnic->nic_cq.cq_pbase = pmem;
1753         nesvnic->nic_cq.cq_head = 0;
1754         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1755
1756         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1757
1758         /* Send CreateCQ request to CQP */
1759         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1760         cqp_head = nesdev->cqp.sq_head;
1761
1762         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1763         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1764
1765         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1766                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1767                         ((u32)nesvnic->nic_cq.cq_size << 16));
1768         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1769                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1770         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1771         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1772         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1773         u64temp = (unsigned long)&nesvnic->nic_cq;
1774         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1775         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1776                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1777         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1778         if (++cqp_head >= nesdev->cqp.sq_size)
1779                 cqp_head = 0;
1780         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1781         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1782
1783         /* Send CreateQP request to CQP */
1784         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1785         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1786                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1787                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1788         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1789                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1790                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1791         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1792                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1793         }
1794
1795         u64temp = (u64)nesvnic->nic.sq_pbase;
1796         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1797         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1798         u64temp = (u64)nesvnic->nic.rq_pbase;
1799         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1800         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1801
1802         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1803                         NES_CQP_QP_TYPE_NIC);
1804         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1805         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1806                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1807         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1808
1809         if (++cqp_head >= nesdev->cqp.sq_size)
1810                 cqp_head = 0;
1811         nesdev->cqp.sq_head = cqp_head;
1812
1813         barrier();
1814
1815         /* Ring doorbell (2 WQEs) */
1816         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1817
1818         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1819         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1820                         nesvnic->nic.qp_id);
1821
1822         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1823                         NES_EVENT_TIMEOUT);
1824         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1825                         nesvnic->nic.qp_id, ret);
1826         if (!ret) {
1827                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1828                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1829                                 nesvnic->nic_pbase);
1830                 return -EIO;
1831         }
1832
1833         /* Populate the RQ */
1834         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1835                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1836                 if (!skb) {
1837                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1838
1839                         nes_destroy_nic_qp(nesvnic);
1840                         return -ENOMEM;
1841                 }
1842
1843                 skb->dev = netdev;
1844
1845                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1846                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1847
1848                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1849                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1850                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1851                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1852                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1853                 nesvnic->nic.rx_skb[counter] = skb;
1854         }
1855
1856         wqe_count = NES_NIC_WQ_SIZE - 1;
1857         nesvnic->nic.rq_head = wqe_count;
1858         barrier();
1859         do {
1860                 counter = min(wqe_count, ((u32)255));
1861                 wqe_count -= counter;
1862                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1863         } while (wqe_count);
1864         init_timer(&nesvnic->rq_wqes_timer);
1865         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1866         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1867         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1868         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1869         {
1870                 nes_nic_init_timer(nesdev);
1871                 if (netdev->mtu > 1500)
1872                         jumbomode = 1;
1873                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1874         }
1875         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1876         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1877         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1878         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1879         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1880         nesvnic->lro_mgr.dev            = netdev;
1881         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1882         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1883         return 0;
1884 }
1885
1886
1887 /**
1888  * nes_destroy_nic_qp
1889  */
1890 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1891 {
1892         u64 u64temp;
1893         dma_addr_t bus_address;
1894         struct nes_device *nesdev = nesvnic->nesdev;
1895         struct nes_hw_cqp_wqe *cqp_wqe;
1896         struct nes_hw_nic_sq_wqe *nic_sqe;
1897         struct nes_hw_nic_rq_wqe *nic_rqe;
1898         __le16 *wqe_fragment_length;
1899         u16  wqe_fragment_index;
1900         u64 wqe_frag;
1901         u32 cqp_head;
1902         unsigned long flags;
1903         int ret;
1904
1905         /* Free remaining NIC receive buffers */
1906         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1907                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1908                 wqe_frag  = (u64)le32_to_cpu(
1909                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1910                 wqe_frag |= ((u64)le32_to_cpu(
1911                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1912                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1913                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1914                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1915                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1916         }
1917
1918         /* Free remaining NIC transmit buffers */
1919         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1920                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1921                 wqe_fragment_index = 1;
1922                 wqe_fragment_length = (__le16 *)
1923                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1924                 /* bump past the vlan tag */
1925                 wqe_fragment_length++;
1926                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1927                         u64temp = (u64)le32_to_cpu(
1928                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1929                                 wqe_fragment_index*2]);
1930                         u64temp += ((u64)le32_to_cpu(
1931                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1932                                 + wqe_fragment_index*2]))<<32;
1933                         bus_address = (dma_addr_t)u64temp;
1934                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1935                                         nesvnic->nic.first_frag_overflow)) {
1936                                 pci_unmap_single(nesdev->pcidev,
1937                                                 bus_address,
1938                                                 le16_to_cpu(wqe_fragment_length[
1939                                                         wqe_fragment_index++]),
1940                                                 PCI_DMA_TODEVICE);
1941                         }
1942                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1943                                 if (wqe_fragment_length[wqe_fragment_index]) {
1944                                         u64temp = le32_to_cpu(
1945                                                 nic_sqe->wqe_words[
1946                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1947                                                 wqe_fragment_index*2]);
1948                                         u64temp += ((u64)le32_to_cpu(
1949                                                 nic_sqe->wqe_words[
1950                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1951                                                 wqe_fragment_index*2]))<<32;
1952                                         bus_address = (dma_addr_t)u64temp;
1953                                         pci_unmap_page(nesdev->pcidev,
1954                                                         bus_address,
1955                                                         le16_to_cpu(
1956                                                         wqe_fragment_length[
1957                                                         wqe_fragment_index]),
1958                                                         PCI_DMA_TODEVICE);
1959                                 } else
1960                                         break;
1961                         }
1962                 }
1963                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1964                         dev_kfree_skb(
1965                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1966
1967                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1968                                         & (nesvnic->nic.sq_size - 1);
1969         }
1970
1971         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1972
1973         /* Destroy NIC QP */
1974         cqp_head = nesdev->cqp.sq_head;
1975         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1976         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1977
1978         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1979                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1980         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1981                 nesvnic->nic.qp_id);
1982
1983         if (++cqp_head >= nesdev->cqp.sq_size)
1984                 cqp_head = 0;
1985
1986         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1987
1988         /* Destroy NIC CQ */
1989         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1990         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1991                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1992         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1993                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1994
1995         if (++cqp_head >= nesdev->cqp.sq_size)
1996                 cqp_head = 0;
1997
1998         nesdev->cqp.sq_head = cqp_head;
1999         barrier();
2000
2001         /* Ring doorbell (2 WQEs) */
2002         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2003
2004         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2005         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2006                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2007                         cqp_head, nesdev->cqp.sq_head,
2008                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2009
2010         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2011                         NES_EVENT_TIMEOUT);
2012
2013         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2014                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2015                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2016         if (!ret) {
2017                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2018                                 nesvnic->nic.qp_id);
2019         }
2020
2021         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2022                         nesvnic->nic_pbase);
2023 }
2024
2025 /**
2026  * nes_napi_isr
2027  */
2028 int nes_napi_isr(struct nes_device *nesdev)
2029 {
2030         struct nes_adapter *nesadapter = nesdev->nesadapter;
2031         u32 int_stat;
2032
2033         if (nesdev->napi_isr_ran) {
2034                 /* interrupt status has already been read in ISR */
2035                 int_stat = nesdev->int_stat;
2036         } else {
2037                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2038                 nesdev->int_stat = int_stat;
2039                 nesdev->napi_isr_ran = 1;
2040         }
2041
2042         int_stat &= nesdev->int_req;
2043         /* iff NIC, process here, else wait for DPC */
2044         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2045                 nesdev->napi_isr_ran = 0;
2046                 nes_write32(nesdev->regs + NES_INT_STAT,
2047                         (int_stat &
2048                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2049
2050                 /* Process the CEQs */
2051                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2052
2053                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2054                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2055                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2056                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2057                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2058                                 /* Enable Periodic timer interrupts */
2059                                 nesdev->int_req |= NES_INT_TIMER;
2060                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2061                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2062                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2063                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2064                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2065                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2066                         }
2067
2068                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2069                         {
2070                                 nes_nic_init_timer(nesdev);
2071                         }
2072                         /* Enable interrupts, except CEQs */
2073                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2074                 } else {
2075                         /* Enable interrupts, make sure timer is off */
2076                         nesdev->int_req &= ~NES_INT_TIMER;
2077                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2078                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2079                 }
2080                 nesdev->deepcq_count = 0;
2081                 return 1;
2082         } else {
2083                 return 0;
2084         }
2085 }
2086
2087 static void process_critical_error(struct nes_device *nesdev)
2088 {
2089         u32 debug_error;
2090         u32 nes_idx_debug_error_masks0 = 0;
2091         u16 error_module = 0;
2092
2093         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2094         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2095                         (u16)debug_error);
2096         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2097                         0x01010000 | (debug_error & 0x0000ffff));
2098         if (crit_err_count++ > 10)
2099                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2100         error_module = (u16) (debug_error & 0x1F00) >> 8;
2101         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2102                         nes_max_critical_error_count) {
2103                 printk(KERN_ERR PFX "Masking off critical error for module "
2104                         "0x%02X\n", (u16)error_module);
2105                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2106                         NES_IDX_DEBUG_ERROR_MASKS0);
2107                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2108                         nes_idx_debug_error_masks0 | (1 << error_module));
2109         }
2110 }
2111 /**
2112  * nes_dpc
2113  */
2114 void nes_dpc(unsigned long param)
2115 {
2116         struct nes_device *nesdev = (struct nes_device *)param;
2117         struct nes_adapter *nesadapter = nesdev->nesadapter;
2118         u32 counter;
2119         u32 loop_counter = 0;
2120         u32 int_status_bit;
2121         u32 int_stat;
2122         u32 timer_stat;
2123         u32 temp_int_stat;
2124         u32 intf_int_stat;
2125         u32 processed_intf_int = 0;
2126         u16 processed_timer_int = 0;
2127         u16 completion_ints = 0;
2128         u16 timer_ints = 0;
2129
2130         /* nes_debug(NES_DBG_ISR, "\n"); */
2131
2132         do {
2133                 timer_stat = 0;
2134                 if (nesdev->napi_isr_ran) {
2135                         nesdev->napi_isr_ran = 0;
2136                         int_stat = nesdev->int_stat;
2137                 } else
2138                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2139                 if (processed_intf_int != 0)
2140                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2141                 else
2142                         int_stat &= nesdev->int_req;
2143                 if (processed_timer_int == 0) {
2144                         processed_timer_int = 1;
2145                         if (int_stat & NES_INT_TIMER) {
2146                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2147                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2148                                         int_stat &= ~NES_INT_TIMER;
2149                                 }
2150                         }
2151                 } else {
2152                         int_stat &= ~NES_INT_TIMER;
2153                 }
2154
2155                 if (int_stat) {
2156                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2157                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2158                                 /* Ack the interrupts */
2159                                 nes_write32(nesdev->regs+NES_INT_STAT,
2160                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2161                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2162                         }
2163
2164                         temp_int_stat = int_stat;
2165                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2166                                 if (int_stat & int_status_bit) {
2167                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2168                                         temp_int_stat &= ~int_status_bit;
2169                                         completion_ints = 1;
2170                                 }
2171                                 if (!(temp_int_stat & 0x0000ffff))
2172                                         break;
2173                                 int_status_bit <<= 1;
2174                         }
2175
2176                         /* Process the AEQ for this pci function */
2177                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2178                         if (int_stat & int_status_bit) {
2179                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2180                         }
2181
2182                         /* Process the MAC interrupt for this pci function */
2183                         int_status_bit = 1 << (24 + nesdev->mac_index);
2184                         if (int_stat & int_status_bit) {
2185                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2186                         }
2187
2188                         if (int_stat & NES_INT_TIMER) {
2189                                 if (timer_stat & nesdev->timer_int_req) {
2190                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2191                                                         (timer_stat & nesdev->timer_int_req) |
2192                                                         ~(nesdev->nesadapter->timer_int_req));
2193                                         timer_ints = 1;
2194                                 }
2195                         }
2196
2197                         if (int_stat & NES_INT_INTF) {
2198                                 processed_intf_int = 1;
2199                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2200                                 intf_int_stat &= nesdev->intf_int_req;
2201                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2202                                         process_critical_error(nesdev);
2203                                 }
2204                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2205                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2206                                         BUG();
2207                                 }
2208                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2209                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2210                                         BUG();
2211                                 }
2212                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2213                         }
2214
2215                         if (int_stat & NES_INT_TSW) {
2216                         }
2217                 }
2218                 /* Don't use the interface interrupt bit stay in loop */
2219                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2220                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2221         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2222
2223         if (timer_ints == 1) {
2224                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2225                         if (completion_ints == 0) {
2226                                 nesdev->timer_only_int_count++;
2227                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2228                                         nesdev->timer_only_int_count = 0;
2229                                         nesdev->int_req &= ~NES_INT_TIMER;
2230                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2231                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2232                                 } else {
2233                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2234                                 }
2235                         } else {
2236                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2237                                 {
2238                                         nes_nic_init_timer(nesdev);
2239                                 }
2240                                 nesdev->timer_only_int_count = 0;
2241                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2242                         }
2243                 } else {
2244                         nesdev->timer_only_int_count = 0;
2245                         nesdev->int_req &= ~NES_INT_TIMER;
2246                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2247                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2248                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2249                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2250                 }
2251         } else {
2252                 if ( (completion_ints == 1) &&
2253                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2254                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2255                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2256                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2257                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2258                         nesdev->timer_only_int_count = 0;
2259                         nesdev->int_req |= NES_INT_TIMER;
2260                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2261                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2262                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2263                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2264                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2265                 } else {
2266                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2267                 }
2268         }
2269         nesdev->deepcq_count = 0;
2270 }
2271
2272
2273 /**
2274  * nes_process_ceq
2275  */
2276 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2277 {
2278         u64 u64temp;
2279         struct nes_hw_cq *cq;
2280         u32 head;
2281         u32 ceq_size;
2282
2283         /* nes_debug(NES_DBG_CQ, "\n"); */
2284         head = ceq->ceq_head;
2285         ceq_size = ceq->ceq_size;
2286
2287         do {
2288                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2289                                 NES_CEQE_VALID) {
2290                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2291                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2292                         u64temp <<= 1;
2293                         cq = *((struct nes_hw_cq **)&u64temp);
2294                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2295                         barrier();
2296                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2297
2298                         /* call the event handler */
2299                         cq->ce_handler(nesdev, cq);
2300
2301                         if (++head >= ceq_size)
2302                                 head = 0;
2303                 } else {
2304                         break;
2305                 }
2306
2307         } while (1);
2308
2309         ceq->ceq_head = head;
2310 }
2311
2312
2313 /**
2314  * nes_process_aeq
2315  */
2316 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2317 {
2318         /* u64 u64temp; */
2319         u32 head;
2320         u32 aeq_size;
2321         u32 aeqe_misc;
2322         u32 aeqe_cq_id;
2323         struct nes_hw_aeqe volatile *aeqe;
2324
2325         head = aeq->aeq_head;
2326         aeq_size = aeq->aeq_size;
2327
2328         do {
2329                 aeqe = &aeq->aeq_vbase[head];
2330                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2331                         break;
2332                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2333                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2334                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2335                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2336                                 /* dealing with an accelerated QP related AE */
2337                                 /*
2338                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2339                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2340                                  */
2341                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2342                         } else {
2343                                 /* TODO: dealing with a CQP related AE */
2344                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2345                                                 (u16)(aeqe_misc >> 16));
2346                         }
2347                 }
2348
2349                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2350
2351                 if (++head >= aeq_size)
2352                         head = 0;
2353
2354                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2355         }
2356         while (1);
2357         aeq->aeq_head = head;
2358 }
2359
2360 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2361 {
2362         struct nes_adapter *nesadapter = nesdev->nesadapter;
2363         u32 reset_value;
2364         u32 i=0;
2365         u32 u32temp;
2366
2367         if (nesadapter->hw_rev == NE020_REV) {
2368                 return;
2369         }
2370         mh_detected++;
2371
2372         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2373
2374         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2375                 reset_value |= 0x0000001d;
2376         else
2377                 reset_value |= 0x0000002d;
2378
2379         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2380                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2381                         nesadapter->link_interrupt_count[0] = 0;
2382                         nesadapter->link_interrupt_count[1] = 0;
2383                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2384                         if (0x00000040 & u32temp)
2385                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2386                         else
2387                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2388
2389                         reset_value |= 0x0000003d;
2390                 }
2391                 nesadapter->link_interrupt_count[mac_index] = 0;
2392         }
2393
2394         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2395
2396         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2397                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2398
2399         if (0x0000003d == (reset_value & 0x0000003d)) {
2400                 u32 pcs_control_status0, pcs_control_status1;
2401
2402                 for (i = 0; i < 10; i++) {
2403                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2404                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2405                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2406                              && (pcs_control_status0 & 0x00100000))
2407                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2408                                 && (pcs_control_status1 & 0x00100000)))
2409                                 continue;
2410                         else
2411                                 break;
2412                 }
2413                 if (10 == i) {
2414                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2415                         if (0x00000040 & u32temp)
2416                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2417                         else
2418                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2419
2420                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2421
2422                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2423                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2424                 }
2425         }
2426 }
2427
2428 /**
2429  * nes_process_mac_intr
2430  */
2431 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2432 {
2433         unsigned long flags;
2434         u32 pcs_control_status;
2435         struct nes_adapter *nesadapter = nesdev->nesadapter;
2436         struct nes_vnic *nesvnic;
2437         u32 mac_status;
2438         u32 mac_index = nesdev->mac_index;
2439         u32 u32temp;
2440         u16 phy_data;
2441         u16 temp_phy_data;
2442         u32 pcs_val  = 0x0f0f0000;
2443         u32 pcs_mask = 0x0f1f0000;
2444         u32 cdr_ctrl;
2445
2446         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2447         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2448                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2449                 return;
2450         }
2451         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2452         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2453
2454         /* ack the MAC interrupt */
2455         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2456         /* Clear the interrupt */
2457         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2458
2459         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2460
2461         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2462                 nesdev->link_status_interrupts++;
2463                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2464                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2465                         nes_reset_link(nesdev, mac_index);
2466                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2467                 }
2468                 /* read the PHY interrupt status register */
2469                 if ((nesadapter->OneG_Mode) &&
2470                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2471                         do {
2472                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2473                                                 nesadapter->phy_index[mac_index], &phy_data);
2474                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2475                                                 nesadapter->phy_index[mac_index], phy_data);
2476                         } while (phy_data&0x8000);
2477
2478                         temp_phy_data = 0;
2479                         do {
2480                                 nes_read_1G_phy_reg(nesdev, 0x11,
2481                                                 nesadapter->phy_index[mac_index], &phy_data);
2482                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2483                                                 nesadapter->phy_index[mac_index], phy_data);
2484                                 if (temp_phy_data == phy_data)
2485                                         break;
2486                                 temp_phy_data = phy_data;
2487                         } while (1);
2488
2489                         nes_read_1G_phy_reg(nesdev, 0x1e,
2490                                         nesadapter->phy_index[mac_index], &phy_data);
2491                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2492                                         nesadapter->phy_index[mac_index], phy_data);
2493
2494                         nes_read_1G_phy_reg(nesdev, 1,
2495                                         nesadapter->phy_index[mac_index], &phy_data);
2496                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2497                                         nesadapter->phy_index[mac_index], phy_data);
2498
2499                         if (temp_phy_data & 0x1000) {
2500                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2501                                 phy_data = 4;
2502                         } else {
2503                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2504                         }
2505                 }
2506                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2507                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2508                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2509
2510                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2511                         switch (mac_index) {
2512                         case 1:
2513                         case 3:
2514                                 pcs_control_status = nes_read_indexed(nesdev,
2515                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2516                                 break;
2517                         default:
2518                                 pcs_control_status = nes_read_indexed(nesdev,
2519                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2520                                 break;
2521                         }
2522                 } else {
2523                         pcs_control_status = nes_read_indexed(nesdev,
2524                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2525                         pcs_control_status = nes_read_indexed(nesdev,
2526                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2527                 }
2528
2529                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2530                                 mac_index, pcs_control_status);
2531                 if ((nesadapter->OneG_Mode) &&
2532                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2533                         u32temp = 0x01010000;
2534                         if (nesadapter->port_count > 2) {
2535                                 u32temp |= 0x02020000;
2536                         }
2537                         if ((pcs_control_status & u32temp)!= u32temp) {
2538                                 phy_data = 0;
2539                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2540                         }
2541                 } else {
2542                         switch (nesadapter->phy_type[mac_index]) {
2543                         case NES_PHY_TYPE_ARGUS:
2544                         case NES_PHY_TYPE_SFP_D:
2545                         case NES_PHY_TYPE_KR:
2546                                 /* clear the alarms */
2547                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2548                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2549                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2550                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2551                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2552                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2553                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2554                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2555                                 /* check link status */
2556                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2557                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2558
2559                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2560                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2561                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2562                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2563
2564                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2565
2566                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2567                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2568                                 break;
2569
2570                         case NES_PHY_TYPE_PUMA_1G:
2571                                 if (mac_index < 2)
2572                                         pcs_val = pcs_mask = 0x01010000;
2573                                 else
2574                                         pcs_val = pcs_mask = 0x02020000;
2575                                 /* fall through */
2576                         default:
2577                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2578                                 break;
2579                         }
2580                 }
2581
2582                 if (phy_data & 0x0004) {
2583                         if (wide_ppm_offset &&
2584                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2585                             (nesadapter->hw_rev != NE020_REV)) {
2586                                 cdr_ctrl = nes_read_indexed(nesdev,
2587                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2588                                                             mac_index * 0x200);
2589                                 nes_write_indexed(nesdev,
2590                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2591                                                   mac_index * 0x200,
2592                                                   cdr_ctrl | 0x000F0000);
2593                         }
2594                         nesadapter->mac_link_down[mac_index] = 0;
2595                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2596                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2597                                                 nesvnic->linkup);
2598                                 if (nesvnic->linkup == 0) {
2599                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2600                                                         nesvnic->netdev->name, nesvnic->netdev);
2601                                         if (netif_queue_stopped(nesvnic->netdev))
2602                                                 netif_start_queue(nesvnic->netdev);
2603                                         nesvnic->linkup = 1;
2604                                         netif_carrier_on(nesvnic->netdev);
2605                                 }
2606                         }
2607                 } else {
2608                         if (wide_ppm_offset &&
2609                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2610                             (nesadapter->hw_rev != NE020_REV)) {
2611                                 cdr_ctrl = nes_read_indexed(nesdev,
2612                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2613                                                             mac_index * 0x200);
2614                                 nes_write_indexed(nesdev,
2615                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2616                                                   mac_index * 0x200,
2617                                                   cdr_ctrl & 0xFFF0FFFF);
2618                         }
2619                         nesadapter->mac_link_down[mac_index] = 1;
2620                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2621                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2622                                                 nesvnic->linkup);
2623                                 if (nesvnic->linkup == 1) {
2624                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2625                                                         nesvnic->netdev->name, nesvnic->netdev);
2626                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2627                                                 netif_stop_queue(nesvnic->netdev);
2628                                         nesvnic->linkup = 0;
2629                                         netif_carrier_off(nesvnic->netdev);
2630                                 }
2631                         }
2632                 }
2633         }
2634
2635         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2636 }
2637
2638
2639
2640 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2641 {
2642         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2643
2644         napi_schedule(&nesvnic->napi);
2645 }
2646
2647
2648 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2649 * getting out of nic_ce_handler
2650 */
2651 #define MAX_RQES_TO_PROCESS     384
2652
2653 /**
2654  * nes_nic_ce_handler
2655  */
2656 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2657 {
2658         u64 u64temp;
2659         dma_addr_t bus_address;
2660         struct nes_hw_nic *nesnic;
2661         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2662         struct nes_adapter *nesadapter = nesdev->nesadapter;
2663         struct nes_hw_nic_rq_wqe *nic_rqe;
2664         struct nes_hw_nic_sq_wqe *nic_sqe;
2665         struct sk_buff *skb;
2666         struct sk_buff *rx_skb;
2667         __le16 *wqe_fragment_length;
2668         u32 head;
2669         u32 cq_size;
2670         u32 rx_pkt_size;
2671         u32 cqe_count=0;
2672         u32 cqe_errv;
2673         u32 cqe_misc;
2674         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2675         u16 vlan_tag;
2676         u16 pkt_type;
2677         u16 rqes_processed = 0;
2678         u8 sq_cqes = 0;
2679         u8 nes_use_lro = 0;
2680
2681         head = cq->cq_head;
2682         cq_size = cq->cq_size;
2683         cq->cqes_pending = 1;
2684         if (nesvnic->netdev->features & NETIF_F_LRO)
2685                 nes_use_lro = 1;
2686         do {
2687                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2688                                 NES_NIC_CQE_VALID) {
2689                         nesnic = &nesvnic->nic;
2690                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2691                         if (cqe_misc & NES_NIC_CQE_SQ) {
2692                                 sq_cqes++;
2693                                 wqe_fragment_index = 1;
2694                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2695                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2696                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2697                                 /* bump past the vlan tag */
2698                                 wqe_fragment_length++;
2699                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2700                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2701                                                         wqe_fragment_index * 2]);
2702                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2703                                                         wqe_fragment_index * 2])) << 32;
2704                                         bus_address = (dma_addr_t)u64temp;
2705                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2706                                                 pci_unmap_single(nesdev->pcidev,
2707                                                                 bus_address,
2708                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2709                                                                 PCI_DMA_TODEVICE);
2710                                         }
2711                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2712                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2713                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2714                                                                                 wqe_fragment_index * 2]);
2715                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2716                                                                                 + wqe_fragment_index * 2])) <<32;
2717                                                         bus_address = (dma_addr_t)u64temp;
2718                                                         pci_unmap_page(nesdev->pcidev,
2719                                                                         bus_address,
2720                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2721                                                                         PCI_DMA_TODEVICE);
2722                                                 } else
2723                                                         break;
2724                                         }
2725                                 }
2726                                 if (skb)
2727                                         dev_kfree_skb_any(skb);
2728                                 nesnic->sq_tail++;
2729                                 nesnic->sq_tail &= nesnic->sq_size-1;
2730                                 if (sq_cqes > 128) {
2731                                         barrier();
2732                                 /* restart the queue if it had been stopped */
2733                                 if (netif_queue_stopped(nesvnic->netdev))
2734                                         netif_wake_queue(nesvnic->netdev);
2735                                         sq_cqes = 0;
2736                                 }
2737                         } else {
2738                                 rqes_processed ++;
2739
2740                                 cq->rx_cqes_completed++;
2741                                 cq->rx_pkts_indicated++;
2742                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2743                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2744                                 /* Get the skb */
2745                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2746                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2747                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2748                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2749                                 pci_unmap_single(nesdev->pcidev, bus_address,
2750                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2751                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2752                                 /* rx_skb->len = rx_pkt_size; */
2753                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2754                                 skb_put(rx_skb, rx_pkt_size);
2755                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2756                                 nesnic->rq_tail++;
2757                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2758
2759                                 atomic_inc(&nesvnic->rx_skbs_needed);
2760                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2761                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2762                                                         cq->cq_number | (cqe_count << 16));
2763                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2764                                         nesdev->currcq_count += cqe_count;
2765                                         cqe_count = 0;
2766                                         nes_replenish_nic_rq(nesvnic);
2767                                 }
2768                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2769                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2770                                 rx_skb->ip_summed = CHECKSUM_NONE;
2771
2772                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2773                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2774                                         if ((cqe_errv &
2775                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2776                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2777                                                 if (nesvnic->rx_checksum_disabled == 0) {
2778                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2779                                                 }
2780                                         } else
2781                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2782                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2783                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2784
2785                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2786                                         if ((cqe_errv &
2787                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2788                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2789                                                 if (nesvnic->rx_checksum_disabled == 0) {
2790                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2791                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2792                                                                   nesvnic->netdev->name); */
2793                                                 }
2794                                         } else
2795                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2796                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2797                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2798                                         }
2799                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2800                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2801
2802                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2803                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2804                                                 rx_skb = NULL;
2805                                 }
2806                                 if (rx_skb == NULL)
2807                                         goto skip_rx_indicate0;
2808
2809
2810                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2811                                     (nesvnic->vlan_grp != NULL)) {
2812                                         vlan_tag = (u16)(le32_to_cpu(
2813                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2814                                                         >> 16);
2815                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2816                                                         nesvnic->netdev->name, vlan_tag);
2817                                         if (nes_use_lro)
2818                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2819                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2820                                         else
2821                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2822                                 } else {
2823                                         if (nes_use_lro)
2824                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2825                                         else
2826                                                 nes_netif_rx(rx_skb);
2827                                 }
2828
2829 skip_rx_indicate0:
2830                                 ;
2831                                 /* nesvnic->netstats.rx_packets++; */
2832                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2833                         }
2834
2835                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2836                         /* Accounting... */
2837                         cqe_count++;
2838                         if (++head >= cq_size)
2839                                 head = 0;
2840                         if (cqe_count == 255) {
2841                                 /* Replenish Nic CQ */
2842                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2843                                                 cq->cq_number | (cqe_count << 16));
2844                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2845                                 nesdev->currcq_count += cqe_count;
2846                                 cqe_count = 0;
2847                         }
2848
2849                         if (cq->rx_cqes_completed >= nesvnic->budget)
2850                                 break;
2851                 } else {
2852                         cq->cqes_pending = 0;
2853                         break;
2854                 }
2855
2856         } while (1);
2857
2858         if (nes_use_lro)
2859                 lro_flush_all(&nesvnic->lro_mgr);
2860         if (sq_cqes) {
2861                 barrier();
2862                 /* restart the queue if it had been stopped */
2863                 if (netif_queue_stopped(nesvnic->netdev))
2864                         netif_wake_queue(nesvnic->netdev);
2865         }
2866         cq->cq_head = head;
2867         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2868                         cq->cq_number, cqe_count, cq->cq_head); */
2869         cq->cqe_allocs_pending = cqe_count;
2870         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2871         {
2872                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2873                 nesdev->currcq_count += cqe_count;
2874                 nes_nic_tune_timer(nesdev);
2875         }
2876         if (atomic_read(&nesvnic->rx_skbs_needed))
2877                 nes_replenish_nic_rq(nesvnic);
2878 }
2879
2880
2881 /**
2882  * nes_cqp_ce_handler
2883  */
2884 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2885 {
2886         u64 u64temp;
2887         unsigned long flags;
2888         struct nes_hw_cqp *cqp = NULL;
2889         struct nes_cqp_request *cqp_request;
2890         struct nes_hw_cqp_wqe *cqp_wqe;
2891         u32 head;
2892         u32 cq_size;
2893         u32 cqe_count=0;
2894         u32 error_code;
2895         /* u32 counter; */
2896
2897         head = cq->cq_head;
2898         cq_size = cq->cq_size;
2899
2900         do {
2901                 /* process the CQE */
2902                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2903                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2904
2905                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2906                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2907                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2908                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2909                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2910                         cqp = *((struct nes_hw_cqp **)&u64temp);
2911
2912                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2913                         if (error_code) {
2914                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2915                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2916                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2917                                                 (u16)(error_code >> 16),
2918                                                 (u16)error_code);
2919                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2920                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2921                         }
2922
2923                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2924                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2925                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2926                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2927                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2928                         if (cqp_request) {
2929                                 if (cqp_request->waiting) {
2930                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2931                                         cqp_request->major_code = (u16)(error_code >> 16);
2932                                         cqp_request->minor_code = (u16)error_code;
2933                                         barrier();
2934                                         cqp_request->request_done = 1;
2935                                         wake_up(&cqp_request->waitq);
2936                                         nes_put_cqp_request(nesdev, cqp_request);
2937                                 } else {
2938                                         if (cqp_request->callback)
2939                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2940                                         nes_free_cqp_request(nesdev, cqp_request);
2941                                 }
2942                         } else {
2943                                 wake_up(&nesdev->cqp.waitq);
2944                         }
2945
2946                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2947                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2948                         if (++cqp->sq_tail >= cqp->sq_size)
2949                                 cqp->sq_tail = 0;
2950
2951                         /* Accounting... */
2952                         cqe_count++;
2953                         if (++head >= cq_size)
2954                                 head = 0;
2955                 } else {
2956                         break;
2957                 }
2958         } while (1);
2959         cq->cq_head = head;
2960
2961         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2962         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2963                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2964                         (nesdev->cqp.sq_size - 1)) != 1)) {
2965                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2966                                 struct nes_cqp_request, list);
2967                 list_del_init(&cqp_request->list);
2968                 head = nesdev->cqp.sq_head++;
2969                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2970                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2971                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2972                 barrier();
2973                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2974                         cpu_to_le32((u32)((unsigned long)cqp_request));
2975                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2976                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2977                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2978                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2979                 /* Ring doorbell (1 WQEs) */
2980                 barrier();
2981                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2982         }
2983         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2984
2985         /* Arm the CCQ */
2986         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2987                         cq->cq_number);
2988         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2989 }
2990
2991
2992 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
2993 {
2994         u16 pkt_len;
2995
2996         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
2997                 /* skip over ethernet header */
2998                 pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2));
2999                 pkt += ETH_HLEN;
3000
3001                 /* Skip over IP and TCP headers */
3002                 pkt += 4 * (pkt[0] & 0x0f);
3003                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3004         }
3005         return pkt;
3006 }
3007
3008 /* Determine if incoming error pkt is rdma layer */
3009 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3010 {
3011         u8 *pkt;
3012         u16 *mpa;
3013         u32 opcode = 0xffffffff;
3014
3015         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3016                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3017                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3018                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3019         }
3020
3021         return opcode;
3022 }
3023
3024 /* Build iWARP terminate header */
3025 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3026 {
3027         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3028         u16 ddp_seg_len;
3029         int copy_len = 0;
3030         u8 is_tagged = 0;
3031         u8 flush_code = 0;
3032         struct nes_terminate_hdr *termhdr;
3033
3034         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3035         memset(termhdr, 0, 64);
3036
3037         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3038
3039                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3040                 pkt = locate_mpa(pkt, aeq_info);
3041                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3042                 if (ddp_seg_len) {
3043                         copy_len = 2;
3044                         termhdr->hdrct = DDP_LEN_FLAG;
3045                         if (pkt[2] & 0x80) {
3046                                 is_tagged = 1;
3047                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3048                                         copy_len += TERM_DDP_LEN_TAGGED;
3049                                         termhdr->hdrct |= DDP_HDR_FLAG;
3050                                 }
3051                         } else {
3052                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3053                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3054                                         termhdr->hdrct |= DDP_HDR_FLAG;
3055                                 }
3056
3057                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3058                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3059                                                 copy_len += TERM_RDMA_LEN;
3060                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3061                                         }
3062                                 }
3063                         }
3064                 }
3065         }
3066
3067         switch (async_event_id) {
3068         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3069                 switch (iwarp_opcode(nesqp, aeq_info)) {
3070                 case IWARP_OPCODE_WRITE:
3071                         flush_code = IB_WC_LOC_PROT_ERR;
3072                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3073                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3074                         break;
3075                 default:
3076                         flush_code = IB_WC_REM_ACCESS_ERR;
3077                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3078                         termhdr->error_code = RDMAP_INV_STAG;
3079                 }
3080                 break;
3081         case NES_AEQE_AEID_AMP_INVALID_STAG:
3082                 flush_code = IB_WC_REM_ACCESS_ERR;
3083                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3084                 termhdr->error_code = RDMAP_INV_STAG;
3085                 break;
3086         case NES_AEQE_AEID_AMP_BAD_QP:
3087                 flush_code = IB_WC_LOC_QP_OP_ERR;
3088                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3089                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3090                 break;
3091         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3092         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3093                 switch (iwarp_opcode(nesqp, aeq_info)) {
3094                 case IWARP_OPCODE_SEND_INV:
3095                 case IWARP_OPCODE_SEND_SE_INV:
3096                         flush_code = IB_WC_REM_OP_ERR;
3097                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3098                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3099                         break;
3100                 default:
3101                         flush_code = IB_WC_REM_ACCESS_ERR;
3102                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3103                         termhdr->error_code = RDMAP_INV_STAG;
3104                 }
3105                 break;
3106         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3107                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3108                         flush_code = IB_WC_LOC_PROT_ERR;
3109                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3110                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3111                 } else {
3112                         flush_code = IB_WC_REM_ACCESS_ERR;
3113                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3114                         termhdr->error_code = RDMAP_INV_BOUNDS;
3115                 }
3116                 break;
3117         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3118         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3119         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3120                 flush_code = IB_WC_REM_ACCESS_ERR;
3121                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3122                 termhdr->error_code = RDMAP_ACCESS;
3123                 break;
3124         case NES_AEQE_AEID_AMP_TO_WRAP:
3125                 flush_code = IB_WC_REM_ACCESS_ERR;
3126                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3127                 termhdr->error_code = RDMAP_TO_WRAP;
3128                 break;
3129         case NES_AEQE_AEID_AMP_BAD_PD:
3130                 switch (iwarp_opcode(nesqp, aeq_info)) {
3131                 case IWARP_OPCODE_WRITE:
3132                         flush_code = IB_WC_LOC_PROT_ERR;
3133                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3134                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3135                         break;
3136                 case IWARP_OPCODE_SEND_INV:
3137                 case IWARP_OPCODE_SEND_SE_INV:
3138                         flush_code = IB_WC_REM_ACCESS_ERR;
3139                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3140                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3141                         break;
3142                 default:
3143                         flush_code = IB_WC_REM_ACCESS_ERR;
3144                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3145                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3146                 }
3147                 break;
3148         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3149                 flush_code = IB_WC_LOC_LEN_ERR;
3150                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3151                 termhdr->error_code = MPA_MARKER;
3152                 break;
3153         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3154                 flush_code = IB_WC_GENERAL_ERR;
3155                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3156                 termhdr->error_code = MPA_CRC;
3157                 break;
3158         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3159         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3160                 flush_code = IB_WC_LOC_LEN_ERR;
3161                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3162                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3163                 break;
3164         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3165         case NES_AEQE_AEID_DDP_NO_L_BIT:
3166                 flush_code = IB_WC_FATAL_ERR;
3167                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3168                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3169                 break;
3170         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3171         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3172                 flush_code = IB_WC_GENERAL_ERR;
3173                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3174                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3175                 break;
3176         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3177                 flush_code = IB_WC_LOC_LEN_ERR;
3178                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3179                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3180                 break;
3181         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3182                 flush_code = IB_WC_GENERAL_ERR;
3183                 if (is_tagged) {
3184                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3185                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3186                 } else {
3187                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3188                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3189                 }
3190                 break;
3191         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3192                 flush_code = IB_WC_GENERAL_ERR;
3193                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3194                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3195                 break;
3196         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3197                 flush_code = IB_WC_REM_OP_ERR;
3198                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3199                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3200                 break;
3201         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3202                 flush_code = IB_WC_GENERAL_ERR;
3203                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3204                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3205                 break;
3206         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3207                 flush_code = IB_WC_GENERAL_ERR;
3208                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3209                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3210                 break;
3211         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3212                 flush_code = IB_WC_LOC_QP_OP_ERR;
3213                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3214                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3215                 break;
3216         default:
3217                 flush_code = IB_WC_FATAL_ERR;
3218                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3219                 termhdr->error_code = RDMAP_UNSPECIFIED;
3220                 break;
3221         }
3222
3223         if (copy_len)
3224                 memcpy(termhdr + 1, pkt, copy_len);
3225
3226         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3227                 if (aeq_info & NES_AEQE_SQ)
3228                         nesqp->term_sq_flush_code = flush_code;
3229                 else
3230                         nesqp->term_rq_flush_code = flush_code;
3231         }
3232
3233         return sizeof(struct nes_terminate_hdr) + copy_len;
3234 }
3235
3236 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3237                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3238 {
3239         u64 context;
3240         unsigned long flags;
3241         u32 aeq_info;
3242         u16 async_event_id;
3243         u8 tcp_state;
3244         u8 iwarp_state;
3245         u32 termlen = 0;
3246         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3247                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3248         struct nes_adapter *nesadapter = nesdev->nesadapter;
3249
3250         if (nesqp->term_flags & NES_TERM_SENT)
3251                 return; /* Sanity check */
3252
3253         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3254         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3255         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3256         async_event_id = (u16)aeq_info;
3257
3258         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3259                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3260         if (!context) {
3261                 WARN_ON(!context);
3262                 return;
3263         }
3264
3265         nesqp = (struct nes_qp *)(unsigned long)context;
3266         spin_lock_irqsave(&nesqp->lock, flags);
3267         nesqp->hw_iwarp_state = iwarp_state;
3268         nesqp->hw_tcp_state = tcp_state;
3269         nesqp->last_aeq = async_event_id;
3270         nesqp->terminate_eventtype = eventtype;
3271         spin_unlock_irqrestore(&nesqp->lock, flags);
3272
3273         if (nesadapter->send_term_ok)
3274                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3275         else
3276                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3277
3278         nes_terminate_start_timer(nesqp);
3279         nesqp->term_flags |= NES_TERM_SENT;
3280         nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3281 }
3282
3283 static void nes_terminate_send_fin(struct nes_device *nesdev,
3284                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3285 {
3286         u32 aeq_info;
3287         u16 async_event_id;
3288         u8 tcp_state;
3289         u8 iwarp_state;
3290         unsigned long flags;
3291
3292         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3293         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3294         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3295         async_event_id = (u16)aeq_info;
3296
3297         spin_lock_irqsave(&nesqp->lock, flags);
3298         nesqp->hw_iwarp_state = iwarp_state;
3299         nesqp->hw_tcp_state = tcp_state;
3300         nesqp->last_aeq = async_event_id;
3301         spin_unlock_irqrestore(&nesqp->lock, flags);
3302
3303         /* Send the fin only */
3304         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3305                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3306 }
3307
3308 /* Cleanup after a terminate sent or received */
3309 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3310 {
3311         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3312         unsigned long flags;
3313         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3314         struct nes_device *nesdev = nesvnic->nesdev;
3315         u8 first_time = 0;
3316
3317         spin_lock_irqsave(&nesqp->lock, flags);
3318         if (nesqp->hte_added) {
3319                 nesqp->hte_added = 0;
3320                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3321         }
3322
3323         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3324         nesqp->term_flags |= NES_TERM_DONE;
3325         spin_unlock_irqrestore(&nesqp->lock, flags);
3326
3327         /* Make sure we go through this only once */
3328         if (first_time) {
3329                 if (timeout_occurred == 0)
3330                         del_timer(&nesqp->terminate_timer);
3331                 else
3332                         next_iwarp_state |= NES_CQP_QP_RESET;
3333
3334                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3335                 nes_cm_disconn(nesqp);
3336         }
3337 }
3338
3339 static void nes_terminate_received(struct nes_device *nesdev,
3340                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3341 {
3342         u32 aeq_info;
3343         u8 *pkt;
3344         u32 *mpa;
3345         u8 ddp_ctl;
3346         u8 rdma_ctl;
3347         u16 aeq_id = 0;
3348
3349         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3350         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3351                 /* Terminate is not a performance path so the silicon */
3352                 /* did not validate the frame - do it now */
3353                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3354                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3355                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3356                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3357                 if ((ddp_ctl & 0xc0) != 0x40)
3358                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3359                 else if ((ddp_ctl & 0x03) != 1)
3360                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3361                 else if (be32_to_cpu(mpa[2]) != 2)
3362                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3363                 else if (be32_to_cpu(mpa[3]) != 1)
3364                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3365                 else if (be32_to_cpu(mpa[4]) != 0)
3366                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3367                 else if ((rdma_ctl & 0xc0) != 0x40)
3368                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3369
3370                 if (aeq_id) {
3371                         /* Bad terminate recvd - send back a terminate */
3372                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3373                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3374                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3375                         return;
3376                 }
3377         }
3378
3379         nesqp->term_flags |= NES_TERM_RCVD;
3380         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3381         nes_terminate_start_timer(nesqp);
3382         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3383 }
3384
3385 /* Timeout routine in case terminate fails to complete */
3386 static void nes_terminate_timeout(unsigned long context)
3387 {
3388         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3389
3390         nes_terminate_done(nesqp, 1);
3391 }
3392
3393 /* Set a timer in case hw cannot complete the terminate sequence */
3394 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3395 {
3396         init_timer(&nesqp->terminate_timer);
3397         nesqp->terminate_timer.function = nes_terminate_timeout;
3398         nesqp->terminate_timer.expires = jiffies + HZ;
3399         nesqp->terminate_timer.data = (unsigned long)nesqp;
3400         add_timer(&nesqp->terminate_timer);
3401 }
3402
3403 /**
3404  * nes_process_iwarp_aeqe
3405  */
3406 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3407                                    struct nes_hw_aeqe *aeqe)
3408 {
3409         u64 context;
3410         unsigned long flags;
3411         struct nes_qp *nesqp;
3412         struct nes_hw_cq *hw_cq;
3413         struct nes_cq *nescq;
3414         int resource_allocated;
3415         struct nes_adapter *nesadapter = nesdev->nesadapter;
3416         u32 aeq_info;
3417         u32 next_iwarp_state = 0;
3418         u16 async_event_id;
3419         u8 tcp_state;
3420         u8 iwarp_state;
3421         struct ib_event ibevent;
3422
3423         nes_debug(NES_DBG_AEQ, "\n");
3424         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3425         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3426                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3427                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3428         } else {
3429                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3430                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3431                 BUG_ON(!context);
3432         }
3433
3434         /* context is nesqp unless async_event_id == CQ ERROR */
3435         nesqp = (struct nes_qp *)(unsigned long)context;
3436         async_event_id = (u16)aeq_info;
3437         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3438         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3439         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3440                         " Tcp state = %s, iWARP state = %s\n",
3441                         async_event_id,
3442                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3443                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3444
3445         switch (async_event_id) {
3446                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3447                         if (nesqp->term_flags)
3448                                 return; /* Ignore it, wait for close complete */
3449
3450                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3451                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3452                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3453                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3454                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3455                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3456                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3457                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3458                                                 async_event_id, nesqp->last_aeq, tcp_state);
3459                         }
3460
3461                         break;
3462                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3463                         if (nesqp->term_flags) {
3464                                 nes_terminate_done(nesqp, 0);
3465                                 return;
3466                         }
3467                         spin_lock_irqsave(&nesqp->lock, flags);
3468                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3469                         spin_unlock_irqrestore(&nesqp->lock, flags);
3470                         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_CLOSING, 0, 0);
3471                         nes_cm_disconn(nesqp);
3472                         break;
3473
3474                 case NES_AEQE_AEID_RESET_SENT:
3475                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3476                         spin_lock_irqsave(&nesqp->lock, flags);
3477                         nesqp->hw_iwarp_state = iwarp_state;
3478                         nesqp->hw_tcp_state = tcp_state;
3479                         nesqp->last_aeq = async_event_id;
3480                         nesqp->hte_added = 0;
3481                         spin_unlock_irqrestore(&nesqp->lock, flags);
3482                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3483                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3484                         nes_cm_disconn(nesqp);
3485                         break;
3486
3487                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3488                         if (atomic_read(&nesqp->close_timer_started))
3489                                 return;
3490                         spin_lock_irqsave(&nesqp->lock, flags);
3491                         nesqp->hw_iwarp_state = iwarp_state;
3492                         nesqp->hw_tcp_state = tcp_state;
3493                         nesqp->last_aeq = async_event_id;
3494                         spin_unlock_irqrestore(&nesqp->lock, flags);
3495                         nes_cm_disconn(nesqp);
3496                         break;
3497
3498                 case NES_AEQE_AEID_TERMINATE_SENT:
3499                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3500                         break;
3501
3502                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3503                         nes_terminate_received(nesdev, nesqp, aeqe);
3504                         break;
3505
3506                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3507                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3508                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3509                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3510                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3511                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3512                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3513                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3514                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3515                 case NES_AEQE_AEID_AMP_TO_WRAP:
3516                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3517                                         nesqp->hwqp.qp_id, async_event_id);
3518                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3519                         break;
3520
3521                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3522                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3523                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3524                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3525                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3526                                 aeq_info &= 0xffff0000;
3527                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3528                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3529                         }
3530
3531                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3532                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3533                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3534                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3535                 case NES_AEQE_AEID_AMP_BAD_QP:
3536                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3537                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3538                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3539                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3540                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3541                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3542                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3543                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3544                 case NES_AEQE_AEID_AMP_BAD_PD:
3545                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3546                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3547                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3548                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3549                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3550                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3551                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3552                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3553                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3554                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3555                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3556                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3557                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3558                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3559                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3560                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3561                 case NES_AEQE_AEID_BAD_CLOSE:
3562                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3563                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3564                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3565                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3566                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3567                                         nesqp->hwqp.qp_id, async_event_id);
3568                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3569                         break;
3570
3571                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3572                         context <<= 1;
3573                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3574                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3575                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3576                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3577                         if (resource_allocated) {
3578                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3579                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3580                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3581                                 if (hw_cq) {
3582                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3583                                         if (nescq->ibcq.event_handler) {
3584                                                 ibevent.device = nescq->ibcq.device;
3585                                                 ibevent.event = IB_EVENT_CQ_ERR;
3586                                                 ibevent.element.cq = &nescq->ibcq;
3587                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3588                                         }
3589                                 }
3590                         }
3591                         break;
3592
3593                 default:
3594                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3595                                         async_event_id);
3596                         break;
3597         }
3598
3599 }
3600
3601 /**
3602  * nes_iwarp_ce_handler
3603  */
3604 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3605 {
3606         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3607
3608         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3609                         nescq->hw_cq.cq_number); */
3610         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3611
3612         if (nescq->ibcq.comp_handler)
3613                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3614
3615         return;
3616 }
3617
3618
3619 /**
3620  * nes_manage_apbvt()
3621  */
3622 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3623                 u32 nic_index, u32 add_port)
3624 {
3625         struct nes_device *nesdev = nesvnic->nesdev;
3626         struct nes_hw_cqp_wqe *cqp_wqe;
3627         struct nes_cqp_request *cqp_request;
3628         int ret = 0;
3629         u16 major_code;
3630
3631         /* Send manage APBVT request to CQP */
3632         cqp_request = nes_get_cqp_request(nesdev);
3633         if (cqp_request == NULL) {
3634                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3635                 return -ENOMEM;
3636         }
3637         cqp_request->waiting = 1;
3638         cqp_wqe = &cqp_request->cqp_wqe;
3639
3640         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3641                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3642                         accel_local_port, accel_local_port, nic_index);
3643
3644         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3645         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3646                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3647         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3648                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3649
3650         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3651
3652         atomic_set(&cqp_request->refcount, 2);
3653         nes_post_cqp_request(nesdev, cqp_request);
3654
3655         if (add_port == NES_MANAGE_APBVT_ADD)
3656                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3657                                 NES_EVENT_TIMEOUT);
3658         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3659                         ret, cqp_request->major_code, cqp_request->minor_code);
3660         major_code = cqp_request->major_code;
3661
3662         nes_put_cqp_request(nesdev, cqp_request);
3663
3664         if (!ret)
3665                 return -ETIME;
3666         else if (major_code)
3667                 return -EIO;
3668         else
3669                 return 0;
3670 }
3671
3672
3673 /**
3674  * nes_manage_arp_cache
3675  */
3676 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3677                 u32 ip_addr, u32 action)
3678 {
3679         struct nes_hw_cqp_wqe *cqp_wqe;
3680         struct nes_vnic *nesvnic = netdev_priv(netdev);
3681         struct nes_device *nesdev;
3682         struct nes_cqp_request *cqp_request;
3683         int arp_index;
3684
3685         nesdev = nesvnic->nesdev;
3686         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3687         if (arp_index == -1) {
3688                 return;
3689         }
3690
3691         /* update the ARP entry */
3692         cqp_request = nes_get_cqp_request(nesdev);
3693         if (cqp_request == NULL) {
3694                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3695                 return;
3696         }
3697         cqp_request->waiting = 0;
3698         cqp_wqe = &cqp_request->cqp_wqe;
3699         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3700
3701         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3702                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3703         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3704                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3705         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3706
3707         if (action == NES_ARP_ADD) {
3708                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3709                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3710                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3711                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3712                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3713                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3714         } else {
3715                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3716                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3717         }
3718
3719         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3720                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3721
3722         atomic_set(&cqp_request->refcount, 1);
3723         nes_post_cqp_request(nesdev, cqp_request);
3724 }
3725
3726
3727 /**
3728  * flush_wqes
3729  */
3730 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3731                 u32 which_wq, u32 wait_completion)
3732 {
3733         struct nes_cqp_request *cqp_request;
3734         struct nes_hw_cqp_wqe *cqp_wqe;
3735         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3736         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3737         int ret;
3738
3739         cqp_request = nes_get_cqp_request(nesdev);
3740         if (cqp_request == NULL) {
3741                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3742                 return;
3743         }
3744         if (wait_completion) {
3745                 cqp_request->waiting = 1;
3746                 atomic_set(&cqp_request->refcount, 2);
3747         } else {
3748                 cqp_request->waiting = 0;
3749         }
3750         cqp_wqe = &cqp_request->cqp_wqe;
3751         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3752
3753         /* If wqe in error was identified, set code to be put into cqe */
3754         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3755                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3756                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3757                 nesqp->term_sq_flush_code = 0;
3758         }
3759
3760         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3761                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3762                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3763                 nesqp->term_rq_flush_code = 0;
3764         }
3765
3766         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3767                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3768                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3769         }
3770
3771         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3772                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3773         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3774
3775         nes_post_cqp_request(nesdev, cqp_request);
3776
3777         if (wait_completion) {
3778                 /* Wait for CQP */
3779                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3780                                 NES_EVENT_TIMEOUT);
3781                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3782                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3783                                 ret, cqp_request->major_code, cqp_request->minor_code);
3784                 nes_put_cqp_request(nesdev, cqp_request);
3785         }
3786 }