drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32 /**
  33  * bnx2x_move_fp - move content of the fastpath structure.
  34  *
  35  * @bp:         driver handle
  36  * @from:       source FP index
  37  * @to:         destination FP index
  38  *
  39  * Makes sure the contents of the bp->fp[to].napi is kept
  40  * intact. This is done by first copying the napi struct from
  41  * the target to the source, and then mem copying the entire
  42  * source onto the target. Update txdata pointers and related
  43  * content.
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  50         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  51         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  52         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  53         int old_max_eth_txqs, new_max_eth_txqs;
  54         int old_txdata_index = 0, new_txdata_index = 0;
  55
  56         /* Copy the NAPI object as it has been already initialized */
  57         from_fp->napi = to_fp->napi;
  58
  59         /* Move bnx2x_fastpath contents */
  60         memcpy(to_fp, from_fp, sizeof(*to_fp));
  61         to_fp->index = to;
  62
  63         /* move sp_objs contents as well, as their indices match fp ones */
  64         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  65
  66         /* move fp_stats contents as well, as their indices match fp ones */
  67         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  68
  69         /* Update txdata pointers in fp and move txdata content accordingly:
  70          * Each fp consumes 'max_cos' txdata structures, so the index should be
  71          * decremented by max_cos x delta.
  72          */
  73
  74         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  75         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  76                                 (bp)->max_cos;
  77         if (from == FCOE_IDX(bp)) {
  78                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  79                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80         }
  81
  82         memcpy(&bp->bnx2x_txq[new_txdata_index],
  83                &bp->bnx2x_txq[old_txdata_index],
  84                sizeof(struct bnx2x_fp_txdata));
  85         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  86 }
  87
  88 /**
  89  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  90  *
  91  * @bp:        driver handle
  92  * @buf:       character buffer to fill with the fw name
  93  * @buf_len:   length of the above buffer
  94  *
  95  */
  96 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
  97 {
  98         if (IS_PF(bp)) {
  99                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 100
 101                 phy_fw_ver[0] = '\0';
 102                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 103                                              phy_fw_ver, PHY_FW_VER_LEN);
 104                 strlcpy(buf, bp->fw_ver, buf_len);
 105                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 106                          "bc %d.%d.%d%s%s",
 107                          (bp->common.bc_ver & 0xff0000) >> 16,
 108                          (bp->common.bc_ver & 0xff00) >> 8,
 109                          (bp->common.bc_ver & 0xff),
 110                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 111         } else {
 112                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 113         }
 114 }
 115
 116 /**
 117  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 118  *
 119  * @bp: driver handle
 120  * @delta:      number of eth queues which were not allocated
 121  */
 122 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 123 {
 124         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 125
 126         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 127          * backward along the array could cause memory to be overriden
 128          */
 129         for (cos = 1; cos < bp->max_cos; cos++) {
 130                 for (i = 0; i < old_eth_num - delta; i++) {
 131                         struct bnx2x_fastpath *fp = &bp->fp[i];
 132                         int new_idx = cos * (old_eth_num - delta) + i;
 133
 134                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 135                                sizeof(struct bnx2x_fp_txdata));
 136                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 137                 }
 138         }
 139 }
 140
 141 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 142
 143 /* free skb in the packet ring at pos idx
 144  * return idx of last bd freed
 145  */
 146 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 147                              u16 idx, unsigned int *pkts_compl,
 148                              unsigned int *bytes_compl)
 149 {
 150         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 151         struct eth_tx_start_bd *tx_start_bd;
 152         struct eth_tx_bd *tx_data_bd;
 153         struct sk_buff *skb = tx_buf->skb;
 154         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 155         int nbd;
 156         u16 split_bd_len = 0;
 157
 158         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 159         prefetch(&skb->end);
 160
 161         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 162            txdata->txq_index, idx, tx_buf, skb);
 163
 164         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 165
 166
 167         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 168 #ifdef BNX2X_STOP_ON_ERROR
 169         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 170                 BNX2X_ERR("BAD nbd!\n");
 171                 bnx2x_panic();
 172         }
 173 #endif
 174         new_cons = nbd + tx_buf->first_bd;
 175
 176         /* Get the next bd */
 177         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 178
 179         /* Skip a parse bd... */
 180         --nbd;
 181         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 182
 183         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 184         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 185                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 186                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 187                 --nbd;
 188                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 189         }
 190
 191         /* unmap first bd */
 192         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 193                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 194                          DMA_TO_DEVICE);
 195
 196         /* now free frags */
 197         while (nbd > 0) {
 198
 199                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 200                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 201                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 202                 if (--nbd)
 203                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 204         }
 205
 206         /* release skb */
 207         WARN_ON(!skb);
 208         if (likely(skb)) {
 209                 (*pkts_compl)++;
 210                 (*bytes_compl) += skb->len;
 211         }
 212
 213         dev_kfree_skb_any(skb);
 214         tx_buf->first_bd = 0;
 215         tx_buf->skb = NULL;
 216
 217         return new_cons;
 218 }
 219
 220 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 221 {
 222         struct netdev_queue *txq;
 223         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 224         unsigned int pkts_compl = 0, bytes_compl = 0;
 225
 226 #ifdef BNX2X_STOP_ON_ERROR
 227         if (unlikely(bp->panic))
 228                 return -1;
 229 #endif
 230
 231         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 232         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 233         sw_cons = txdata->tx_pkt_cons;
 234
 235         while (sw_cons != hw_cons) {
 236                 u16 pkt_cons;
 237
 238                 pkt_cons = TX_BD(sw_cons);
 239
 240                 DP(NETIF_MSG_TX_DONE,
 241                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 242                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 243
 244                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 245                                             &pkts_compl, &bytes_compl);
 246
 247                 sw_cons++;
 248         }
 249
 250         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 251
 252         txdata->tx_pkt_cons = sw_cons;
 253         txdata->tx_bd_cons = bd_cons;
 254
 255         /* Need to make the tx_bd_cons update visible to start_xmit()
 256          * before checking for netif_tx_queue_stopped().  Without the
 257          * memory barrier, there is a small possibility that
 258          * start_xmit() will miss it and cause the queue to be stopped
 259          * forever.
 260          * On the other hand we need an rmb() here to ensure the proper
 261          * ordering of bit testing in the following
 262          * netif_tx_queue_stopped(txq) call.
 263          */
 264         smp_mb();
 265
 266         if (unlikely(netif_tx_queue_stopped(txq))) {
 267                 /* Taking tx_lock() is needed to prevent reenabling the queue
 268                  * while it's empty. This could have happen if rx_action() gets
 269                  * suspended in bnx2x_tx_int() after the condition before
 270                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 271                  *
 272                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 273                  * sends some packets consuming the whole queue again->
 274                  * stops the queue
 275                  */
 276
 277                 __netif_tx_lock(txq, smp_processor_id());
 278
 279                 if ((netif_tx_queue_stopped(txq)) &&
 280                     (bp->state == BNX2X_STATE_OPEN) &&
 281                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 282                         netif_tx_wake_queue(txq);
 283
 284                 __netif_tx_unlock(txq);
 285         }
 286         return 0;
 287 }
 288
 289 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 290                                              u16 idx)
 291 {
 292         u16 last_max = fp->last_max_sge;
 293
 294         if (SUB_S16(idx, last_max) > 0)
 295                 fp->last_max_sge = idx;
 296 }
 297
 298 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 299                                          u16 sge_len,
 300                                          struct eth_end_agg_rx_cqe *cqe)
 301 {
 302         struct bnx2x *bp = fp->bp;
 303         u16 last_max, last_elem, first_elem;
 304         u16 delta = 0;
 305         u16 i;
 306
 307         if (!sge_len)
 308                 return;
 309
 310         /* First mark all used pages */
 311         for (i = 0; i < sge_len; i++)
 312                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 313                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 314
 315         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 316            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 317
 318         /* Here we assume that the last SGE index is the biggest */
 319         prefetch((void *)(fp->sge_mask));
 320         bnx2x_update_last_max_sge(fp,
 321                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 322
 323         last_max = RX_SGE(fp->last_max_sge);
 324         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 325         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 326
 327         /* If ring is not full */
 328         if (last_elem + 1 != first_elem)
 329                 last_elem++;
 330
 331         /* Now update the prod */
 332         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 333                 if (likely(fp->sge_mask[i]))
 334                         break;
 335
 336                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 337                 delta += BIT_VEC64_ELEM_SZ;
 338         }
 339
 340         if (delta > 0) {
 341                 fp->rx_sge_prod += delta;
 342                 /* clear page-end entries */
 343                 bnx2x_clear_sge_mask_next_elems(fp);
 344         }
 345
 346         DP(NETIF_MSG_RX_STATUS,
 347            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 348            fp->last_max_sge, fp->rx_sge_prod);
 349 }
 350
 351 /* Get Toeplitz hash value in the skb using the value from the
 352  * CQE (calculated by HW).
 353  */
 354 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 355                             const struct eth_fast_path_rx_cqe *cqe,
 356                             bool *l4_rxhash)
 357 {
 358         /* Get Toeplitz hash from CQE */
 359         if ((bp->dev->features & NETIF_F_RXHASH) &&
 360             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 361                 enum eth_rss_hash_type htype;
 362
 363                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 364                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 365                              (htype == TCP_IPV6_HASH_TYPE);
 366                 return le32_to_cpu(cqe->rss_hash_result);
 367         }
 368         *l4_rxhash = false;
 369         return 0;
 370 }
 371
 372 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 373                             u16 cons, u16 prod,
 374                             struct eth_fast_path_rx_cqe *cqe)
 375 {
 376         struct bnx2x *bp = fp->bp;
 377         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 378         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 379         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 380         dma_addr_t mapping;
 381         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 382         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 383
 384         /* print error if current state != stop */
 385         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 386                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 387
 388         /* Try to map an empty data buffer from the aggregation info  */
 389         mapping = dma_map_single(&bp->pdev->dev,
 390                                  first_buf->data + NET_SKB_PAD,
 391                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 392         /*
 393          *  ...if it fails - move the skb from the consumer to the producer
 394          *  and set the current aggregation state as ERROR to drop it
 395          *  when TPA_STOP arrives.
 396          */
 397
 398         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 399                 /* Move the BD from the consumer to the producer */
 400                 bnx2x_reuse_rx_data(fp, cons, prod);
 401                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 402                 return;
 403         }
 404
 405         /* move empty data from pool to prod */
 406         prod_rx_buf->data = first_buf->data;
 407         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 408         /* point prod_bd to new data */
 409         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 410         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 411
 412         /* move partial skb from cons to pool (don't unmap yet) */
 413         *first_buf = *cons_rx_buf;
 414
 415         /* mark bin state as START */
 416         tpa_info->parsing_flags =
 417                 le16_to_cpu(cqe->pars_flags.flags);
 418         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 419         tpa_info->tpa_state = BNX2X_TPA_START;
 420         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 421         tpa_info->placement_offset = cqe->placement_offset;
 422         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 423         if (fp->mode == TPA_MODE_GRO) {
 424                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 425                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 426                 tpa_info->gro_size = gro_size;
 427         }
 428
 429 #ifdef BNX2X_STOP_ON_ERROR
 430         fp->tpa_queue_used |= (1 << queue);
 431 #ifdef _ASM_GENERIC_INT_L64_H
 432         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 433 #else
 434         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 435 #endif
 436            fp->tpa_queue_used);
 437 #endif
 438 }
 439
 440 /* Timestamp option length allowed for TPA aggregation:
 441  *
 442  *              nop nop kind length echo val
 443  */
 444 #define TPA_TSTAMP_OPT_LEN      12
 445 /**
 446  * bnx2x_set_gro_params - compute GRO values
 447  *
 448  * @skb:                packet skb
 449  * @parsing_flags:      parsing flags from the START CQE
 450  * @len_on_bd:          total length of the first packet for the
 451  *                      aggregation.
 452  * @pkt_len:            length of all segments
 453  *
 454  * Approximate value of the MSS for this aggregation calculated using
 455  * the first packet of it.
 456  * Compute number of aggregated segments, and gso_type.
 457  */
 458 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 459                                  u16 len_on_bd, unsigned int pkt_len,
 460                                  u16 num_of_coalesced_segs)
 461 {
 462         /* TPA aggregation won't have either IP options or TCP options
 463          * other than timestamp or IPv6 extension headers.
 464          */
 465         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 466
 467         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 468             PRS_FLAG_OVERETH_IPV6) {
 469                 hdrs_len += sizeof(struct ipv6hdr);
 470                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 471         } else {
 472                 hdrs_len += sizeof(struct iphdr);
 473                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 474         }
 475
 476         /* Check if there was a TCP timestamp, if there is it's will
 477          * always be 12 bytes length: nop nop kind length echo val.
 478          *
 479          * Otherwise FW would close the aggregation.
 480          */
 481         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 482                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 483
 484         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 485
 486         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 487          * to skb_shinfo(skb)->gso_segs
 488          */
 489         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 490 }
 491
 492 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 493                               struct bnx2x_fastpath *fp, u16 index)
 494 {
 495         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 496         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 497         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 498         dma_addr_t mapping;
 499
 500         if (unlikely(page == NULL)) {
 501                 BNX2X_ERR("Can't alloc sge\n");
 502                 return -ENOMEM;
 503         }
 504
 505         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 506                                SGE_PAGES, DMA_FROM_DEVICE);
 507         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 508                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 509                 BNX2X_ERR("Can't map sge\n");
 510                 return -ENOMEM;
 511         }
 512
 513         sw_buf->page = page;
 514         dma_unmap_addr_set(sw_buf, mapping, mapping);
 515
 516         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 517         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 518
 519         return 0;
 520 }
 521
 522 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 523                                struct bnx2x_agg_info *tpa_info,
 524                                u16 pages,
 525                                struct sk_buff *skb,
 526                                struct eth_end_agg_rx_cqe *cqe,
 527                                u16 cqe_idx)
 528 {
 529         struct sw_rx_page *rx_pg, old_rx_pg;
 530         u32 i, frag_len, frag_size;
 531         int err, j, frag_id = 0;
 532         u16 len_on_bd = tpa_info->len_on_bd;
 533         u16 full_page = 0, gro_size = 0;
 534
 535         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 536
 537         if (fp->mode == TPA_MODE_GRO) {
 538                 gro_size = tpa_info->gro_size;
 539                 full_page = tpa_info->full_page;
 540         }
 541
 542         /* This is needed in order to enable forwarding support */
 543         if (frag_size)
 544                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 545                                      le16_to_cpu(cqe->pkt_len),
 546                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 547
 548 #ifdef BNX2X_STOP_ON_ERROR
 549         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 550                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 551                           pages, cqe_idx);
 552                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 553                 bnx2x_panic();
 554                 return -EINVAL;
 555         }
 556 #endif
 557
 558         /* Run through the SGL and compose the fragmented skb */
 559         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 560                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 561
 562                 /* FW gives the indices of the SGE as if the ring is an array
 563                    (meaning that "next" element will consume 2 indices) */
 564                 if (fp->mode == TPA_MODE_GRO)
 565                         frag_len = min_t(u32, frag_size, (u32)full_page);
 566                 else /* LRO */
 567                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 568
 569                 rx_pg = &fp->rx_page_ring[sge_idx];
 570                 old_rx_pg = *rx_pg;
 571
 572                 /* If we fail to allocate a substitute page, we simply stop
 573                    where we are and drop the whole packet */
 574                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 575                 if (unlikely(err)) {
 576                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 577                         return err;
 578                 }
 579
 580                 /* Unmap the page as we r going to pass it to the stack */
 581                 dma_unmap_page(&bp->pdev->dev,
 582                                dma_unmap_addr(&old_rx_pg, mapping),
 583                                SGE_PAGES, DMA_FROM_DEVICE);
 584                 /* Add one frag and update the appropriate fields in the skb */
 585                 if (fp->mode == TPA_MODE_LRO)
 586                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 587                 else { /* GRO */
 588                         int rem;
 589                         int offset = 0;
 590                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 591                                 int len = rem > gro_size ? gro_size : rem;
 592                                 skb_fill_page_desc(skb, frag_id++,
 593                                                    old_rx_pg.page, offset, len);
 594                                 if (offset)
 595                                         get_page(old_rx_pg.page);
 596                                 offset += len;
 597                         }
 598                 }
 599
 600                 skb->data_len += frag_len;
 601                 skb->truesize += SGE_PAGES;
 602                 skb->len += frag_len;
 603
 604                 frag_size -= frag_len;
 605         }
 606
 607         return 0;
 608 }
 609
 610 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 611 {
 612         if (fp->rx_frag_size)
 613                 put_page(virt_to_head_page(data));
 614         else
 615                 kfree(data);
 616 }
 617
 618 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 619 {
 620         if (fp->rx_frag_size)
 621                 return netdev_alloc_frag(fp->rx_frag_size);
 622
 623         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 624 }
 625
 626 #ifdef CONFIG_INET
 627 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 628 {
 629         const struct iphdr *iph = ip_hdr(skb);
 630         struct tcphdr *th;
 631
 632         skb_set_transport_header(skb, sizeof(struct iphdr));
 633         th = tcp_hdr(skb);
 634
 635         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 636                                   iph->saddr, iph->daddr, 0);
 637 }
 638
 639 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 640 {
 641         struct ipv6hdr *iph = ipv6_hdr(skb);
 642         struct tcphdr *th;
 643
 644         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 645         th = tcp_hdr(skb);
 646
 647         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 648                                   &iph->saddr, &iph->daddr, 0);
 649 }
 650
 651 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 652                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 653 {
 654         skb_set_network_header(skb, 0);
 655         gro_func(bp, skb);
 656         tcp_gro_complete(skb);
 657 }
 658 #endif
 659
 660 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 661                                struct sk_buff *skb)
 662 {
 663 #ifdef CONFIG_INET
 664         if (skb_shinfo(skb)->gso_size) {
 665                 switch (be16_to_cpu(skb->protocol)) {
 666                 case ETH_P_IP:
 667                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 668                         break;
 669                 case ETH_P_IPV6:
 670                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 671                         break;
 672                 default:
 673                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 674                                   be16_to_cpu(skb->protocol));
 675                 }
 676         }
 677 #endif
 678         skb_record_rx_queue(skb, fp->rx_queue);
 679         napi_gro_receive(&fp->napi, skb);
 680 }
 681
 682 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 683                            struct bnx2x_agg_info *tpa_info,
 684                            u16 pages,
 685                            struct eth_end_agg_rx_cqe *cqe,
 686                            u16 cqe_idx)
 687 {
 688         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 689         u8 pad = tpa_info->placement_offset;
 690         u16 len = tpa_info->len_on_bd;
 691         struct sk_buff *skb = NULL;
 692         u8 *new_data, *data = rx_buf->data;
 693         u8 old_tpa_state = tpa_info->tpa_state;
 694
 695         tpa_info->tpa_state = BNX2X_TPA_STOP;
 696
 697         /* If we there was an error during the handling of the TPA_START -
 698          * drop this aggregation.
 699          */
 700         if (old_tpa_state == BNX2X_TPA_ERROR)
 701                 goto drop;
 702
 703         /* Try to allocate the new data */
 704         new_data = bnx2x_frag_alloc(fp);
 705         /* Unmap skb in the pool anyway, as we are going to change
 706            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 707            fails. */
 708         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 709                          fp->rx_buf_size, DMA_FROM_DEVICE);
 710         if (likely(new_data))
 711                 skb = build_skb(data, fp->rx_frag_size);
 712
 713         if (likely(skb)) {
 714 #ifdef BNX2X_STOP_ON_ERROR
 715                 if (pad + len > fp->rx_buf_size) {
 716                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 717                                   pad, len, fp->rx_buf_size);
 718                         bnx2x_panic();
 719                         return;
 720                 }
 721 #endif
 722
 723                 skb_reserve(skb, pad + NET_SKB_PAD);
 724                 skb_put(skb, len);
 725                 skb->rxhash = tpa_info->rxhash;
 726                 skb->l4_rxhash = tpa_info->l4_rxhash;
 727
 728                 skb->protocol = eth_type_trans(skb, bp->dev);
 729                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 730
 731                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 732                                          skb, cqe, cqe_idx)) {
 733                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 734                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 735                         bnx2x_gro_receive(bp, fp, skb);
 736                 } else {
 737                         DP(NETIF_MSG_RX_STATUS,
 738                            "Failed to allocate new pages - dropping packet!\n");
 739                         dev_kfree_skb_any(skb);
 740                 }
 741
 742
 743                 /* put new data in bin */
 744                 rx_buf->data = new_data;
 745
 746                 return;
 747         }
 748         if (new_data)
 749                 bnx2x_frag_free(fp, new_data);
 750 drop:
 751         /* drop the packet and keep the buffer in the bin */
 752         DP(NETIF_MSG_RX_STATUS,
 753            "Failed to allocate or map a new skb - dropping packet!\n");
 754         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 755 }
 756
 757 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 758                                struct bnx2x_fastpath *fp, u16 index)
 759 {
 760         u8 *data;
 761         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 762         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 763         dma_addr_t mapping;
 764
 765         data = bnx2x_frag_alloc(fp);
 766         if (unlikely(data == NULL))
 767                 return -ENOMEM;
 768
 769         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 770                                  fp->rx_buf_size,
 771                                  DMA_FROM_DEVICE);
 772         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 773                 bnx2x_frag_free(fp, data);
 774                 BNX2X_ERR("Can't map rx data\n");
 775                 return -ENOMEM;
 776         }
 777
 778         rx_buf->data = data;
 779         dma_unmap_addr_set(rx_buf, mapping, mapping);
 780
 781         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 782         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 783
 784         return 0;
 785 }
 786
 787 static
 788 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 789                                  struct bnx2x_fastpath *fp,
 790                                  struct bnx2x_eth_q_stats *qstats)
 791 {
 792         /* Do nothing if no L4 csum validation was done.
 793          * We do not check whether IP csum was validated. For IPv4 we assume
 794          * that if the card got as far as validating the L4 csum, it also
 795          * validated the IP csum. IPv6 has no IP csum.
 796          */
 797         if (cqe->fast_path_cqe.status_flags &
 798             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 799                 return;
 800
 801         /* If L4 validation was done, check if an error was found. */
 802
 803         if (cqe->fast_path_cqe.type_error_flags &
 804             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 805              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 806                 qstats->hw_csum_err++;
 807         else
 808                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 809 }
 810
 811 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 812 {
 813         struct bnx2x *bp = fp->bp;
 814         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 815         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 816         int rx_pkt = 0;
 817
 818 #ifdef BNX2X_STOP_ON_ERROR
 819         if (unlikely(bp->panic))
 820                 return 0;
 821 #endif
 822
 823         /* CQ "next element" is of the size of the regular element,
 824            that's why it's ok here */
 825         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 826         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 827                 hw_comp_cons++;
 828
 829         bd_cons = fp->rx_bd_cons;
 830         bd_prod = fp->rx_bd_prod;
 831         bd_prod_fw = bd_prod;
 832         sw_comp_cons = fp->rx_comp_cons;
 833         sw_comp_prod = fp->rx_comp_prod;
 834
 835         /* Memory barrier necessary as speculative reads of the rx
 836          * buffer can be ahead of the index in the status block
 837          */
 838         rmb();
 839
 840         DP(NETIF_MSG_RX_STATUS,
 841            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 842            fp->index, hw_comp_cons, sw_comp_cons);
 843
 844         while (sw_comp_cons != hw_comp_cons) {
 845                 struct sw_rx_bd *rx_buf = NULL;
 846                 struct sk_buff *skb;
 847                 union eth_rx_cqe *cqe;
 848                 struct eth_fast_path_rx_cqe *cqe_fp;
 849                 u8 cqe_fp_flags;
 850                 enum eth_rx_cqe_type cqe_fp_type;
 851                 u16 len, pad, queue;
 852                 u8 *data;
 853                 bool l4_rxhash;
 854
 855 #ifdef BNX2X_STOP_ON_ERROR
 856                 if (unlikely(bp->panic))
 857                         return 0;
 858 #endif
 859
 860                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 861                 bd_prod = RX_BD(bd_prod);
 862                 bd_cons = RX_BD(bd_cons);
 863
 864                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 865                 cqe_fp = &cqe->fast_path_cqe;
 866                 cqe_fp_flags = cqe_fp->type_error_flags;
 867                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 868
 869                 DP(NETIF_MSG_RX_STATUS,
 870                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 871                    CQE_TYPE(cqe_fp_flags),
 872                    cqe_fp_flags, cqe_fp->status_flags,
 873                    le32_to_cpu(cqe_fp->rss_hash_result),
 874                    le16_to_cpu(cqe_fp->vlan_tag),
 875                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 876
 877                 /* is this a slowpath msg? */
 878                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 879                         bnx2x_sp_event(fp, cqe);
 880                         goto next_cqe;
 881                 }
 882
 883                 rx_buf = &fp->rx_buf_ring[bd_cons];
 884                 data = rx_buf->data;
 885
 886                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 887                         struct bnx2x_agg_info *tpa_info;
 888                         u16 frag_size, pages;
 889 #ifdef BNX2X_STOP_ON_ERROR
 890                         /* sanity check */
 891                         if (fp->disable_tpa &&
 892                             (CQE_TYPE_START(cqe_fp_type) ||
 893                              CQE_TYPE_STOP(cqe_fp_type)))
 894                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 895                                           CQE_TYPE(cqe_fp_type));
 896 #endif
 897
 898                         if (CQE_TYPE_START(cqe_fp_type)) {
 899                                 u16 queue = cqe_fp->queue_index;
 900                                 DP(NETIF_MSG_RX_STATUS,
 901                                    "calling tpa_start on queue %d\n",
 902                                    queue);
 903
 904                                 bnx2x_tpa_start(fp, queue,
 905                                                 bd_cons, bd_prod,
 906                                                 cqe_fp);
 907
 908                                 goto next_rx;
 909
 910                         }
 911                         queue = cqe->end_agg_cqe.queue_index;
 912                         tpa_info = &fp->tpa_info[queue];
 913                         DP(NETIF_MSG_RX_STATUS,
 914                            "calling tpa_stop on queue %d\n",
 915                            queue);
 916
 917                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 918                                     tpa_info->len_on_bd;
 919
 920                         if (fp->mode == TPA_MODE_GRO)
 921                                 pages = (frag_size + tpa_info->full_page - 1) /
 922                                          tpa_info->full_page;
 923                         else
 924                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 925                                         SGE_PAGE_SHIFT;
 926
 927                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 928                                        &cqe->end_agg_cqe, comp_ring_cons);
 929 #ifdef BNX2X_STOP_ON_ERROR
 930                         if (bp->panic)
 931                                 return 0;
 932 #endif
 933
 934                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 935                         goto next_cqe;
 936                 }
 937                 /* non TPA */
 938                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 939                 pad = cqe_fp->placement_offset;
 940                 dma_sync_single_for_cpu(&bp->pdev->dev,
 941                                         dma_unmap_addr(rx_buf, mapping),
 942                                         pad + RX_COPY_THRESH,
 943                                         DMA_FROM_DEVICE);
 944                 pad += NET_SKB_PAD;
 945                 prefetch(data + pad); /* speedup eth_type_trans() */
 946                 /* is this an error packet? */
 947                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 948                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 949                            "ERROR  flags %x  rx packet %u\n",
 950                            cqe_fp_flags, sw_comp_cons);
 951                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 952                         goto reuse_rx;
 953                 }
 954
 955                 /* Since we don't have a jumbo ring
 956                  * copy small packets if mtu > 1500
 957                  */
 958                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 959                     (len <= RX_COPY_THRESH)) {
 960                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 961                         if (skb == NULL) {
 962                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 963                                    "ERROR  packet dropped because of alloc failure\n");
 964                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 965                                 goto reuse_rx;
 966                         }
 967                         memcpy(skb->data, data + pad, len);
 968                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 969                 } else {
 970                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 971                                 dma_unmap_single(&bp->pdev->dev,
 972                                                  dma_unmap_addr(rx_buf, mapping),
 973                                                  fp->rx_buf_size,
 974                                                  DMA_FROM_DEVICE);
 975                                 skb = build_skb(data, fp->rx_frag_size);
 976                                 if (unlikely(!skb)) {
 977                                         bnx2x_frag_free(fp, data);
 978                                         bnx2x_fp_qstats(bp, fp)->
 979                                                         rx_skb_alloc_failed++;
 980                                         goto next_rx;
 981                                 }
 982                                 skb_reserve(skb, pad);
 983                         } else {
 984                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 985                                    "ERROR  packet dropped because of alloc failure\n");
 986                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 987 reuse_rx:
 988                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 989                                 goto next_rx;
 990                         }
 991                 }
 992
 993                 skb_put(skb, len);
 994                 skb->protocol = eth_type_trans(skb, bp->dev);
 995
 996                 /* Set Toeplitz hash for a none-LRO skb */
 997                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 998                 skb->l4_rxhash = l4_rxhash;
 999
1000                 skb_checksum_none_assert(skb);
1001
1002                 if (bp->dev->features & NETIF_F_RXCSUM)
1003                         bnx2x_csum_validate(skb, cqe, fp,
1004                                             bnx2x_fp_qstats(bp, fp));
1005
1006                 skb_record_rx_queue(skb, fp->rx_queue);
1007
1008                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1009                     PARSING_FLAGS_VLAN)
1010                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1011                                                le16_to_cpu(cqe_fp->vlan_tag));
1012                 napi_gro_receive(&fp->napi, skb);
1013
1014
1015 next_rx:
1016                 rx_buf->data = NULL;
1017
1018                 bd_cons = NEXT_RX_IDX(bd_cons);
1019                 bd_prod = NEXT_RX_IDX(bd_prod);
1020                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1021                 rx_pkt++;
1022 next_cqe:
1023                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1024                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1025
1026                 if (rx_pkt == budget)
1027                         break;
1028         } /* while */
1029
1030         fp->rx_bd_cons = bd_cons;
1031         fp->rx_bd_prod = bd_prod_fw;
1032         fp->rx_comp_cons = sw_comp_cons;
1033         fp->rx_comp_prod = sw_comp_prod;
1034
1035         /* Update producers */
1036         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1037                              fp->rx_sge_prod);
1038
1039         fp->rx_pkt += rx_pkt;
1040         fp->rx_calls++;
1041
1042         return rx_pkt;
1043 }
1044
1045 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1046 {
1047         struct bnx2x_fastpath *fp = fp_cookie;
1048         struct bnx2x *bp = fp->bp;
1049         u8 cos;
1050
1051         DP(NETIF_MSG_INTR,
1052            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1053            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1054
1055         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1056
1057 #ifdef BNX2X_STOP_ON_ERROR
1058         if (unlikely(bp->panic))
1059                 return IRQ_HANDLED;
1060 #endif
1061
1062         /* Handle Rx and Tx according to MSI-X vector */
1063         prefetch(fp->rx_cons_sb);
1064
1065         for_each_cos_in_tx_queue(fp, cos)
1066                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1067
1068         prefetch(&fp->sb_running_index[SM_RX_ID]);
1069         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1070
1071         return IRQ_HANDLED;
1072 }
1073
1074 /* HW Lock for shared dual port PHYs */
1075 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1076 {
1077         mutex_lock(&bp->port.phy_mutex);
1078
1079         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1080 }
1081
1082 void bnx2x_release_phy_lock(struct bnx2x *bp)
1083 {
1084         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1085
1086         mutex_unlock(&bp->port.phy_mutex);
1087 }
1088
1089 /* calculates MF speed according to current linespeed and MF configuration */
1090 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1091 {
1092         u16 line_speed = bp->link_vars.line_speed;
1093         if (IS_MF(bp)) {
1094                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1095                                                    bp->mf_config[BP_VN(bp)]);
1096
1097                 /* Calculate the current MAX line speed limit for the MF
1098                  * devices
1099                  */
1100                 if (IS_MF_SI(bp))
1101                         line_speed = (line_speed * maxCfg) / 100;
1102                 else { /* SD mode */
1103                         u16 vn_max_rate = maxCfg * 100;
1104
1105                         if (vn_max_rate < line_speed)
1106                                 line_speed = vn_max_rate;
1107                 }
1108         }
1109
1110         return line_speed;
1111 }
1112
1113 /**
1114  * bnx2x_fill_report_data - fill link report data to report
1115  *
1116  * @bp:         driver handle
1117  * @data:       link state to update
1118  *
1119  * It uses a none-atomic bit operations because is called under the mutex.
1120  */
1121 static void bnx2x_fill_report_data(struct bnx2x *bp,
1122                                    struct bnx2x_link_report_data *data)
1123 {
1124         u16 line_speed = bnx2x_get_mf_speed(bp);
1125
1126         memset(data, 0, sizeof(*data));
1127
1128         /* Fill the report data: efective line speed */
1129         data->line_speed = line_speed;
1130
1131         /* Link is down */
1132         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1133                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1134                           &data->link_report_flags);
1135
1136         /* Full DUPLEX */
1137         if (bp->link_vars.duplex == DUPLEX_FULL)
1138                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1139
1140         /* Rx Flow Control is ON */
1141         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1142                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1143
1144         /* Tx Flow Control is ON */
1145         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1146                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1147 }
1148
1149 /**
1150  * bnx2x_link_report - report link status to OS.
1151  *
1152  * @bp:         driver handle
1153  *
1154  * Calls the __bnx2x_link_report() under the same locking scheme
1155  * as a link/PHY state managing code to ensure a consistent link
1156  * reporting.
1157  */
1158
1159 void bnx2x_link_report(struct bnx2x *bp)
1160 {
1161         bnx2x_acquire_phy_lock(bp);
1162         __bnx2x_link_report(bp);
1163         bnx2x_release_phy_lock(bp);
1164 }
1165
1166 /**
1167  * __bnx2x_link_report - report link status to OS.
1168  *
1169  * @bp:         driver handle
1170  *
1171  * None atomic inmlementation.
1172  * Should be called under the phy_lock.
1173  */
1174 void __bnx2x_link_report(struct bnx2x *bp)
1175 {
1176         struct bnx2x_link_report_data cur_data;
1177
1178         /* reread mf_cfg */
1179         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1180                 bnx2x_read_mf_cfg(bp);
1181
1182         /* Read the current link report info */
1183         bnx2x_fill_report_data(bp, &cur_data);
1184
1185         /* Don't report link down or exactly the same link status twice */
1186         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1187             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1188                       &bp->last_reported_link.link_report_flags) &&
1189              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1190                       &cur_data.link_report_flags)))
1191                 return;
1192
1193         bp->link_cnt++;
1194
1195         /* We are going to report a new link parameters now -
1196          * remember the current data for the next time.
1197          */
1198         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1199
1200         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1201                      &cur_data.link_report_flags)) {
1202                 netif_carrier_off(bp->dev);
1203                 netdev_err(bp->dev, "NIC Link is Down\n");
1204                 return;
1205         } else {
1206                 const char *duplex;
1207                 const char *flow;
1208
1209                 netif_carrier_on(bp->dev);
1210
1211                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1212                                        &cur_data.link_report_flags))
1213                         duplex = "full";
1214                 else
1215                         duplex = "half";
1216
1217                 /* Handle the FC at the end so that only these flags would be
1218                  * possibly set. This way we may easily check if there is no FC
1219                  * enabled.
1220                  */
1221                 if (cur_data.link_report_flags) {
1222                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1223                                      &cur_data.link_report_flags)) {
1224                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1225                                      &cur_data.link_report_flags))
1226                                         flow = "ON - receive & transmit";
1227                                 else
1228                                         flow = "ON - receive";
1229                         } else {
1230                                 flow = "ON - transmit";
1231                         }
1232                 } else {
1233                         flow = "none";
1234                 }
1235                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1236                             cur_data.line_speed, duplex, flow);
1237         }
1238 }
1239
1240 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1241 {
1242         int i;
1243
1244         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1245                 struct eth_rx_sge *sge;
1246
1247                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1248                 sge->addr_hi =
1249                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1250                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1251
1252                 sge->addr_lo =
1253                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1254                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1255         }
1256 }
1257
1258 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1259                                 struct bnx2x_fastpath *fp, int last)
1260 {
1261         int i;
1262
1263         for (i = 0; i < last; i++) {
1264                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1265                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1266                 u8 *data = first_buf->data;
1267
1268                 if (data == NULL) {
1269                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1270                         continue;
1271                 }
1272                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1273                         dma_unmap_single(&bp->pdev->dev,
1274                                          dma_unmap_addr(first_buf, mapping),
1275                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1276                 bnx2x_frag_free(fp, data);
1277                 first_buf->data = NULL;
1278         }
1279 }
1280
1281 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1282 {
1283         int j;
1284
1285         for_each_rx_queue_cnic(bp, j) {
1286                 struct bnx2x_fastpath *fp = &bp->fp[j];
1287
1288                 fp->rx_bd_cons = 0;
1289
1290                 /* Activate BD ring */
1291                 /* Warning!
1292                  * this will generate an interrupt (to the TSTORM)
1293                  * must only be done after chip is initialized
1294                  */
1295                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1296                                      fp->rx_sge_prod);
1297         }
1298 }
1299
1300 void bnx2x_init_rx_rings(struct bnx2x *bp)
1301 {
1302         int func = BP_FUNC(bp);
1303         u16 ring_prod;
1304         int i, j;
1305
1306         /* Allocate TPA resources */
1307         for_each_eth_queue(bp, j) {
1308                 struct bnx2x_fastpath *fp = &bp->fp[j];
1309
1310                 DP(NETIF_MSG_IFUP,
1311                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1312
1313                 if (!fp->disable_tpa) {
1314                         /* Fill the per-aggregtion pool */
1315                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1316                                 struct bnx2x_agg_info *tpa_info =
1317                                         &fp->tpa_info[i];
1318                                 struct sw_rx_bd *first_buf =
1319                                         &tpa_info->first_buf;
1320
1321                                 first_buf->data = bnx2x_frag_alloc(fp);
1322                                 if (!first_buf->data) {
1323                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1324                                                   j);
1325                                         bnx2x_free_tpa_pool(bp, fp, i);
1326                                         fp->disable_tpa = 1;
1327                                         break;
1328                                 }
1329                                 dma_unmap_addr_set(first_buf, mapping, 0);
1330                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1331                         }
1332
1333                         /* "next page" elements initialization */
1334                         bnx2x_set_next_page_sgl(fp);
1335
1336                         /* set SGEs bit mask */
1337                         bnx2x_init_sge_ring_bit_mask(fp);
1338
1339                         /* Allocate SGEs and initialize the ring elements */
1340                         for (i = 0, ring_prod = 0;
1341                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1342
1343                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1344                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1345                                                   i);
1346                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1347                                                   j);
1348                                         /* Cleanup already allocated elements */
1349                                         bnx2x_free_rx_sge_range(bp, fp,
1350                                                                 ring_prod);
1351                                         bnx2x_free_tpa_pool(bp, fp,
1352                                                             MAX_AGG_QS(bp));
1353                                         fp->disable_tpa = 1;
1354                                         ring_prod = 0;
1355                                         break;
1356                                 }
1357                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1358                         }
1359
1360                         fp->rx_sge_prod = ring_prod;
1361                 }
1362         }
1363
1364         for_each_eth_queue(bp, j) {
1365                 struct bnx2x_fastpath *fp = &bp->fp[j];
1366
1367                 fp->rx_bd_cons = 0;
1368
1369                 /* Activate BD ring */
1370                 /* Warning!
1371                  * this will generate an interrupt (to the TSTORM)
1372                  * must only be done after chip is initialized
1373                  */
1374                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1375                                      fp->rx_sge_prod);
1376
1377                 if (j != 0)
1378                         continue;
1379
1380                 if (CHIP_IS_E1(bp)) {
1381                         REG_WR(bp, BAR_USTRORM_INTMEM +
1382                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1383                                U64_LO(fp->rx_comp_mapping));
1384                         REG_WR(bp, BAR_USTRORM_INTMEM +
1385                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1386                                U64_HI(fp->rx_comp_mapping));
1387                 }
1388         }
1389 }
1390
1391 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1392 {
1393         u8 cos;
1394         struct bnx2x *bp = fp->bp;
1395
1396         for_each_cos_in_tx_queue(fp, cos) {
1397                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1398                 unsigned pkts_compl = 0, bytes_compl = 0;
1399
1400                 u16 sw_prod = txdata->tx_pkt_prod;
1401                 u16 sw_cons = txdata->tx_pkt_cons;
1402
1403                 while (sw_cons != sw_prod) {
1404                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1405                                           &pkts_compl, &bytes_compl);
1406                         sw_cons++;
1407                 }
1408
1409                 netdev_tx_reset_queue(
1410                         netdev_get_tx_queue(bp->dev,
1411                                             txdata->txq_index));
1412         }
1413 }
1414
1415 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1416 {
1417         int i;
1418
1419         for_each_tx_queue_cnic(bp, i) {
1420                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1421         }
1422 }
1423
1424 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1425 {
1426         int i;
1427
1428         for_each_eth_queue(bp, i) {
1429                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1430         }
1431 }
1432
1433 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1434 {
1435         struct bnx2x *bp = fp->bp;
1436         int i;
1437
1438         /* ring wasn't allocated */
1439         if (fp->rx_buf_ring == NULL)
1440                 return;
1441
1442         for (i = 0; i < NUM_RX_BD; i++) {
1443                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1444                 u8 *data = rx_buf->data;
1445
1446                 if (data == NULL)
1447                         continue;
1448                 dma_unmap_single(&bp->pdev->dev,
1449                                  dma_unmap_addr(rx_buf, mapping),
1450                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1451
1452                 rx_buf->data = NULL;
1453                 bnx2x_frag_free(fp, data);
1454         }
1455 }
1456
1457 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1458 {
1459         int j;
1460
1461         for_each_rx_queue_cnic(bp, j) {
1462                 bnx2x_free_rx_bds(&bp->fp[j]);
1463         }
1464 }
1465
1466 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1467 {
1468         int j;
1469
1470         for_each_eth_queue(bp, j) {
1471                 struct bnx2x_fastpath *fp = &bp->fp[j];
1472
1473                 bnx2x_free_rx_bds(fp);
1474
1475                 if (!fp->disable_tpa)
1476                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1477         }
1478 }
1479
1480 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1481 {
1482         bnx2x_free_tx_skbs_cnic(bp);
1483         bnx2x_free_rx_skbs_cnic(bp);
1484 }
1485
1486 void bnx2x_free_skbs(struct bnx2x *bp)
1487 {
1488         bnx2x_free_tx_skbs(bp);
1489         bnx2x_free_rx_skbs(bp);
1490 }
1491
1492 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1493 {
1494         /* load old values */
1495         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1496
1497         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1498                 /* leave all but MAX value */
1499                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1500
1501                 /* set new MAX value */
1502                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1503                                 & FUNC_MF_CFG_MAX_BW_MASK;
1504
1505                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1506         }
1507 }
1508
1509 /**
1510  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1511  *
1512  * @bp:         driver handle
1513  * @nvecs:      number of vectors to be released
1514  */
1515 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1516 {
1517         int i, offset = 0;
1518
1519         if (nvecs == offset)
1520                 return;
1521
1522         /* VFs don't have a default SB */
1523         if (IS_PF(bp)) {
1524                 free_irq(bp->msix_table[offset].vector, bp->dev);
1525                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1526                    bp->msix_table[offset].vector);
1527                 offset++;
1528         }
1529
1530         if (CNIC_SUPPORT(bp)) {
1531                 if (nvecs == offset)
1532                         return;
1533                 offset++;
1534         }
1535
1536         for_each_eth_queue(bp, i) {
1537                 if (nvecs == offset)
1538                         return;
1539                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1540                    i, bp->msix_table[offset].vector);
1541
1542                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1543         }
1544 }
1545
1546 void bnx2x_free_irq(struct bnx2x *bp)
1547 {
1548         if (bp->flags & USING_MSIX_FLAG &&
1549             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1550                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1551
1552                 /* vfs don't have a default status block */
1553                 if (IS_PF(bp))
1554                         nvecs++;
1555
1556                 bnx2x_free_msix_irqs(bp, nvecs);
1557         } else {
1558                 free_irq(bp->dev->irq, bp->dev);
1559         }
1560 }
1561
1562 int bnx2x_enable_msix(struct bnx2x *bp)
1563 {
1564         int msix_vec = 0, i, rc;
1565
1566         /* VFs don't have a default status block */
1567         if (IS_PF(bp)) {
1568                 bp->msix_table[msix_vec].entry = msix_vec;
1569                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1570                                bp->msix_table[0].entry);
1571                 msix_vec++;
1572         }
1573
1574         /* Cnic requires an msix vector for itself */
1575         if (CNIC_SUPPORT(bp)) {
1576                 bp->msix_table[msix_vec].entry = msix_vec;
1577                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1578                                msix_vec, bp->msix_table[msix_vec].entry);
1579                 msix_vec++;
1580         }
1581
1582         /* We need separate vectors for ETH queues only (not FCoE) */
1583         for_each_eth_queue(bp, i) {
1584                 bp->msix_table[msix_vec].entry = msix_vec;
1585                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1586                                msix_vec, msix_vec, i);
1587                 msix_vec++;
1588         }
1589
1590         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1591            msix_vec);
1592
1593         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1594
1595         /*
1596          * reconfigure number of tx/rx queues according to available
1597          * MSI-X vectors
1598          */
1599         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1600                 /* how less vectors we will have? */
1601                 int diff = msix_vec - rc;
1602
1603                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1604
1605                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1606
1607                 if (rc) {
1608                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1609                         goto no_msix;
1610                 }
1611                 /*
1612                  * decrease number of queues by number of unallocated entries
1613                  */
1614                 bp->num_ethernet_queues -= diff;
1615                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1616
1617                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1618                                bp->num_queues);
1619         } else if (rc > 0) {
1620                 /* Get by with single vector */
1621                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1622                 if (rc) {
1623                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1624                                        rc);
1625                         goto no_msix;
1626                 }
1627
1628                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1629                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1630
1631                 BNX2X_DEV_INFO("set number of queues to 1\n");
1632                 bp->num_ethernet_queues = 1;
1633                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1634         } else if (rc < 0) {
1635                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1636                 goto no_msix;
1637         }
1638
1639         bp->flags |= USING_MSIX_FLAG;
1640
1641         return 0;
1642
1643 no_msix:
1644         /* fall to INTx if not enough memory */
1645         if (rc == -ENOMEM)
1646                 bp->flags |= DISABLE_MSI_FLAG;
1647
1648         return rc;
1649 }
1650
1651 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1652 {
1653         int i, rc, offset = 0;
1654
1655         /* no default status block for vf */
1656         if (IS_PF(bp)) {
1657                 rc = request_irq(bp->msix_table[offset++].vector,
1658                                  bnx2x_msix_sp_int, 0,
1659                                  bp->dev->name, bp->dev);
1660                 if (rc) {
1661                         BNX2X_ERR("request sp irq failed\n");
1662                         return -EBUSY;
1663                 }
1664         }
1665
1666         if (CNIC_SUPPORT(bp))
1667                 offset++;
1668
1669         for_each_eth_queue(bp, i) {
1670                 struct bnx2x_fastpath *fp = &bp->fp[i];
1671                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1672                          bp->dev->name, i);
1673
1674                 rc = request_irq(bp->msix_table[offset].vector,
1675                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1676                 if (rc) {
1677                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1678                               bp->msix_table[offset].vector, rc);
1679                         bnx2x_free_msix_irqs(bp, offset);
1680                         return -EBUSY;
1681                 }
1682
1683                 offset++;
1684         }
1685
1686         i = BNX2X_NUM_ETH_QUEUES(bp);
1687         if (IS_PF(bp)) {
1688                 offset = 1 + CNIC_SUPPORT(bp);
1689                 netdev_info(bp->dev,
1690                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1691                             bp->msix_table[0].vector,
1692                             0, bp->msix_table[offset].vector,
1693                             i - 1, bp->msix_table[offset + i - 1].vector);
1694         } else {
1695                 offset = CNIC_SUPPORT(bp);
1696                 netdev_info(bp->dev,
1697                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1698                             0, bp->msix_table[offset].vector,
1699                             i - 1, bp->msix_table[offset + i - 1].vector);
1700         }
1701         return 0;
1702 }
1703
1704 int bnx2x_enable_msi(struct bnx2x *bp)
1705 {
1706         int rc;
1707
1708         rc = pci_enable_msi(bp->pdev);
1709         if (rc) {
1710                 BNX2X_DEV_INFO("MSI is not attainable\n");
1711                 return -1;
1712         }
1713         bp->flags |= USING_MSI_FLAG;
1714
1715         return 0;
1716 }
1717
1718 static int bnx2x_req_irq(struct bnx2x *bp)
1719 {
1720         unsigned long flags;
1721         unsigned int irq;
1722
1723         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1724                 flags = 0;
1725         else
1726                 flags = IRQF_SHARED;
1727
1728         if (bp->flags & USING_MSIX_FLAG)
1729                 irq = bp->msix_table[0].vector;
1730         else
1731                 irq = bp->pdev->irq;
1732
1733         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1734 }
1735
1736 int bnx2x_setup_irqs(struct bnx2x *bp)
1737 {
1738         int rc = 0;
1739         if (bp->flags & USING_MSIX_FLAG &&
1740             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1741                 rc = bnx2x_req_msix_irqs(bp);
1742                 if (rc)
1743                         return rc;
1744         } else {
1745                 rc = bnx2x_req_irq(bp);
1746                 if (rc) {
1747                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1748                         return rc;
1749                 }
1750                 if (bp->flags & USING_MSI_FLAG) {
1751                         bp->dev->irq = bp->pdev->irq;
1752                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1753                                     bp->dev->irq);
1754                 }
1755                 if (bp->flags & USING_MSIX_FLAG) {
1756                         bp->dev->irq = bp->msix_table[0].vector;
1757                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1758                                     bp->dev->irq);
1759                 }
1760         }
1761
1762         return 0;
1763 }
1764
1765 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1766 {
1767         int i;
1768
1769         for_each_rx_queue_cnic(bp, i)
1770                 napi_enable(&bnx2x_fp(bp, i, napi));
1771 }
1772
1773 static void bnx2x_napi_enable(struct bnx2x *bp)
1774 {
1775         int i;
1776
1777         for_each_eth_queue(bp, i)
1778                 napi_enable(&bnx2x_fp(bp, i, napi));
1779 }
1780
1781 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1782 {
1783         int i;
1784
1785         for_each_rx_queue_cnic(bp, i)
1786                 napi_disable(&bnx2x_fp(bp, i, napi));
1787 }
1788
1789 static void bnx2x_napi_disable(struct bnx2x *bp)
1790 {
1791         int i;
1792
1793         for_each_eth_queue(bp, i)
1794                 napi_disable(&bnx2x_fp(bp, i, napi));
1795 }
1796
1797 void bnx2x_netif_start(struct bnx2x *bp)
1798 {
1799         if (netif_running(bp->dev)) {
1800                 bnx2x_napi_enable(bp);
1801                 if (CNIC_LOADED(bp))
1802                         bnx2x_napi_enable_cnic(bp);
1803                 bnx2x_int_enable(bp);
1804                 if (bp->state == BNX2X_STATE_OPEN)
1805                         netif_tx_wake_all_queues(bp->dev);
1806         }
1807 }
1808
1809 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1810 {
1811         bnx2x_int_disable_sync(bp, disable_hw);
1812         bnx2x_napi_disable(bp);
1813         if (CNIC_LOADED(bp))
1814                 bnx2x_napi_disable_cnic(bp);
1815 }
1816
1817 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1818 {
1819         struct bnx2x *bp = netdev_priv(dev);
1820
1821         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1822                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1823                 u16 ether_type = ntohs(hdr->h_proto);
1824
1825                 /* Skip VLAN tag if present */
1826                 if (ether_type == ETH_P_8021Q) {
1827                         struct vlan_ethhdr *vhdr =
1828                                 (struct vlan_ethhdr *)skb->data;
1829
1830                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1831                 }
1832
1833                 /* If ethertype is FCoE or FIP - use FCoE ring */
1834                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1835                         return bnx2x_fcoe_tx(bp, txq_index);
1836         }
1837
1838         /* select a non-FCoE queue */
1839         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1840 }
1841
1842 void bnx2x_set_num_queues(struct bnx2x *bp)
1843 {
1844         /* RSS queues */
1845         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1846
1847         /* override in STORAGE SD modes */
1848         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1849                 bp->num_ethernet_queues = 1;
1850
1851         /* Add special queues */
1852         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1853         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1854
1855         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1856 }
1857
1858 /**
1859  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1860  *
1861  * @bp:         Driver handle
1862  *
1863  * We currently support for at most 16 Tx queues for each CoS thus we will
1864  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1865  * bp->max_cos.
1866  *
1867  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1868  * index after all ETH L2 indices.
1869  *
1870  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1871  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1872  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1873  *
1874  * The proper configuration of skb->queue_mapping is handled by
1875  * bnx2x_select_queue() and __skb_tx_hash().
1876  *
1877  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1878  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1879  */
1880 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1881 {
1882         int rc, tx, rx;
1883
1884         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1885         rx = BNX2X_NUM_ETH_QUEUES(bp);
1886
1887 /* account for fcoe queue */
1888         if (include_cnic && !NO_FCOE(bp)) {
1889                 rx++;
1890                 tx++;
1891         }
1892
1893         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1894         if (rc) {
1895                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1896                 return rc;
1897         }
1898         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1899         if (rc) {
1900                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1901                 return rc;
1902         }
1903
1904         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1905                           tx, rx);
1906
1907         return rc;
1908 }
1909
1910 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1911 {
1912         int i;
1913
1914         for_each_queue(bp, i) {
1915                 struct bnx2x_fastpath *fp = &bp->fp[i];
1916                 u32 mtu;
1917
1918                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1919                 if (IS_FCOE_IDX(i))
1920                         /*
1921                          * Although there are no IP frames expected to arrive to
1922                          * this ring we still want to add an
1923                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1924                          * overrun attack.
1925                          */
1926                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1927                 else
1928                         mtu = bp->dev->mtu;
1929                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1930                                   IP_HEADER_ALIGNMENT_PADDING +
1931                                   ETH_OVREHEAD +
1932                                   mtu +
1933                                   BNX2X_FW_RX_ALIGN_END;
1934                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1935                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1936                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1937                 else
1938                         fp->rx_frag_size = 0;
1939         }
1940 }
1941
1942 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1943 {
1944         int i;
1945         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1946
1947         /* Prepare the initial contents fo the indirection table if RSS is
1948          * enabled
1949          */
1950         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1951                 bp->rss_conf_obj.ind_table[i] =
1952                         bp->fp->cl_id +
1953                         ethtool_rxfh_indir_default(i, num_eth_queues);
1954
1955         /*
1956          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1957          * per-port, so if explicit configuration is needed , do it only
1958          * for a PMF.
1959          *
1960          * For 57712 and newer on the other hand it's a per-function
1961          * configuration.
1962          */
1963         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1964 }
1965
1966 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1967                         bool config_hash)
1968 {
1969         struct bnx2x_config_rss_params params = {NULL};
1970
1971         /* Although RSS is meaningless when there is a single HW queue we
1972          * still need it enabled in order to have HW Rx hash generated.
1973          *
1974          * if (!is_eth_multi(bp))
1975          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1976          */
1977
1978         params.rss_obj = rss_obj;
1979
1980         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1981
1982         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1983
1984         /* RSS configuration */
1985         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1986         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1987         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1988         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1989         if (rss_obj->udp_rss_v4)
1990                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1991         if (rss_obj->udp_rss_v6)
1992                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1993
1994         /* Hash bits */
1995         params.rss_result_mask = MULTI_MASK;
1996
1997         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1998
1999         if (config_hash) {
2000                 /* RSS keys */
2001                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
2002                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2003         }
2004
2005         return bnx2x_config_rss(bp, &params);
2006 }
2007
2008 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2009 {
2010         struct bnx2x_func_state_params func_params = {NULL};
2011
2012         /* Prepare parameters for function state transitions */
2013         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2014
2015         func_params.f_obj = &bp->func_obj;
2016         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2017
2018         func_params.params.hw_init.load_phase = load_code;
2019
2020         return bnx2x_func_state_change(bp, &func_params);
2021 }
2022
2023 /*
2024  * Cleans the object that have internal lists without sending
2025  * ramrods. Should be run when interrutps are disabled.
2026  */
2027 void bnx2x_squeeze_objects(struct bnx2x *bp)
2028 {
2029         int rc;
2030         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2031         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2032         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2033
2034         /***************** Cleanup MACs' object first *************************/
2035
2036         /* Wait for completion of requested */
2037         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2038         /* Perform a dry cleanup */
2039         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2040
2041         /* Clean ETH primary MAC */
2042         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2043         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2044                                  &ramrod_flags);
2045         if (rc != 0)
2046                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2047
2048         /* Cleanup UC list */
2049         vlan_mac_flags = 0;
2050         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2051         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2052                                  &ramrod_flags);
2053         if (rc != 0)
2054                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2055
2056         /***************** Now clean mcast object *****************************/
2057         rparam.mcast_obj = &bp->mcast_obj;
2058         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2059
2060         /* Add a DEL command... */
2061         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2062         if (rc < 0)
2063                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2064                           rc);
2065
2066         /* ...and wait until all pending commands are cleared */
2067         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2068         while (rc != 0) {
2069                 if (rc < 0) {
2070                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2071                                   rc);
2072                         return;
2073                 }
2074
2075                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2076         }
2077 }
2078
2079 #ifndef BNX2X_STOP_ON_ERROR
2080 #define LOAD_ERROR_EXIT(bp, label) \
2081         do { \
2082                 (bp)->state = BNX2X_STATE_ERROR; \
2083                 goto label; \
2084         } while (0)
2085
2086 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2087         do { \
2088                 bp->cnic_loaded = false; \
2089                 goto label; \
2090         } while (0)
2091 #else /*BNX2X_STOP_ON_ERROR*/
2092 #define LOAD_ERROR_EXIT(bp, label) \
2093         do { \
2094                 (bp)->state = BNX2X_STATE_ERROR; \
2095                 (bp)->panic = 1; \
2096                 return -EBUSY; \
2097         } while (0)
2098 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2099         do { \
2100                 bp->cnic_loaded = false; \
2101                 (bp)->panic = 1; \
2102                 return -EBUSY; \
2103         } while (0)
2104 #endif /*BNX2X_STOP_ON_ERROR*/
2105
2106 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2107 {
2108         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2109                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2110         return;
2111 }
2112
2113 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2114 {
2115         int num_groups, vf_headroom = 0;
2116         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2117
2118         /* number of queues for statistics is number of eth queues + FCoE */
2119         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2120
2121         /* Total number of FW statistics requests =
2122          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2123          * and fcoe l2 queue) stats + num of queues (which includes another 1
2124          * for fcoe l2 queue if applicable)
2125          */
2126         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2127
2128         /* vf stats appear in the request list, but their data is allocated by
2129          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2130          * it is used to determine where to place the vf stats queries in the
2131          * request struct
2132          */
2133         if (IS_SRIOV(bp))
2134                 vf_headroom = bnx2x_vf_headroom(bp);
2135
2136         /* Request is built from stats_query_header and an array of
2137          * stats_query_cmd_group each of which contains
2138          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2139          * configured in the stats_query_header.
2140          */
2141         num_groups =
2142                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2143                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2144                  1 : 0));
2145
2146         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2147            bp->fw_stats_num, vf_headroom, num_groups);
2148         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2149                 num_groups * sizeof(struct stats_query_cmd_group);
2150
2151         /* Data for statistics requests + stats_counter
2152          * stats_counter holds per-STORM counters that are incremented
2153          * when STORM has finished with the current request.
2154          * memory for FCoE offloaded statistics are counted anyway,
2155          * even if they will not be sent.
2156          * VF stats are not accounted for here as the data of VF stats is stored
2157          * in memory allocated by the VF, not here.
2158          */
2159         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2160                 sizeof(struct per_pf_stats) +
2161                 sizeof(struct fcoe_statistics_params) +
2162                 sizeof(struct per_queue_stats) * num_queue_stats +
2163                 sizeof(struct stats_counter);
2164
2165         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2166                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2167
2168         /* Set shortcuts */
2169         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2170         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2171         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2172                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2173         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2174                 bp->fw_stats_req_sz;
2175
2176         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x",
2177            U64_HI(bp->fw_stats_req_mapping),
2178            U64_LO(bp->fw_stats_req_mapping));
2179         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x",
2180            U64_HI(bp->fw_stats_data_mapping),
2181            U64_LO(bp->fw_stats_data_mapping));
2182         return 0;
2183
2184 alloc_mem_err:
2185         bnx2x_free_fw_stats_mem(bp);
2186         BNX2X_ERR("Can't allocate FW stats memory\n");
2187         return -ENOMEM;
2188 }
2189
2190 /* send load request to mcp and analyze response */
2191 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2192 {
2193         /* init fw_seq */
2194         bp->fw_seq =
2195                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2196                  DRV_MSG_SEQ_NUMBER_MASK);
2197         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2198
2199         /* Get current FW pulse sequence */
2200         bp->fw_drv_pulse_wr_seq =
2201                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2202                  DRV_PULSE_SEQ_MASK);
2203         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2204
2205         /* load request */
2206         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2207                                         DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2208
2209         /* if mcp fails to respond we must abort */
2210         if (!(*load_code)) {
2211                 BNX2X_ERR("MCP response failure, aborting\n");
2212                 return -EBUSY;
2213         }
2214
2215         /* If mcp refused (e.g. other port is in diagnostic mode) we
2216          * must abort
2217          */
2218         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2219                 BNX2X_ERR("MCP refused load request, aborting\n");
2220                 return -EBUSY;
2221         }
2222         return 0;
2223 }
2224
2225 /* check whether another PF has already loaded FW to chip. In
2226  * virtualized environments a pf from another VM may have already
2227  * initialized the device including loading FW
2228  */
2229 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2230 {
2231         /* is another pf loaded on this engine? */
2232         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2233             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2234                 /* build my FW version dword */
2235                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2236                         (BCM_5710_FW_MINOR_VERSION << 8) +
2237                         (BCM_5710_FW_REVISION_VERSION << 16) +
2238                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2239
2240                 /* read loaded FW from chip */
2241                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2242
2243                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2244                    loaded_fw, my_fw);
2245
2246                 /* abort nic load if version mismatch */
2247                 if (my_fw != loaded_fw) {
2248                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. aborting\n",
2249                                   loaded_fw, my_fw);
2250                         return -EBUSY;
2251                 }
2252         }
2253         return 0;
2254 }
2255
2256 /* returns the "mcp load_code" according to global load_count array */
2257 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2258 {
2259         int path = BP_PATH(bp);
2260
2261         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2262            path, load_count[path][0], load_count[path][1],
2263            load_count[path][2]);
2264         load_count[path][0]++;
2265         load_count[path][1 + port]++;
2266         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2267            path, load_count[path][0], load_count[path][1],
2268            load_count[path][2]);
2269         if (load_count[path][0] == 1)
2270                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2271         else if (load_count[path][1 + port] == 1)
2272                 return FW_MSG_CODE_DRV_LOAD_PORT;
2273         else
2274                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2275 }
2276
2277 /* mark PMF if applicable */
2278 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2279 {
2280         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2281             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2282             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2283                 bp->port.pmf = 1;
2284                 /* We need the barrier to ensure the ordering between the
2285                  * writing to bp->port.pmf here and reading it from the
2286                  * bnx2x_periodic_task().
2287                  */
2288                 smp_mb();
2289         } else {
2290                 bp->port.pmf = 0;
2291         }
2292
2293         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2294 }
2295
2296 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2297 {
2298         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2299              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2300             (bp->common.shmem2_base)) {
2301                 if (SHMEM2_HAS(bp, dcc_support))
2302                         SHMEM2_WR(bp, dcc_support,
2303                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2304                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2305                 if (SHMEM2_HAS(bp, afex_driver_support))
2306                         SHMEM2_WR(bp, afex_driver_support,
2307                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2308         }
2309
2310         /* Set AFEX default VLAN tag to an invalid value */
2311         bp->afex_def_vlan_tag = -1;
2312 }
2313
2314 /**
2315  * bnx2x_bz_fp - zero content of the fastpath structure.
2316  *
2317  * @bp:         driver handle
2318  * @index:      fastpath index to be zeroed
2319  *
2320  * Makes sure the contents of the bp->fp[index].napi is kept
2321  * intact.
2322  */
2323 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2324 {
2325         struct bnx2x_fastpath *fp = &bp->fp[index];
2326
2327         int cos;
2328         struct napi_struct orig_napi = fp->napi;
2329         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2330         /* bzero bnx2x_fastpath contents */
2331         if (fp->tpa_info)
2332                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2333                        sizeof(struct bnx2x_agg_info));
2334         memset(fp, 0, sizeof(*fp));
2335
2336         /* Restore the NAPI object as it has been already initialized */
2337         fp->napi = orig_napi;
2338         fp->tpa_info = orig_tpa_info;
2339         fp->bp = bp;
2340         fp->index = index;
2341         if (IS_ETH_FP(fp))
2342                 fp->max_cos = bp->max_cos;
2343         else
2344                 /* Special queues support only one CoS */
2345                 fp->max_cos = 1;
2346
2347         /* Init txdata pointers */
2348         if (IS_FCOE_FP(fp))
2349                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2350         if (IS_ETH_FP(fp))
2351                 for_each_cos_in_tx_queue(fp, cos)
2352                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2353                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2354
2355         /*
2356          * set the tpa flag for each queue. The tpa flag determines the queue
2357          * minimal size so it must be set prior to queue memory allocation
2358          */
2359         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2360                                   (bp->flags & GRO_ENABLE_FLAG &&
2361                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2362         if (bp->flags & TPA_ENABLE_FLAG)
2363                 fp->mode = TPA_MODE_LRO;
2364         else if (bp->flags & GRO_ENABLE_FLAG)
2365                 fp->mode = TPA_MODE_GRO;
2366
2367         /* We don't want TPA on an FCoE L2 ring */
2368         if (IS_FCOE_FP(fp))
2369                 fp->disable_tpa = 1;
2370 }
2371
2372 int bnx2x_load_cnic(struct bnx2x *bp)
2373 {
2374         int i, rc, port = BP_PORT(bp);
2375
2376         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2377
2378         mutex_init(&bp->cnic_mutex);
2379
2380         if (IS_PF(bp)) {
2381                 rc = bnx2x_alloc_mem_cnic(bp);
2382                 if (rc) {
2383                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2384                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2385                 }
2386         }
2387
2388         rc = bnx2x_alloc_fp_mem_cnic(bp);
2389         if (rc) {
2390                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2391                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2392         }
2393
2394         /* Update the number of queues with the cnic queues */
2395         rc = bnx2x_set_real_num_queues(bp, 1);
2396         if (rc) {
2397                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2398                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2399         }
2400
2401         /* Add all CNIC NAPI objects */
2402         bnx2x_add_all_napi_cnic(bp);
2403         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2404         bnx2x_napi_enable_cnic(bp);
2405
2406         rc = bnx2x_init_hw_func_cnic(bp);
2407         if (rc)
2408                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2409
2410         bnx2x_nic_init_cnic(bp);
2411
2412         if (IS_PF(bp)) {
2413                 /* Enable Timer scan */
2414                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2415
2416                 /* setup cnic queues */
2417                 for_each_cnic_queue(bp, i) {
2418                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2419                         if (rc) {
2420                                 BNX2X_ERR("Queue setup failed\n");
2421                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2422                         }
2423                 }
2424         }
2425
2426         /* Initialize Rx filter. */
2427         netif_addr_lock_bh(bp->dev);
2428         bnx2x_set_rx_mode(bp->dev);
2429         netif_addr_unlock_bh(bp->dev);
2430
2431         /* re-read iscsi info */
2432         bnx2x_get_iscsi_info(bp);
2433         bnx2x_setup_cnic_irq_info(bp);
2434         bnx2x_setup_cnic_info(bp);
2435         bp->cnic_loaded = true;
2436         if (bp->state == BNX2X_STATE_OPEN)
2437                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2438
2439
2440         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2441
2442         return 0;
2443
2444 #ifndef BNX2X_STOP_ON_ERROR
2445 load_error_cnic2:
2446         /* Disable Timer scan */
2447         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2448
2449 load_error_cnic1:
2450         bnx2x_napi_disable_cnic(bp);
2451         /* Update the number of queues without the cnic queues */
2452         rc = bnx2x_set_real_num_queues(bp, 0);
2453         if (rc)
2454                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2455 load_error_cnic0:
2456         BNX2X_ERR("CNIC-related load failed\n");
2457         bnx2x_free_fp_mem_cnic(bp);
2458         bnx2x_free_mem_cnic(bp);
2459         return rc;
2460 #endif /* ! BNX2X_STOP_ON_ERROR */
2461 }
2462
2463 /* must be called with rtnl_lock */
2464 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2465 {
2466         int port = BP_PORT(bp);
2467         int i, rc = 0, load_code = 0;
2468
2469         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2470         DP(NETIF_MSG_IFUP,
2471            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2472
2473 #ifdef BNX2X_STOP_ON_ERROR
2474         if (unlikely(bp->panic)) {
2475                 BNX2X_ERR("Can't load NIC when there is panic\n");
2476                 return -EPERM;
2477         }
2478 #endif
2479
2480         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2481
2482         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2483         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2484                 &bp->last_reported_link.link_report_flags);
2485
2486         if (IS_PF(bp))
2487                 /* must be called before memory allocation and HW init */
2488                 bnx2x_ilt_set_info(bp);
2489
2490         /*
2491          * Zero fastpath structures preserving invariants like napi, which are
2492          * allocated only once, fp index, max_cos, bp pointer.
2493          * Also set fp->disable_tpa and txdata_ptr.
2494          */
2495         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2496         for_each_queue(bp, i)
2497                 bnx2x_bz_fp(bp, i);
2498         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2499                                   bp->num_cnic_queues) *
2500                                   sizeof(struct bnx2x_fp_txdata));
2501
2502         bp->fcoe_init = false;
2503
2504         /* Set the receive queues buffer size */
2505         bnx2x_set_rx_buf_size(bp);
2506
2507         if (IS_PF(bp)) {
2508                 rc = bnx2x_alloc_mem(bp);
2509                 if (rc) {
2510                         BNX2X_ERR("Unable to allocate bp memory\n");
2511                         return rc;
2512                 }
2513         }
2514
2515         /* Allocated memory for FW statistics  */
2516         if (bnx2x_alloc_fw_stats_mem(bp))
2517                 LOAD_ERROR_EXIT(bp, load_error0);
2518
2519         /* need to be done after alloc mem, since it's self adjusting to amount
2520          * of memory available for RSS queues
2521          */
2522         rc = bnx2x_alloc_fp_mem(bp);
2523         if (rc) {
2524                 BNX2X_ERR("Unable to allocate memory for fps\n");
2525                 LOAD_ERROR_EXIT(bp, load_error0);
2526         }
2527
2528         /* request pf to initialize status blocks */
2529         if (IS_VF(bp)) {
2530                 rc = bnx2x_vfpf_init(bp);
2531                 if (rc)
2532                         LOAD_ERROR_EXIT(bp, load_error0);
2533         }
2534
2535         /* As long as bnx2x_alloc_mem() may possibly update
2536          * bp->num_queues, bnx2x_set_real_num_queues() should always
2537          * come after it. At this stage cnic queues are not counted.
2538          */
2539         rc = bnx2x_set_real_num_queues(bp, 0);
2540         if (rc) {
2541                 BNX2X_ERR("Unable to set real_num_queues\n");
2542                 LOAD_ERROR_EXIT(bp, load_error0);
2543         }
2544
2545         /* configure multi cos mappings in kernel.
2546          * this configuration may be overriden by a multi class queue discipline
2547          * or by a dcbx negotiation result.
2548          */
2549         bnx2x_setup_tc(bp->dev, bp->max_cos);
2550
2551         /* Add all NAPI objects */
2552         bnx2x_add_all_napi(bp);
2553         DP(NETIF_MSG_IFUP, "napi added\n");
2554         bnx2x_napi_enable(bp);
2555
2556         if (IS_PF(bp)) {
2557                 /* set pf load just before approaching the MCP */
2558                 bnx2x_set_pf_load(bp);
2559
2560                 /* if mcp exists send load request and analyze response */
2561                 if (!BP_NOMCP(bp)) {
2562                         /* attempt to load pf */
2563                         rc = bnx2x_nic_load_request(bp, &load_code);
2564                         if (rc)
2565                                 LOAD_ERROR_EXIT(bp, load_error1);
2566
2567                         /* what did mcp say? */
2568                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2569                         if (rc) {
2570                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2571                                 LOAD_ERROR_EXIT(bp, load_error2);
2572                         }
2573                 } else {
2574                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2575                 }
2576
2577                 /* mark pmf if applicable */
2578                 bnx2x_nic_load_pmf(bp, load_code);
2579
2580                 /* Init Function state controlling object */
2581                 bnx2x__init_func_obj(bp);
2582
2583                 /* Initialize HW */
2584                 rc = bnx2x_init_hw(bp, load_code);
2585                 if (rc) {
2586                         BNX2X_ERR("HW init failed, aborting\n");
2587                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2588                         LOAD_ERROR_EXIT(bp, load_error2);
2589                 }
2590         }
2591
2592         bnx2x_pre_irq_nic_init(bp);
2593
2594         /* Connect to IRQs */
2595         rc = bnx2x_setup_irqs(bp);
2596         if (rc) {
2597                 BNX2X_ERR("setup irqs failed\n");
2598                 if (IS_PF(bp))
2599                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2600                 LOAD_ERROR_EXIT(bp, load_error2);
2601         }
2602
2603         /* Init per-function objects */
2604         if (IS_PF(bp)) {
2605                 /* Setup NIC internals and enable interrupts */
2606                 bnx2x_post_irq_nic_init(bp, load_code);
2607
2608                 bnx2x_init_bp_objs(bp);
2609                 bnx2x_iov_nic_init(bp);
2610
2611                 /* Set AFEX default VLAN tag to an invalid value */
2612                 bp->afex_def_vlan_tag = -1;
2613                 bnx2x_nic_load_afex_dcc(bp, load_code);
2614                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2615                 rc = bnx2x_func_start(bp);
2616                 if (rc) {
2617                         BNX2X_ERR("Function start failed!\n");
2618                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2619
2620                         LOAD_ERROR_EXIT(bp, load_error3);
2621                 }
2622
2623                 /* Send LOAD_DONE command to MCP */
2624                 if (!BP_NOMCP(bp)) {
2625                         load_code = bnx2x_fw_command(bp,
2626                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2627                         if (!load_code) {
2628                                 BNX2X_ERR("MCP response failure, aborting\n");
2629                                 rc = -EBUSY;
2630                                 LOAD_ERROR_EXIT(bp, load_error3);
2631                         }
2632                 }
2633
2634                 /* initialize FW coalescing state machines in RAM */
2635                 bnx2x_update_coalesce(bp);
2636
2637                 /* setup the leading queue */
2638                 rc = bnx2x_setup_leading(bp);
2639                 if (rc) {
2640                         BNX2X_ERR("Setup leading failed!\n");
2641                         LOAD_ERROR_EXIT(bp, load_error3);
2642                 }
2643
2644                 /* set up the rest of the queues */
2645                 for_each_nondefault_eth_queue(bp, i) {
2646                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2647                         if (rc) {
2648                                 BNX2X_ERR("Queue setup failed\n");
2649                                 LOAD_ERROR_EXIT(bp, load_error3);
2650                         }
2651                 }
2652
2653                 /* setup rss */
2654                 rc = bnx2x_init_rss_pf(bp);
2655                 if (rc) {
2656                         BNX2X_ERR("PF RSS init failed\n");
2657                         LOAD_ERROR_EXIT(bp, load_error3);
2658                 }
2659
2660         } else { /* vf */
2661                 for_each_eth_queue(bp, i) {
2662                         rc = bnx2x_vfpf_setup_q(bp, i);
2663                         if (rc) {
2664                                 BNX2X_ERR("Queue setup failed\n");
2665                                 LOAD_ERROR_EXIT(bp, load_error3);
2666                         }
2667                 }
2668         }
2669
2670         /* Now when Clients are configured we are ready to work */
2671         bp->state = BNX2X_STATE_OPEN;
2672
2673         /* Configure a ucast MAC */
2674         if (IS_PF(bp))
2675                 rc = bnx2x_set_eth_mac(bp, true);
2676         else /* vf */
2677                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2678                                            true);
2679         if (rc) {
2680                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2681                 LOAD_ERROR_EXIT(bp, load_error3);
2682         }
2683
2684         if (IS_PF(bp) && bp->pending_max) {
2685                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2686                 bp->pending_max = 0;
2687         }
2688
2689         if (bp->port.pmf) {
2690                 rc = bnx2x_initial_phy_init(bp, load_mode);
2691                 if (rc)
2692                         LOAD_ERROR_EXIT(bp, load_error3);
2693         }
2694         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2695
2696         /* Start fast path */
2697
2698         /* Initialize Rx filter. */
2699         netif_addr_lock_bh(bp->dev);
2700         bnx2x_set_rx_mode(bp->dev);
2701         netif_addr_unlock_bh(bp->dev);
2702
2703         /* Start the Tx */
2704         switch (load_mode) {
2705         case LOAD_NORMAL:
2706                 /* Tx queue should be only reenabled */
2707                 netif_tx_wake_all_queues(bp->dev);
2708                 break;
2709
2710         case LOAD_OPEN:
2711                 netif_tx_start_all_queues(bp->dev);
2712                 smp_mb__after_clear_bit();
2713                 break;
2714
2715         case LOAD_DIAG:
2716         case LOAD_LOOPBACK_EXT:
2717                 bp->state = BNX2X_STATE_DIAG;
2718                 break;
2719
2720         default:
2721                 break;
2722         }
2723
2724         if (bp->port.pmf)
2725                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2726         else
2727                 bnx2x__link_status_update(bp);
2728
2729         /* start the timer */
2730         mod_timer(&bp->timer, jiffies + bp->current_interval);
2731
2732         if (CNIC_ENABLED(bp))
2733                 bnx2x_load_cnic(bp);
2734
2735         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2736                 /* mark driver is loaded in shmem2 */
2737                 u32 val;
2738                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2739                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2740                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2741                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2742         }
2743
2744         /* Wait for all pending SP commands to complete */
2745         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2746                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2747                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2748                 return -EBUSY;
2749         }
2750
2751         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2752         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2753                 bnx2x_dcbx_init(bp, false);
2754
2755         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2756
2757         return 0;
2758
2759 #ifndef BNX2X_STOP_ON_ERROR
2760 load_error3:
2761         if (IS_PF(bp)) {
2762                 bnx2x_int_disable_sync(bp, 1);
2763
2764                 /* Clean queueable objects */
2765                 bnx2x_squeeze_objects(bp);
2766         }
2767
2768         /* Free SKBs, SGEs, TPA pool and driver internals */
2769         bnx2x_free_skbs(bp);
2770         for_each_rx_queue(bp, i)
2771                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2772
2773         /* Release IRQs */
2774         bnx2x_free_irq(bp);
2775 load_error2:
2776         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2777                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2778                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2779         }
2780
2781         bp->port.pmf = 0;
2782 load_error1:
2783         bnx2x_napi_disable(bp);
2784         bnx2x_del_all_napi(bp);
2785
2786         /* clear pf_load status, as it was already set */
2787         if (IS_PF(bp))
2788                 bnx2x_clear_pf_load(bp);
2789 load_error0:
2790         bnx2x_free_fp_mem(bp);
2791         bnx2x_free_fw_stats_mem(bp);
2792         bnx2x_free_mem(bp);
2793
2794         return rc;
2795 #endif /* ! BNX2X_STOP_ON_ERROR */
2796 }
2797
2798 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2799 {
2800         u8 rc = 0, cos, i;
2801
2802         /* Wait until tx fastpath tasks complete */
2803         for_each_tx_queue(bp, i) {
2804                 struct bnx2x_fastpath *fp = &bp->fp[i];
2805
2806                 for_each_cos_in_tx_queue(fp, cos)
2807                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2808                 if (rc)
2809                         return rc;
2810         }
2811         return 0;
2812 }
2813
2814 /* must be called with rtnl_lock */
2815 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2816 {
2817         int i;
2818         bool global = false;
2819
2820         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2821
2822         /* mark driver is unloaded in shmem2 */
2823         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2824                 u32 val;
2825                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2826                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2827                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2828         }
2829
2830         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2831             (bp->state == BNX2X_STATE_CLOSED ||
2832              bp->state == BNX2X_STATE_ERROR)) {
2833                 /* We can get here if the driver has been unloaded
2834                  * during parity error recovery and is either waiting for a
2835                  * leader to complete or for other functions to unload and
2836                  * then ifdown has been issued. In this case we want to
2837                  * unload and let other functions to complete a recovery
2838                  * process.
2839                  */
2840                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2841                 bp->is_leader = 0;
2842                 bnx2x_release_leader_lock(bp);
2843                 smp_mb();
2844
2845                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2846                 BNX2X_ERR("Can't unload in closed or error state\n");
2847                 return -EINVAL;
2848         }
2849
2850         /* Nothing to do during unload if previous bnx2x_nic_load()
2851          * have not completed succesfully - all resourses are released.
2852          *
2853          * we can get here only after unsuccessful ndo_* callback, during which
2854          * dev->IFF_UP flag is still on.
2855          */
2856         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2857                 return 0;
2858
2859         /* It's important to set the bp->state to the value different from
2860          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2861          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2862          */
2863         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2864         smp_mb();
2865
2866         if (CNIC_LOADED(bp))
2867                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2868
2869         /* Stop Tx */
2870         bnx2x_tx_disable(bp);
2871         netdev_reset_tc(bp->dev);
2872
2873         bp->rx_mode = BNX2X_RX_MODE_NONE;
2874
2875         del_timer_sync(&bp->timer);
2876
2877         if (IS_PF(bp)) {
2878                 /* Set ALWAYS_ALIVE bit in shmem */
2879                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2880                 bnx2x_drv_pulse(bp);
2881                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2882                 bnx2x_save_statistics(bp);
2883         }
2884
2885         /* wait till consumers catch up with producers in all queues */
2886         bnx2x_drain_tx_queues(bp);
2887
2888         /* if VF indicate to PF this function is going down (PF will delete sp
2889          * elements and clear initializations
2890          */
2891         if (IS_VF(bp))
2892                 bnx2x_vfpf_close_vf(bp);
2893         else if (unload_mode != UNLOAD_RECOVERY)
2894                 /* if this is a normal/close unload need to clean up chip*/
2895                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2896         else {
2897                 /* Send the UNLOAD_REQUEST to the MCP */
2898                 bnx2x_send_unload_req(bp, unload_mode);
2899
2900                 /*
2901                  * Prevent transactions to host from the functions on the
2902                  * engine that doesn't reset global blocks in case of global
2903                  * attention once gloabl blocks are reset and gates are opened
2904                  * (the engine which leader will perform the recovery
2905                  * last).
2906                  */
2907                 if (!CHIP_IS_E1x(bp))
2908                         bnx2x_pf_disable(bp);
2909
2910                 /* Disable HW interrupts, NAPI */
2911                 bnx2x_netif_stop(bp, 1);
2912                 /* Delete all NAPI objects */
2913                 bnx2x_del_all_napi(bp);
2914                 if (CNIC_LOADED(bp))
2915                         bnx2x_del_all_napi_cnic(bp);
2916                 /* Release IRQs */
2917                 bnx2x_free_irq(bp);
2918
2919                 /* Report UNLOAD_DONE to MCP */
2920                 bnx2x_send_unload_done(bp, false);
2921         }
2922
2923         /*
2924          * At this stage no more interrupts will arrive so we may safly clean
2925          * the queueable objects here in case they failed to get cleaned so far.
2926          */
2927         if (IS_PF(bp))
2928                 bnx2x_squeeze_objects(bp);
2929
2930         /* There should be no more pending SP commands at this stage */
2931         bp->sp_state = 0;
2932
2933         bp->port.pmf = 0;
2934
2935         /* Free SKBs, SGEs, TPA pool and driver internals */
2936         bnx2x_free_skbs(bp);
2937         if (CNIC_LOADED(bp))
2938                 bnx2x_free_skbs_cnic(bp);
2939         for_each_rx_queue(bp, i)
2940                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2941
2942         bnx2x_free_fp_mem(bp);
2943         if (CNIC_LOADED(bp))
2944                 bnx2x_free_fp_mem_cnic(bp);
2945
2946         if (IS_PF(bp)) {
2947                 if (CNIC_LOADED(bp))
2948                         bnx2x_free_mem_cnic(bp);
2949                 bnx2x_free_mem(bp);
2950         }
2951         bp->state = BNX2X_STATE_CLOSED;
2952         bp->cnic_loaded = false;
2953
2954         /* Check if there are pending parity attentions. If there are - set
2955          * RECOVERY_IN_PROGRESS.
2956          */
2957         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2958                 bnx2x_set_reset_in_progress(bp);
2959
2960                 /* Set RESET_IS_GLOBAL if needed */
2961                 if (global)
2962                         bnx2x_set_reset_global(bp);
2963         }
2964
2965
2966         /* The last driver must disable a "close the gate" if there is no
2967          * parity attention or "process kill" pending.
2968          */
2969         if (IS_PF(bp) &&
2970             !bnx2x_clear_pf_load(bp) &&
2971             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2972                 bnx2x_disable_close_the_gate(bp);
2973
2974         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2975
2976         return 0;
2977 }
2978
2979 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2980 {
2981         u16 pmcsr;
2982
2983         /* If there is no power capability, silently succeed */
2984         if (!bp->pm_cap) {
2985                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2986                 return 0;
2987         }
2988
2989         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2990
2991         switch (state) {
2992         case PCI_D0:
2993                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2994                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2995                                        PCI_PM_CTRL_PME_STATUS));
2996
2997                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2998                         /* delay required during transition out of D3hot */
2999                         msleep(20);
3000                 break;
3001
3002         case PCI_D3hot:
3003                 /* If there are other clients above don't
3004                    shut down the power */
3005                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3006                         return 0;
3007                 /* Don't shut down the power for emulation and FPGA */
3008                 if (CHIP_REV_IS_SLOW(bp))
3009                         return 0;
3010
3011                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3012                 pmcsr |= 3;
3013
3014                 if (bp->wol)
3015                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3016
3017                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3018                                       pmcsr);
3019
3020                 /* No more memory access after this point until
3021                 * device is brought back to D0.
3022                 */
3023                 break;
3024
3025         default:
3026                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3027                 return -EINVAL;
3028         }
3029         return 0;
3030 }
3031
3032 /*
3033  * net_device service functions
3034  */
3035 int bnx2x_poll(struct napi_struct *napi, int budget)
3036 {
3037         int work_done = 0;
3038         u8 cos;
3039         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3040                                                  napi);
3041         struct bnx2x *bp = fp->bp;
3042
3043         while (1) {
3044 #ifdef BNX2X_STOP_ON_ERROR
3045                 if (unlikely(bp->panic)) {
3046                         napi_complete(napi);
3047                         return 0;
3048                 }
3049 #endif
3050
3051                 for_each_cos_in_tx_queue(fp, cos)
3052                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3053                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3054
3055                 if (bnx2x_has_rx_work(fp)) {
3056                         work_done += bnx2x_rx_int(fp, budget - work_done);
3057
3058                         /* must not complete if we consumed full budget */
3059                         if (work_done >= budget)
3060                                 break;
3061                 }
3062
3063                 /* Fall out from the NAPI loop if needed */
3064                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3065
3066                         /* No need to update SB for FCoE L2 ring as long as
3067                          * it's connected to the default SB and the SB
3068                          * has been updated when NAPI was scheduled.
3069                          */
3070                         if (IS_FCOE_FP(fp)) {
3071                                 napi_complete(napi);
3072                                 break;
3073                         }
3074                         bnx2x_update_fpsb_idx(fp);
3075                         /* bnx2x_has_rx_work() reads the status block,
3076                          * thus we need to ensure that status block indices
3077                          * have been actually read (bnx2x_update_fpsb_idx)
3078                          * prior to this check (bnx2x_has_rx_work) so that
3079                          * we won't write the "newer" value of the status block
3080                          * to IGU (if there was a DMA right after
3081                          * bnx2x_has_rx_work and if there is no rmb, the memory
3082                          * reading (bnx2x_update_fpsb_idx) may be postponed
3083                          * to right before bnx2x_ack_sb). In this case there
3084                          * will never be another interrupt until there is
3085                          * another update of the status block, while there
3086                          * is still unhandled work.
3087                          */
3088                         rmb();
3089
3090                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3091                                 napi_complete(napi);
3092                                 /* Re-enable interrupts */
3093                                 DP(NETIF_MSG_RX_STATUS,
3094                                    "Update index to %d\n", fp->fp_hc_idx);
3095                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3096                                              le16_to_cpu(fp->fp_hc_idx),
3097                                              IGU_INT_ENABLE, 1);
3098                                 break;
3099                         }
3100                 }
3101         }
3102
3103         return work_done;
3104 }
3105
3106 /* we split the first BD into headers and data BDs
3107  * to ease the pain of our fellow microcode engineers
3108  * we use one mapping for both BDs
3109  */
3110 static u16 bnx2x_tx_split(struct bnx2x *bp,
3111                           struct bnx2x_fp_txdata *txdata,
3112                           struct sw_tx_bd *tx_buf,
3113                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3114                           u16 bd_prod)
3115 {
3116         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3117         struct eth_tx_bd *d_tx_bd;
3118         dma_addr_t mapping;
3119         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3120
3121         /* first fix first BD */
3122         h_tx_bd->nbytes = cpu_to_le16(hlen);
3123
3124         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3125            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3126
3127         /* now get a new data BD
3128          * (after the pbd) and fill it */
3129         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3130         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3131
3132         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3133                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3134
3135         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3136         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3137         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3138
3139         /* this marks the BD as one that has no individual mapping */
3140         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3141
3142         DP(NETIF_MSG_TX_QUEUED,
3143            "TSO split data size is %d (%x:%x)\n",
3144            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3145
3146         /* update tx_bd */
3147         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3148
3149         return bd_prod;
3150 }
3151
3152 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3153 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3154 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3155 {
3156         __sum16 tsum = (__force __sum16) csum;
3157
3158         if (fix > 0)
3159                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3160                                   csum_partial(t_header - fix, fix, 0)));
3161
3162         else if (fix < 0)
3163                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3164                                   csum_partial(t_header, -fix, 0)));
3165
3166         return bswab16(tsum);
3167 }
3168
3169 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3170 {
3171         u32 rc;
3172         __u8 prot = 0;
3173         __be16 protocol;
3174
3175         if (skb->ip_summed != CHECKSUM_PARTIAL)
3176                 return XMIT_PLAIN;
3177
3178         protocol = vlan_get_protocol(skb);
3179         if (protocol == htons(ETH_P_IPV6)) {
3180                 rc = XMIT_CSUM_V6;
3181                 prot = ipv6_hdr(skb)->nexthdr;
3182         } else {
3183                 rc = XMIT_CSUM_V4;
3184                 prot = ip_hdr(skb)->protocol;
3185         }
3186
3187         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3188                 if (inner_ip_hdr(skb)->version == 6) {
3189                         rc |= XMIT_CSUM_ENC_V6;
3190                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3191                                 rc |= XMIT_CSUM_TCP;
3192                 } else {
3193                         rc |= XMIT_CSUM_ENC_V4;
3194                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3195                                 rc |= XMIT_CSUM_TCP;
3196                 }
3197         }
3198         if (prot == IPPROTO_TCP)
3199                 rc |= XMIT_CSUM_TCP;
3200
3201         if (skb_is_gso_v6(skb)) {
3202                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3203                 if (rc & XMIT_CSUM_ENC)
3204                         rc |= XMIT_GSO_ENC_V6;
3205         } else if (skb_is_gso(skb)) {
3206                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3207                 if (rc & XMIT_CSUM_ENC)
3208                         rc |= XMIT_GSO_ENC_V4;
3209         }
3210
3211         return rc;
3212 }
3213
3214 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3215 /* check if packet requires linearization (packet is too fragmented)
3216    no need to check fragmentation if page size > 8K (there will be no
3217    violation to FW restrictions) */
3218 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3219                              u32 xmit_type)
3220 {
3221         int to_copy = 0;
3222         int hlen = 0;
3223         int first_bd_sz = 0;
3224
3225         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3226         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3227
3228                 if (xmit_type & XMIT_GSO) {
3229                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3230                         /* Check if LSO packet needs to be copied:
3231                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3232                         int wnd_size = MAX_FETCH_BD - 3;
3233                         /* Number of windows to check */
3234                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3235                         int wnd_idx = 0;
3236                         int frag_idx = 0;
3237                         u32 wnd_sum = 0;
3238
3239                         /* Headers length */
3240                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3241                                 tcp_hdrlen(skb);
3242
3243                         /* Amount of data (w/o headers) on linear part of SKB*/
3244                         first_bd_sz = skb_headlen(skb) - hlen;
3245
3246                         wnd_sum  = first_bd_sz;
3247
3248                         /* Calculate the first sum - it's special */
3249                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3250                                 wnd_sum +=
3251                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3252
3253                         /* If there was data on linear skb data - check it */
3254                         if (first_bd_sz > 0) {
3255                                 if (unlikely(wnd_sum < lso_mss)) {
3256                                         to_copy = 1;
3257                                         goto exit_lbl;
3258                                 }
3259
3260                                 wnd_sum -= first_bd_sz;
3261                         }
3262
3263                         /* Others are easier: run through the frag list and
3264                            check all windows */
3265                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3266                                 wnd_sum +=
3267                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3268
3269                                 if (unlikely(wnd_sum < lso_mss)) {
3270                                         to_copy = 1;
3271                                         break;
3272                                 }
3273                                 wnd_sum -=
3274                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3275                         }
3276                 } else {
3277                         /* in non-LSO too fragmented packet should always
3278                            be linearized */
3279                         to_copy = 1;
3280                 }
3281         }
3282
3283 exit_lbl:
3284         if (unlikely(to_copy))
3285                 DP(NETIF_MSG_TX_QUEUED,
3286                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3287                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3288                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3289
3290         return to_copy;
3291 }
3292 #endif
3293
3294 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3295                                  u32 xmit_type)
3296 {
3297         struct ipv6hdr *ipv6;
3298
3299         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3300                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3301                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3302
3303         if (xmit_type & XMIT_GSO_ENC_V6)
3304                 ipv6 = inner_ipv6_hdr(skb);
3305         else if (xmit_type & XMIT_GSO_V6)
3306                 ipv6 = ipv6_hdr(skb);
3307         else
3308                 ipv6 = NULL;
3309
3310         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3311                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3312 }
3313
3314 /**
3315  * bnx2x_set_pbd_gso - update PBD in GSO case.
3316  *
3317  * @skb:        packet skb
3318  * @pbd:        parse BD
3319  * @xmit_type:  xmit flags
3320  */
3321 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3322                               struct eth_tx_parse_bd_e1x *pbd,
3323                               struct eth_tx_start_bd *tx_start_bd,
3324                               u32 xmit_type)
3325 {
3326         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3327         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3328         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3329
3330         if (xmit_type & XMIT_GSO_V4) {
3331                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3332                 pbd->tcp_pseudo_csum =
3333                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3334                                                    ip_hdr(skb)->daddr,
3335                                                    0, IPPROTO_TCP, 0));
3336
3337                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3338                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3339         } else {
3340                 pbd->tcp_pseudo_csum =
3341                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3342                                                  &ipv6_hdr(skb)->daddr,
3343                                                  0, IPPROTO_TCP, 0));
3344         }
3345
3346         pbd->global_data |=
3347                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3348 }
3349
3350 /**
3351  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3352  *
3353  * @bp:                 driver handle
3354  * @skb:                packet skb
3355  * @parsing_data:       data to be updated
3356  * @xmit_type:          xmit flags
3357  *
3358  * 57712/578xx related, when skb has encapsulation
3359  */
3360 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3361                                  u32 *parsing_data, u32 xmit_type)
3362 {
3363         *parsing_data |=
3364                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3365                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3366                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3367
3368         if (xmit_type & XMIT_CSUM_TCP) {
3369                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3370                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3371                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3372
3373                 return skb_inner_transport_header(skb) +
3374                         inner_tcp_hdrlen(skb) - skb->data;
3375         }
3376
3377         /* We support checksum offload for TCP and UDP only.
3378          * No need to pass the UDP header length - it's a constant.
3379          */
3380         return skb_inner_transport_header(skb) +
3381                 sizeof(struct udphdr) - skb->data;
3382 }
3383
3384 /**
3385  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3386  *
3387  * @bp:                 driver handle
3388  * @skb:                packet skb
3389  * @parsing_data:       data to be updated
3390  * @xmit_type:          xmit flags
3391  *
3392  * 57712/578xx related
3393  */
3394 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3395                                 u32 *parsing_data, u32 xmit_type)
3396 {
3397         *parsing_data |=
3398                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3399                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3400                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3401
3402         if (xmit_type & XMIT_CSUM_TCP) {
3403                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3404                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3405                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3406
3407                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3408         }
3409         /* We support checksum offload for TCP and UDP only.
3410          * No need to pass the UDP header length - it's a constant.
3411          */
3412         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3413 }
3414
3415 /* set FW indication according to inner or outer protocols if tunneled */
3416 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3417                                struct eth_tx_start_bd *tx_start_bd,
3418                                u32 xmit_type)
3419 {
3420         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3421
3422         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3423                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3424
3425         if (!(xmit_type & XMIT_CSUM_TCP))
3426                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3427 }
3428
3429 /**
3430  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3431  *
3432  * @bp:         driver handle
3433  * @skb:        packet skb
3434  * @pbd:        parse BD to be updated
3435  * @xmit_type:  xmit flags
3436  */
3437 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3438                              struct eth_tx_parse_bd_e1x *pbd,
3439                              u32 xmit_type)
3440 {
3441         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3442
3443         /* for now NS flag is not used in Linux */
3444         pbd->global_data =
3445                 cpu_to_le16(hlen |
3446                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3447                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3448
3449         pbd->ip_hlen_w = (skb_transport_header(skb) -
3450                         skb_network_header(skb)) >> 1;
3451
3452         hlen += pbd->ip_hlen_w;
3453
3454         /* We support checksum offload for TCP and UDP only */
3455         if (xmit_type & XMIT_CSUM_TCP)
3456                 hlen += tcp_hdrlen(skb) / 2;
3457         else
3458                 hlen += sizeof(struct udphdr) / 2;
3459
3460         pbd->total_hlen_w = cpu_to_le16(hlen);
3461         hlen = hlen*2;
3462
3463         if (xmit_type & XMIT_CSUM_TCP) {
3464                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3465
3466         } else {
3467                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3468
3469                 DP(NETIF_MSG_TX_QUEUED,
3470                    "hlen %d  fix %d  csum before fix %x\n",
3471                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3472
3473                 /* HW bug: fixup the CSUM */
3474                 pbd->tcp_pseudo_csum =
3475                         bnx2x_csum_fix(skb_transport_header(skb),
3476                                        SKB_CS(skb), fix);
3477
3478                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3479                    pbd->tcp_pseudo_csum);
3480         }
3481
3482         return hlen;
3483 }
3484
3485 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3486                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3487                                       struct eth_tx_parse_2nd_bd *pbd2,
3488                                       u16 *global_data,
3489                                       u32 xmit_type)
3490 {
3491         u16 hlen_w = 0;
3492         u8 outerip_off, outerip_len = 0;
3493
3494         /* from outer IP to transport */
3495         hlen_w = (skb_inner_transport_header(skb) -
3496                   skb_network_header(skb)) >> 1;
3497
3498         /* transport len */
3499         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3500
3501         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3502
3503         /* outer IP header info */
3504         if (xmit_type & XMIT_CSUM_V4) {
3505                 struct iphdr *iph = ip_hdr(skb);
3506                 pbd2->fw_ip_csum_wo_len_flags_frag =
3507                         bswab16(csum_fold((~iph->check) -
3508                                           iph->tot_len - iph->frag_off));
3509         } else {
3510                 pbd2->fw_ip_hdr_to_payload_w =
3511                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3512         }
3513
3514         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3515
3516         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3517
3518         if (xmit_type & XMIT_GSO_V4) {
3519                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3520
3521                 pbd_e2->data.tunnel_data.pseudo_csum =
3522                         bswab16(~csum_tcpudp_magic(
3523                                         inner_ip_hdr(skb)->saddr,
3524                                         inner_ip_hdr(skb)->daddr,
3525                                         0, IPPROTO_TCP, 0));
3526
3527                 outerip_len = ip_hdr(skb)->ihl << 1;
3528         } else {
3529                 pbd_e2->data.tunnel_data.pseudo_csum =
3530                         bswab16(~csum_ipv6_magic(
3531                                         &inner_ipv6_hdr(skb)->saddr,
3532                                         &inner_ipv6_hdr(skb)->daddr,
3533                                         0, IPPROTO_TCP, 0));
3534         }
3535
3536         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3537
3538         *global_data |=
3539                 outerip_off |
3540                 (!!(xmit_type & XMIT_CSUM_V6) <<
3541                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3542                 (outerip_len <<
3543                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3544                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3545                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3546
3547         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3548                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3549                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3550         }
3551 }
3552
3553 /* called with netif_tx_lock
3554  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3555  * netif_wake_queue()
3556  */
3557 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3558 {
3559         struct bnx2x *bp = netdev_priv(dev);
3560
3561         struct netdev_queue *txq;
3562         struct bnx2x_fp_txdata *txdata;
3563         struct sw_tx_bd *tx_buf;
3564         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3565         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3566         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3567         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3568         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3569         u32 pbd_e2_parsing_data = 0;
3570         u16 pkt_prod, bd_prod;
3571         int nbd, txq_index;
3572         dma_addr_t mapping;
3573         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3574         int i;
3575         u8 hlen = 0;
3576         __le16 pkt_size = 0;
3577         struct ethhdr *eth;
3578         u8 mac_type = UNICAST_ADDRESS;
3579
3580 #ifdef BNX2X_STOP_ON_ERROR
3581         if (unlikely(bp->panic))
3582                 return NETDEV_TX_BUSY;
3583 #endif
3584
3585         txq_index = skb_get_queue_mapping(skb);
3586         txq = netdev_get_tx_queue(dev, txq_index);
3587
3588         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3589
3590         txdata = &bp->bnx2x_txq[txq_index];
3591
3592         /* enable this debug print to view the transmission queue being used
3593         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3594            txq_index, fp_index, txdata_index); */
3595
3596         /* enable this debug print to view the tranmission details
3597         DP(NETIF_MSG_TX_QUEUED,
3598            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3599            txdata->cid, fp_index, txdata_index, txdata, fp); */
3600
3601         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3602                         skb_shinfo(skb)->nr_frags +
3603                         BDS_PER_TX_PKT +
3604                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3605                 /* Handle special storage cases separately */
3606                 if (txdata->tx_ring_size == 0) {
3607                         struct bnx2x_eth_q_stats *q_stats =
3608                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3609                         q_stats->driver_filtered_tx_pkt++;
3610                         dev_kfree_skb(skb);
3611                         return NETDEV_TX_OK;
3612                 }
3613                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3614                 netif_tx_stop_queue(txq);
3615                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3616
3617                 return NETDEV_TX_BUSY;
3618         }
3619
3620         DP(NETIF_MSG_TX_QUEUED,
3621            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3622            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3623            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3624            skb->len);
3625
3626         eth = (struct ethhdr *)skb->data;
3627
3628         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3629         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3630                 if (is_broadcast_ether_addr(eth->h_dest))
3631                         mac_type = BROADCAST_ADDRESS;
3632                 else
3633                         mac_type = MULTICAST_ADDRESS;
3634         }
3635
3636 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3637         /* First, check if we need to linearize the skb (due to FW
3638            restrictions). No need to check fragmentation if page size > 8K
3639            (there will be no violation to FW restrictions) */
3640         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3641                 /* Statistics of linearization */
3642                 bp->lin_cnt++;
3643                 if (skb_linearize(skb) != 0) {
3644                         DP(NETIF_MSG_TX_QUEUED,
3645                            "SKB linearization failed - silently dropping this SKB\n");
3646                         dev_kfree_skb_any(skb);
3647                         return NETDEV_TX_OK;
3648                 }
3649         }
3650 #endif
3651         /* Map skb linear data for DMA */
3652         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3653                                  skb_headlen(skb), DMA_TO_DEVICE);
3654         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3655                 DP(NETIF_MSG_TX_QUEUED,
3656                    "SKB mapping failed - silently dropping this SKB\n");
3657                 dev_kfree_skb_any(skb);
3658                 return NETDEV_TX_OK;
3659         }
3660         /*
3661         Please read carefully. First we use one BD which we mark as start,
3662         then we have a parsing info BD (used for TSO or xsum),
3663         and only then we have the rest of the TSO BDs.
3664         (don't forget to mark the last one as last,
3665         and to unmap only AFTER you write to the BD ...)
3666         And above all, all pdb sizes are in words - NOT DWORDS!
3667         */
3668
3669         /* get current pkt produced now - advance it just before sending packet
3670          * since mapping of pages may fail and cause packet to be dropped
3671          */
3672         pkt_prod = txdata->tx_pkt_prod;
3673         bd_prod = TX_BD(txdata->tx_bd_prod);
3674
3675         /* get a tx_buf and first BD
3676          * tx_start_bd may be changed during SPLIT,
3677          * but first_bd will always stay first
3678          */
3679         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3680         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3681         first_bd = tx_start_bd;
3682
3683         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3684
3685         /* header nbd: indirectly zero other flags! */
3686         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3687
3688         /* remember the first BD of the packet */
3689         tx_buf->first_bd = txdata->tx_bd_prod;
3690         tx_buf->skb = skb;
3691         tx_buf->flags = 0;
3692
3693         DP(NETIF_MSG_TX_QUEUED,
3694            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3695            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3696
3697         if (vlan_tx_tag_present(skb)) {
3698                 tx_start_bd->vlan_or_ethertype =
3699                     cpu_to_le16(vlan_tx_tag_get(skb));
3700                 tx_start_bd->bd_flags.as_bitfield |=
3701                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3702         } else {
3703                 /* when transmitting in a vf, start bd must hold the ethertype
3704                  * for fw to enforce it
3705                  */
3706                 if (IS_VF(bp))
3707                         tx_start_bd->vlan_or_ethertype =
3708                                 cpu_to_le16(ntohs(eth->h_proto));
3709                 else
3710                         /* used by FW for packet accounting */
3711                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3712         }
3713
3714         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3715
3716         /* turn on parsing and get a BD */
3717         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3718
3719         if (xmit_type & XMIT_CSUM)
3720                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3721
3722         if (!CHIP_IS_E1x(bp)) {
3723                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3724                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3725
3726                 if (xmit_type & XMIT_CSUM_ENC) {
3727                         u16 global_data = 0;
3728
3729                         /* Set PBD in enc checksum offload case */
3730                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3731                                                       &pbd_e2_parsing_data,
3732                                                       xmit_type);
3733
3734                         /* turn on 2nd parsing and get a BD */
3735                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3736
3737                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3738
3739                         memset(pbd2, 0, sizeof(*pbd2));
3740
3741                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3742                                 (skb_inner_network_header(skb) -
3743                                  skb->data) >> 1;
3744
3745                         if (xmit_type & XMIT_GSO_ENC)
3746                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3747                                                           &global_data,
3748                                                           xmit_type);
3749
3750                         pbd2->global_data = cpu_to_le16(global_data);
3751
3752                         /* add addition parse BD indication to start BD */
3753                         SET_FLAG(tx_start_bd->general_data,
3754                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3755                         /* set encapsulation flag in start BD */
3756                         SET_FLAG(tx_start_bd->general_data,
3757                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3758                         nbd++;
3759                 } else if (xmit_type & XMIT_CSUM) {
3760                         /* Set PBD in checksum offload case w/o encapsulation */
3761                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3762                                                      &pbd_e2_parsing_data,
3763                                                      xmit_type);
3764                 }
3765
3766                 /* Add the macs to the parsing BD this is a vf */
3767                 if (IS_VF(bp)) {
3768                         /* override GRE parameters in BD */
3769                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3770                                               &pbd_e2->data.mac_addr.src_mid,
3771                                               &pbd_e2->data.mac_addr.src_lo,
3772                                               eth->h_source);
3773
3774                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3775                                               &pbd_e2->data.mac_addr.dst_mid,
3776                                               &pbd_e2->data.mac_addr.dst_lo,
3777                                               eth->h_dest);
3778                 }
3779
3780                 SET_FLAG(pbd_e2_parsing_data,
3781                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3782         } else {
3783                 u16 global_data = 0;
3784                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3785                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3786                 /* Set PBD in checksum offload case */
3787                 if (xmit_type & XMIT_CSUM)
3788                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3789
3790                 SET_FLAG(global_data,
3791                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3792                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3793         }
3794
3795         /* Setup the data pointer of the first BD of the packet */
3796         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3797         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3798         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3799         pkt_size = tx_start_bd->nbytes;
3800
3801         DP(NETIF_MSG_TX_QUEUED,
3802            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3803            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3804            le16_to_cpu(tx_start_bd->nbytes),
3805            tx_start_bd->bd_flags.as_bitfield,
3806            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3807
3808         if (xmit_type & XMIT_GSO) {
3809
3810                 DP(NETIF_MSG_TX_QUEUED,
3811                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3812                    skb->len, hlen, skb_headlen(skb),
3813                    skb_shinfo(skb)->gso_size);
3814
3815                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3816
3817                 if (unlikely(skb_headlen(skb) > hlen)) {
3818                         nbd++;
3819                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3820                                                  &tx_start_bd, hlen,
3821                                                  bd_prod);
3822                 }
3823                 if (!CHIP_IS_E1x(bp))
3824                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3825                                              xmit_type);
3826                 else
3827                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3828         }
3829
3830         /* Set the PBD's parsing_data field if not zero
3831          * (for the chips newer than 57711).
3832          */
3833         if (pbd_e2_parsing_data)
3834                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3835
3836         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3837
3838         /* Handle fragmented skb */
3839         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3840                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3841
3842                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3843                                            skb_frag_size(frag), DMA_TO_DEVICE);
3844                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3845                         unsigned int pkts_compl = 0, bytes_compl = 0;
3846
3847                         DP(NETIF_MSG_TX_QUEUED,
3848                            "Unable to map page - dropping packet...\n");
3849
3850                         /* we need unmap all buffers already mapped
3851                          * for this SKB;
3852                          * first_bd->nbd need to be properly updated
3853                          * before call to bnx2x_free_tx_pkt
3854                          */
3855                         first_bd->nbd = cpu_to_le16(nbd);
3856                         bnx2x_free_tx_pkt(bp, txdata,
3857                                           TX_BD(txdata->tx_pkt_prod),
3858                                           &pkts_compl, &bytes_compl);
3859                         return NETDEV_TX_OK;
3860                 }
3861
3862                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3863                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3864                 if (total_pkt_bd == NULL)
3865                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3866
3867                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3868                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3869                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3870                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3871                 nbd++;
3872
3873                 DP(NETIF_MSG_TX_QUEUED,
3874                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3875                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3876                    le16_to_cpu(tx_data_bd->nbytes));
3877         }
3878
3879         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3880
3881         /* update with actual num BDs */
3882         first_bd->nbd = cpu_to_le16(nbd);
3883
3884         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3885
3886         /* now send a tx doorbell, counting the next BD
3887          * if the packet contains or ends with it
3888          */
3889         if (TX_BD_POFF(bd_prod) < nbd)
3890                 nbd++;
3891
3892         /* total_pkt_bytes should be set on the first data BD if
3893          * it's not an LSO packet and there is more than one
3894          * data BD. In this case pkt_size is limited by an MTU value.
3895          * However we prefer to set it for an LSO packet (while we don't
3896          * have to) in order to save some CPU cycles in a none-LSO
3897          * case, when we much more care about them.
3898          */
3899         if (total_pkt_bd != NULL)
3900                 total_pkt_bd->total_pkt_bytes = pkt_size;
3901
3902         if (pbd_e1x)
3903                 DP(NETIF_MSG_TX_QUEUED,
3904                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3905                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3906                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3907                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3908                     le16_to_cpu(pbd_e1x->total_hlen_w));
3909         if (pbd_e2)
3910                 DP(NETIF_MSG_TX_QUEUED,
3911                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3912                    pbd_e2,
3913                    pbd_e2->data.mac_addr.dst_hi,
3914                    pbd_e2->data.mac_addr.dst_mid,
3915                    pbd_e2->data.mac_addr.dst_lo,
3916                    pbd_e2->data.mac_addr.src_hi,
3917                    pbd_e2->data.mac_addr.src_mid,
3918                    pbd_e2->data.mac_addr.src_lo,
3919                    pbd_e2->parsing_data);
3920         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3921
3922         netdev_tx_sent_queue(txq, skb->len);
3923
3924         skb_tx_timestamp(skb);
3925
3926         txdata->tx_pkt_prod++;
3927         /*
3928          * Make sure that the BD data is updated before updating the producer
3929          * since FW might read the BD right after the producer is updated.
3930          * This is only applicable for weak-ordered memory model archs such
3931          * as IA-64. The following barrier is also mandatory since FW will
3932          * assumes packets must have BDs.
3933          */
3934         wmb();
3935
3936         txdata->tx_db.data.prod += nbd;
3937         barrier();
3938
3939         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3940
3941         mmiowb();
3942
3943         txdata->tx_bd_prod += nbd;
3944
3945         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3946                 netif_tx_stop_queue(txq);
3947
3948                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3949                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3950                  * fp->bd_tx_cons */
3951                 smp_mb();
3952
3953                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3954                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3955                         netif_tx_wake_queue(txq);
3956         }
3957         txdata->tx_pkt++;
3958
3959         return NETDEV_TX_OK;
3960 }
3961
3962 /**
3963  * bnx2x_setup_tc - routine to configure net_device for multi tc
3964  *
3965  * @netdev: net device to configure
3966  * @tc: number of traffic classes to enable
3967  *
3968  * callback connected to the ndo_setup_tc function pointer
3969  */
3970 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3971 {
3972         int cos, prio, count, offset;
3973         struct bnx2x *bp = netdev_priv(dev);
3974
3975         /* setup tc must be called under rtnl lock */
3976         ASSERT_RTNL();
3977
3978         /* no traffic classes requested. aborting */
3979         if (!num_tc) {
3980                 netdev_reset_tc(dev);
3981                 return 0;
3982         }
3983
3984         /* requested to support too many traffic classes */
3985         if (num_tc > bp->max_cos) {
3986                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3987                           num_tc, bp->max_cos);
3988                 return -EINVAL;
3989         }
3990
3991         /* declare amount of supported traffic classes */
3992         if (netdev_set_num_tc(dev, num_tc)) {
3993                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3994                 return -EINVAL;
3995         }
3996
3997         /* configure priority to traffic class mapping */
3998         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3999                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4000                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4001                    "mapping priority %d to tc %d\n",
4002                    prio, bp->prio_to_cos[prio]);
4003         }
4004
4005
4006         /* Use this configuration to diffrentiate tc0 from other COSes
4007            This can be used for ets or pfc, and save the effort of setting
4008            up a multio class queue disc or negotiating DCBX with a switch
4009         netdev_set_prio_tc_map(dev, 0, 0);
4010         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4011         for (prio = 1; prio < 16; prio++) {
4012                 netdev_set_prio_tc_map(dev, prio, 1);
4013                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4014         } */
4015
4016         /* configure traffic class to transmission queue mapping */
4017         for (cos = 0; cos < bp->max_cos; cos++) {
4018                 count = BNX2X_NUM_ETH_QUEUES(bp);
4019                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4020                 netdev_set_tc_queue(dev, cos, count, offset);
4021                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4022                    "mapping tc %d to offset %d count %d\n",
4023                    cos, offset, count);
4024         }
4025
4026         return 0;
4027 }
4028
4029 /* called with rtnl_lock */
4030 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4031 {
4032         struct sockaddr *addr = p;
4033         struct bnx2x *bp = netdev_priv(dev);
4034         int rc = 0;
4035
4036         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4037                 BNX2X_ERR("Requested MAC address is not valid\n");
4038                 return -EINVAL;
4039         }
4040
4041         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4042             !is_zero_ether_addr(addr->sa_data)) {
4043                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4044                 return -EINVAL;
4045         }
4046
4047         if (netif_running(dev))  {
4048                 rc = bnx2x_set_eth_mac(bp, false);
4049                 if (rc)
4050                         return rc;
4051         }
4052
4053         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4054
4055         if (netif_running(dev))
4056                 rc = bnx2x_set_eth_mac(bp, true);
4057
4058         return rc;
4059 }
4060
4061 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4062 {
4063         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4064         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4065         u8 cos;
4066
4067         /* Common */
4068
4069         if (IS_FCOE_IDX(fp_index)) {
4070                 memset(sb, 0, sizeof(union host_hc_status_block));
4071                 fp->status_blk_mapping = 0;
4072         } else {
4073                 /* status blocks */
4074                 if (!CHIP_IS_E1x(bp))
4075                         BNX2X_PCI_FREE(sb->e2_sb,
4076                                        bnx2x_fp(bp, fp_index,
4077                                                 status_blk_mapping),
4078                                        sizeof(struct host_hc_status_block_e2));
4079                 else
4080                         BNX2X_PCI_FREE(sb->e1x_sb,
4081                                        bnx2x_fp(bp, fp_index,
4082                                                 status_blk_mapping),
4083                                        sizeof(struct host_hc_status_block_e1x));
4084         }
4085
4086         /* Rx */
4087         if (!skip_rx_queue(bp, fp_index)) {
4088                 bnx2x_free_rx_bds(fp);
4089
4090                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4091                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4092                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4093                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4094                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4095
4096                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4097                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4098                                sizeof(struct eth_fast_path_rx_cqe) *
4099                                NUM_RCQ_BD);
4100
4101                 /* SGE ring */
4102                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4103                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4104                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4105                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4106         }
4107
4108         /* Tx */
4109         if (!skip_tx_queue(bp, fp_index)) {
4110                 /* fastpath tx rings: tx_buf tx_desc */
4111                 for_each_cos_in_tx_queue(fp, cos) {
4112                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4113
4114                         DP(NETIF_MSG_IFDOWN,
4115                            "freeing tx memory of fp %d cos %d cid %d\n",
4116                            fp_index, cos, txdata->cid);
4117
4118                         BNX2X_FREE(txdata->tx_buf_ring);
4119                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4120                                 txdata->tx_desc_mapping,
4121                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4122                 }
4123         }
4124         /* end of fastpath */
4125 }
4126
4127 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4128 {
4129         int i;
4130         for_each_cnic_queue(bp, i)
4131                 bnx2x_free_fp_mem_at(bp, i);
4132 }
4133
4134 void bnx2x_free_fp_mem(struct bnx2x *bp)
4135 {
4136         int i;
4137         for_each_eth_queue(bp, i)
4138                 bnx2x_free_fp_mem_at(bp, i);
4139 }
4140
4141 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4142 {
4143         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4144         if (!CHIP_IS_E1x(bp)) {
4145                 bnx2x_fp(bp, index, sb_index_values) =
4146                         (__le16 *)status_blk.e2_sb->sb.index_values;
4147                 bnx2x_fp(bp, index, sb_running_index) =
4148                         (__le16 *)status_blk.e2_sb->sb.running_index;
4149         } else {
4150                 bnx2x_fp(bp, index, sb_index_values) =
4151                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4152                 bnx2x_fp(bp, index, sb_running_index) =
4153                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4154         }
4155 }
4156
4157 /* Returns the number of actually allocated BDs */
4158 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4159                               int rx_ring_size)
4160 {
4161         struct bnx2x *bp = fp->bp;
4162         u16 ring_prod, cqe_ring_prod;
4163         int i, failure_cnt = 0;
4164
4165         fp->rx_comp_cons = 0;
4166         cqe_ring_prod = ring_prod = 0;
4167
4168         /* This routine is called only during fo init so
4169          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4170          */
4171         for (i = 0; i < rx_ring_size; i++) {
4172                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4173                         failure_cnt++;
4174                         continue;
4175                 }
4176                 ring_prod = NEXT_RX_IDX(ring_prod);
4177                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4178                 WARN_ON(ring_prod <= (i - failure_cnt));
4179         }
4180
4181         if (failure_cnt)
4182                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4183                           i - failure_cnt, fp->index);
4184
4185         fp->rx_bd_prod = ring_prod;
4186         /* Limit the CQE producer by the CQE ring size */
4187         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4188                                cqe_ring_prod);
4189         fp->rx_pkt = fp->rx_calls = 0;
4190
4191         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4192
4193         return i - failure_cnt;
4194 }
4195
4196 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4197 {
4198         int i;
4199
4200         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4201                 struct eth_rx_cqe_next_page *nextpg;
4202
4203                 nextpg = (struct eth_rx_cqe_next_page *)
4204                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4205                 nextpg->addr_hi =
4206                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4207                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4208                 nextpg->addr_lo =
4209                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4210                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4211         }
4212 }
4213
4214 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4215 {
4216         union host_hc_status_block *sb;
4217         struct bnx2x_fastpath *fp = &bp->fp[index];
4218         int ring_size = 0;
4219         u8 cos;
4220         int rx_ring_size = 0;
4221
4222         if (!bp->rx_ring_size &&
4223             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4224                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4225                 bp->rx_ring_size = rx_ring_size;
4226         } else if (!bp->rx_ring_size) {
4227                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4228
4229                 if (CHIP_IS_E3(bp)) {
4230                         u32 cfg = SHMEM_RD(bp,
4231                                            dev_info.port_hw_config[BP_PORT(bp)].
4232                                            default_cfg);
4233
4234                         /* Decrease ring size for 1G functions */
4235                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4236                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4237                                 rx_ring_size /= 10;
4238                 }
4239
4240                 /* allocate at least number of buffers required by FW */
4241                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4242                                      MIN_RX_SIZE_TPA, rx_ring_size);
4243
4244                 bp->rx_ring_size = rx_ring_size;
4245         } else /* if rx_ring_size specified - use it */
4246                 rx_ring_size = bp->rx_ring_size;
4247
4248         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4249
4250         /* Common */
4251         sb = &bnx2x_fp(bp, index, status_blk);
4252
4253         if (!IS_FCOE_IDX(index)) {
4254                 /* status blocks */
4255                 if (!CHIP_IS_E1x(bp))
4256                         BNX2X_PCI_ALLOC(sb->e2_sb,
4257                                 &bnx2x_fp(bp, index, status_blk_mapping),
4258                                 sizeof(struct host_hc_status_block_e2));
4259                 else
4260                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4261                                 &bnx2x_fp(bp, index, status_blk_mapping),
4262                             sizeof(struct host_hc_status_block_e1x));
4263         }
4264
4265         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4266          * set shortcuts for it.
4267          */
4268         if (!IS_FCOE_IDX(index))
4269                 set_sb_shortcuts(bp, index);
4270
4271         /* Tx */
4272         if (!skip_tx_queue(bp, index)) {
4273                 /* fastpath tx rings: tx_buf tx_desc */
4274                 for_each_cos_in_tx_queue(fp, cos) {
4275                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4276
4277                         DP(NETIF_MSG_IFUP,
4278                            "allocating tx memory of fp %d cos %d\n",
4279                            index, cos);
4280
4281                         BNX2X_ALLOC(txdata->tx_buf_ring,
4282                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4283                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4284                                 &txdata->tx_desc_mapping,
4285                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4286                 }
4287         }
4288
4289         /* Rx */
4290         if (!skip_rx_queue(bp, index)) {
4291                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4292                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4293                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4294                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4295                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4296                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4297
4298                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4299                                 &bnx2x_fp(bp, index, rx_comp_mapping),
4300                                 sizeof(struct eth_fast_path_rx_cqe) *
4301                                 NUM_RCQ_BD);
4302
4303                 /* SGE ring */
4304                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4305                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4306                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4307                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4308                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4309                 /* RX BD ring */
4310                 bnx2x_set_next_page_rx_bd(fp);
4311
4312                 /* CQ ring */
4313                 bnx2x_set_next_page_rx_cq(fp);
4314
4315                 /* BDs */
4316                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4317                 if (ring_size < rx_ring_size)
4318                         goto alloc_mem_err;
4319         }
4320
4321         return 0;
4322
4323 /* handles low memory cases */
4324 alloc_mem_err:
4325         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4326                                                 index, ring_size);
4327         /* FW will drop all packets if queue is not big enough,
4328          * In these cases we disable the queue
4329          * Min size is different for OOO, TPA and non-TPA queues
4330          */
4331         if (ring_size < (fp->disable_tpa ?
4332                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4333                         /* release memory allocated for this queue */
4334                         bnx2x_free_fp_mem_at(bp, index);
4335                         return -ENOMEM;
4336         }
4337         return 0;
4338 }
4339
4340 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4341 {
4342         if (!NO_FCOE(bp))
4343                 /* FCoE */
4344                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4345                         /* we will fail load process instead of mark
4346                          * NO_FCOE_FLAG
4347                          */
4348                         return -ENOMEM;
4349
4350         return 0;
4351 }
4352
4353 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4354 {
4355         int i;
4356
4357         /* 1. Allocate FP for leading - fatal if error
4358          * 2. Allocate RSS - fix number of queues if error
4359          */
4360
4361         /* leading */
4362         if (bnx2x_alloc_fp_mem_at(bp, 0))
4363                 return -ENOMEM;
4364
4365         /* RSS */
4366         for_each_nondefault_eth_queue(bp, i)
4367                 if (bnx2x_alloc_fp_mem_at(bp, i))
4368                         break;
4369
4370         /* handle memory failures */
4371         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4372                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4373
4374                 WARN_ON(delta < 0);
4375                 bnx2x_shrink_eth_fp(bp, delta);
4376                 if (CNIC_SUPPORT(bp))
4377                         /* move non eth FPs next to last eth FP
4378                          * must be done in that order
4379                          * FCOE_IDX < FWD_IDX < OOO_IDX
4380                          */
4381
4382                         /* move FCoE fp even NO_FCOE_FLAG is on */
4383                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4384                 bp->num_ethernet_queues -= delta;
4385                 bp->num_queues = bp->num_ethernet_queues +
4386                                  bp->num_cnic_queues;
4387                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4388                           bp->num_queues + delta, bp->num_queues);
4389         }
4390
4391         return 0;
4392 }
4393
4394 void bnx2x_free_mem_bp(struct bnx2x *bp)
4395 {
4396         int i;
4397
4398         for (i = 0; i < bp->fp_array_size; i++)
4399                 kfree(bp->fp[i].tpa_info);
4400         kfree(bp->fp);
4401         kfree(bp->sp_objs);
4402         kfree(bp->fp_stats);
4403         kfree(bp->bnx2x_txq);
4404         kfree(bp->msix_table);
4405         kfree(bp->ilt);
4406 }
4407
4408 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4409 {
4410         struct bnx2x_fastpath *fp;
4411         struct msix_entry *tbl;
4412         struct bnx2x_ilt *ilt;
4413         int msix_table_size = 0;
4414         int fp_array_size, txq_array_size;
4415         int i;
4416
4417         /*
4418          * The biggest MSI-X table we might need is as a maximum number of fast
4419          * path IGU SBs plus default SB (for PF only).
4420          */
4421         msix_table_size = bp->igu_sb_cnt;
4422         if (IS_PF(bp))
4423                 msix_table_size++;
4424         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4425
4426         /* fp array: RSS plus CNIC related L2 queues */
4427         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4428         bp->fp_array_size = fp_array_size;
4429         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4430
4431         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4432         if (!fp)
4433                 goto alloc_err;
4434         for (i = 0; i < bp->fp_array_size; i++) {
4435                 fp[i].tpa_info =
4436                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4437                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4438                 if (!(fp[i].tpa_info))
4439                         goto alloc_err;
4440         }
4441
4442         bp->fp = fp;
4443
4444         /* allocate sp objs */
4445         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4446                               GFP_KERNEL);
4447         if (!bp->sp_objs)
4448                 goto alloc_err;
4449
4450         /* allocate fp_stats */
4451         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4452                                GFP_KERNEL);
4453         if (!bp->fp_stats)
4454                 goto alloc_err;
4455
4456         /* Allocate memory for the transmission queues array */
4457         txq_array_size =
4458                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4459         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4460
4461         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4462                                 GFP_KERNEL);
4463         if (!bp->bnx2x_txq)
4464                 goto alloc_err;
4465
4466         /* msix table */
4467         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4468         if (!tbl)
4469                 goto alloc_err;
4470         bp->msix_table = tbl;
4471
4472         /* ilt */
4473         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4474         if (!ilt)
4475                 goto alloc_err;
4476         bp->ilt = ilt;
4477
4478         return 0;
4479 alloc_err:
4480         bnx2x_free_mem_bp(bp);
4481         return -ENOMEM;
4482
4483 }
4484
4485 int bnx2x_reload_if_running(struct net_device *dev)
4486 {
4487         struct bnx2x *bp = netdev_priv(dev);
4488
4489         if (unlikely(!netif_running(dev)))
4490                 return 0;
4491
4492         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4493         return bnx2x_nic_load(bp, LOAD_NORMAL);
4494 }
4495
4496 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4497 {
4498         u32 sel_phy_idx = 0;
4499         if (bp->link_params.num_phys <= 1)
4500                 return INT_PHY;
4501
4502         if (bp->link_vars.link_up) {
4503                 sel_phy_idx = EXT_PHY1;
4504                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4505                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4506                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4507                         sel_phy_idx = EXT_PHY2;
4508         } else {
4509
4510                 switch (bnx2x_phy_selection(&bp->link_params)) {
4511                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4512                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4513                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4514                        sel_phy_idx = EXT_PHY1;
4515                        break;
4516                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4517                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4518                        sel_phy_idx = EXT_PHY2;
4519                        break;
4520                 }
4521         }
4522
4523         return sel_phy_idx;
4524
4525 }
4526 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4527 {
4528         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4529         /*
4530          * The selected activated PHY is always after swapping (in case PHY
4531          * swapping is enabled). So when swapping is enabled, we need to reverse
4532          * the configuration
4533          */
4534
4535         if (bp->link_params.multi_phy_config &
4536             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4537                 if (sel_phy_idx == EXT_PHY1)
4538                         sel_phy_idx = EXT_PHY2;
4539                 else if (sel_phy_idx == EXT_PHY2)
4540                         sel_phy_idx = EXT_PHY1;
4541         }
4542         return LINK_CONFIG_IDX(sel_phy_idx);
4543 }
4544
4545 #ifdef NETDEV_FCOE_WWNN
4546 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4547 {
4548         struct bnx2x *bp = netdev_priv(dev);
4549         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4550
4551         switch (type) {
4552         case NETDEV_FCOE_WWNN:
4553                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4554                                 cp->fcoe_wwn_node_name_lo);
4555                 break;
4556         case NETDEV_FCOE_WWPN:
4557                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4558                                 cp->fcoe_wwn_port_name_lo);
4559                 break;
4560         default:
4561                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4562                 return -EINVAL;
4563         }
4564
4565         return 0;
4566 }
4567 #endif
4568
4569 /* called with rtnl_lock */
4570 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4571 {
4572         struct bnx2x *bp = netdev_priv(dev);
4573
4574         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4575                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4576                 return -EAGAIN;
4577         }
4578
4579         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4580             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4581                 BNX2X_ERR("Can't support requested MTU size\n");
4582                 return -EINVAL;
4583         }
4584
4585         /* This does not race with packet allocation
4586          * because the actual alloc size is
4587          * only updated as part of load
4588          */
4589         dev->mtu = new_mtu;
4590
4591         return bnx2x_reload_if_running(dev);
4592 }
4593
4594 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4595                                      netdev_features_t features)
4596 {
4597         struct bnx2x *bp = netdev_priv(dev);
4598
4599         /* TPA requires Rx CSUM offloading */
4600         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4601                 features &= ~NETIF_F_LRO;
4602                 features &= ~NETIF_F_GRO;
4603         }
4604
4605         return features;
4606 }
4607
4608 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4609 {
4610         struct bnx2x *bp = netdev_priv(dev);
4611         u32 flags = bp->flags;
4612         bool bnx2x_reload = false;
4613
4614         if (features & NETIF_F_LRO)
4615                 flags |= TPA_ENABLE_FLAG;
4616         else
4617                 flags &= ~TPA_ENABLE_FLAG;
4618
4619         if (features & NETIF_F_GRO)
4620                 flags |= GRO_ENABLE_FLAG;
4621         else
4622                 flags &= ~GRO_ENABLE_FLAG;
4623
4624         if (features & NETIF_F_LOOPBACK) {
4625                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4626                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4627                         bnx2x_reload = true;
4628                 }
4629         } else {
4630                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4631                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4632                         bnx2x_reload = true;
4633                 }
4634         }
4635
4636         if (flags ^ bp->flags) {
4637                 bp->flags = flags;
4638                 bnx2x_reload = true;
4639         }
4640
4641         if (bnx2x_reload) {
4642                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4643                         return bnx2x_reload_if_running(dev);
4644                 /* else: bnx2x_nic_load() will be called at end of recovery */
4645         }
4646
4647         return 0;
4648 }
4649
4650 void bnx2x_tx_timeout(struct net_device *dev)
4651 {
4652         struct bnx2x *bp = netdev_priv(dev);
4653
4654 #ifdef BNX2X_STOP_ON_ERROR
4655         if (!bp->panic)
4656                 bnx2x_panic();
4657 #endif
4658
4659         smp_mb__before_clear_bit();
4660         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4661         smp_mb__after_clear_bit();
4662
4663         /* This allows the netif to be shutdown gracefully before resetting */
4664         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4665 }
4666
4667 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4668 {
4669         struct net_device *dev = pci_get_drvdata(pdev);
4670         struct bnx2x *bp;
4671
4672         if (!dev) {
4673                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4674                 return -ENODEV;
4675         }
4676         bp = netdev_priv(dev);
4677
4678         rtnl_lock();
4679
4680         pci_save_state(pdev);
4681
4682         if (!netif_running(dev)) {
4683                 rtnl_unlock();
4684                 return 0;
4685         }
4686
4687         netif_device_detach(dev);
4688
4689         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4690
4691         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4692
4693         rtnl_unlock();
4694
4695         return 0;
4696 }
4697
4698 int bnx2x_resume(struct pci_dev *pdev)
4699 {
4700         struct net_device *dev = pci_get_drvdata(pdev);
4701         struct bnx2x *bp;
4702         int rc;
4703
4704         if (!dev) {
4705                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4706                 return -ENODEV;
4707         }
4708         bp = netdev_priv(dev);
4709
4710         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4711                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4712                 return -EAGAIN;
4713         }
4714
4715         rtnl_lock();
4716
4717         pci_restore_state(pdev);
4718
4719         if (!netif_running(dev)) {
4720                 rtnl_unlock();
4721                 return 0;
4722         }
4723
4724         bnx2x_set_power_state(bp, PCI_D0);
4725         netif_device_attach(dev);
4726
4727         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4728
4729         rtnl_unlock();
4730
4731         return rc;
4732 }
4733
4734
4735 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4736                               u32 cid)
4737 {
4738         /* ustorm cxt validation */
4739         cxt->ustorm_ag_context.cdu_usage =
4740                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4741                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4742         /* xcontext validation */
4743         cxt->xstorm_ag_context.cdu_reserved =
4744                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4745                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4746 }
4747
4748 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4749                                     u8 fw_sb_id, u8 sb_index,
4750                                     u8 ticks)
4751 {
4752
4753         u32 addr = BAR_CSTRORM_INTMEM +
4754                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4755         REG_WR8(bp, addr, ticks);
4756         DP(NETIF_MSG_IFUP,
4757            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4758            port, fw_sb_id, sb_index, ticks);
4759 }
4760
4761 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4762                                     u16 fw_sb_id, u8 sb_index,
4763                                     u8 disable)
4764 {
4765         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4766         u32 addr = BAR_CSTRORM_INTMEM +
4767                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4768         u8 flags = REG_RD8(bp, addr);
4769         /* clear and set */
4770         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4771         flags |= enable_flag;
4772         REG_WR8(bp, addr, flags);
4773         DP(NETIF_MSG_IFUP,
4774            "port %x fw_sb_id %d sb_index %d disable %d\n",
4775            port, fw_sb_id, sb_index, disable);
4776 }
4777
4778 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4779                                     u8 sb_index, u8 disable, u16 usec)
4780 {
4781         int port = BP_PORT(bp);
4782         u8 ticks = usec / BNX2X_BTR;
4783
4784         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4785
4786         disable = disable ? 1 : (usec ? 0 : 1);
4787         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4788 }