drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/busy_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
  34 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
  35 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
  36 static int bnx2x_poll(struct napi_struct *napi, int budget);
  37
  38 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
  39 {
  40         int i;
  41
  42         /* Add NAPI objects */
  43         for_each_rx_queue_cnic(bp, i) {
  44                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  45                                bnx2x_poll, NAPI_POLL_WEIGHT);
  46                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  47         }
  48 }
  49
  50 static void bnx2x_add_all_napi(struct bnx2x *bp)
  51 {
  52         int i;
  53
  54         /* Add NAPI objects */
  55         for_each_eth_queue(bp, i) {
  56                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  57                                bnx2x_poll, NAPI_POLL_WEIGHT);
  58                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  59         }
  60 }
  61
  62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
  63 {
  64         int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
  65
  66         /* Reduce memory usage in kdump environment by using only one queue */
  67         if (reset_devices)
  68                 nq = 1;
  69
  70         nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
  71         return nq;
  72 }
  73
  74 /**
  75  * bnx2x_move_fp - move content of the fastpath structure.
  76  *
  77  * @bp:         driver handle
  78  * @from:       source FP index
  79  * @to:         destination FP index
  80  *
  81  * Makes sure the contents of the bp->fp[to].napi is kept
  82  * intact. This is done by first copying the napi struct from
  83  * the target to the source, and then mem copying the entire
  84  * source onto the target. Update txdata pointers and related
  85  * content.
  86  */
  87 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  88 {
  89         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  90         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  91         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  92         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  93         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  94         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  95         int old_max_eth_txqs, new_max_eth_txqs;
  96         int old_txdata_index = 0, new_txdata_index = 0;
  97         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  98
  99         /* Copy the NAPI object as it has been already initialized */
 100         from_fp->napi = to_fp->napi;
 101
 102         /* Move bnx2x_fastpath contents */
 103         memcpy(to_fp, from_fp, sizeof(*to_fp));
 104         to_fp->index = to;
 105
 106         /* Retain the tpa_info of the original `to' version as we don't want
 107          * 2 FPs to contain the same tpa_info pointer.
 108          */
 109         to_fp->tpa_info = old_tpa_info;
 110
 111         /* move sp_objs contents as well, as their indices match fp ones */
 112         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
 113
 114         /* move fp_stats contents as well, as their indices match fp ones */
 115         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
 116
 117         /* Update txdata pointers in fp and move txdata content accordingly:
 118          * Each fp consumes 'max_cos' txdata structures, so the index should be
 119          * decremented by max_cos x delta.
 120          */
 121
 122         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
 123         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
 124                                 (bp)->max_cos;
 125         if (from == FCOE_IDX(bp)) {
 126                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 127                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 128         }
 129
 130         memcpy(&bp->bnx2x_txq[new_txdata_index],
 131                &bp->bnx2x_txq[old_txdata_index],
 132                sizeof(struct bnx2x_fp_txdata));
 133         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
 134 }
 135
 136 /**
 137  * bnx2x_fill_fw_str - Fill buffer with FW version string.
 138  *
 139  * @bp:        driver handle
 140  * @buf:       character buffer to fill with the fw name
 141  * @buf_len:   length of the above buffer
 142  *
 143  */
 144 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 145 {
 146         if (IS_PF(bp)) {
 147                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 148
 149                 phy_fw_ver[0] = '\0';
 150                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 151                                              phy_fw_ver, PHY_FW_VER_LEN);
 152                 strlcpy(buf, bp->fw_ver, buf_len);
 153                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 154                          "bc %d.%d.%d%s%s",
 155                          (bp->common.bc_ver & 0xff0000) >> 16,
 156                          (bp->common.bc_ver & 0xff00) >> 8,
 157                          (bp->common.bc_ver & 0xff),
 158                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 159         } else {
 160                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 161         }
 162 }
 163
 164 /**
 165  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 166  *
 167  * @bp: driver handle
 168  * @delta:      number of eth queues which were not allocated
 169  */
 170 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 171 {
 172         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 173
 174         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 175          * backward along the array could cause memory to be overridden
 176          */
 177         for (cos = 1; cos < bp->max_cos; cos++) {
 178                 for (i = 0; i < old_eth_num - delta; i++) {
 179                         struct bnx2x_fastpath *fp = &bp->fp[i];
 180                         int new_idx = cos * (old_eth_num - delta) + i;
 181
 182                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 183                                sizeof(struct bnx2x_fp_txdata));
 184                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 185                 }
 186         }
 187 }
 188
 189 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 190
 191 /* free skb in the packet ring at pos idx
 192  * return idx of last bd freed
 193  */
 194 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 195                              u16 idx, unsigned int *pkts_compl,
 196                              unsigned int *bytes_compl)
 197 {
 198         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 199         struct eth_tx_start_bd *tx_start_bd;
 200         struct eth_tx_bd *tx_data_bd;
 201         struct sk_buff *skb = tx_buf->skb;
 202         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 203         int nbd;
 204         u16 split_bd_len = 0;
 205
 206         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 207         prefetch(&skb->end);
 208
 209         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 210            txdata->txq_index, idx, tx_buf, skb);
 211
 212         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 213
 214         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 215 #ifdef BNX2X_STOP_ON_ERROR
 216         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 217                 BNX2X_ERR("BAD nbd!\n");
 218                 bnx2x_panic();
 219         }
 220 #endif
 221         new_cons = nbd + tx_buf->first_bd;
 222
 223         /* Get the next bd */
 224         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 225
 226         /* Skip a parse bd... */
 227         --nbd;
 228         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 229
 230         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 231         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 232                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 233                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 234                 --nbd;
 235                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 236         }
 237
 238         /* unmap first bd */
 239         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 240                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 241                          DMA_TO_DEVICE);
 242
 243         /* now free frags */
 244         while (nbd > 0) {
 245
 246                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 247                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 248                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 249                 if (--nbd)
 250                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 251         }
 252
 253         /* release skb */
 254         WARN_ON(!skb);
 255         if (likely(skb)) {
 256                 (*pkts_compl)++;
 257                 (*bytes_compl) += skb->len;
 258         }
 259
 260         dev_kfree_skb_any(skb);
 261         tx_buf->first_bd = 0;
 262         tx_buf->skb = NULL;
 263
 264         return new_cons;
 265 }
 266
 267 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 268 {
 269         struct netdev_queue *txq;
 270         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 271         unsigned int pkts_compl = 0, bytes_compl = 0;
 272
 273 #ifdef BNX2X_STOP_ON_ERROR
 274         if (unlikely(bp->panic))
 275                 return -1;
 276 #endif
 277
 278         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 279         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 280         sw_cons = txdata->tx_pkt_cons;
 281
 282         while (sw_cons != hw_cons) {
 283                 u16 pkt_cons;
 284
 285                 pkt_cons = TX_BD(sw_cons);
 286
 287                 DP(NETIF_MSG_TX_DONE,
 288                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 289                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 290
 291                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 292                                             &pkts_compl, &bytes_compl);
 293
 294                 sw_cons++;
 295         }
 296
 297         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 298
 299         txdata->tx_pkt_cons = sw_cons;
 300         txdata->tx_bd_cons = bd_cons;
 301
 302         /* Need to make the tx_bd_cons update visible to start_xmit()
 303          * before checking for netif_tx_queue_stopped().  Without the
 304          * memory barrier, there is a small possibility that
 305          * start_xmit() will miss it and cause the queue to be stopped
 306          * forever.
 307          * On the other hand we need an rmb() here to ensure the proper
 308          * ordering of bit testing in the following
 309          * netif_tx_queue_stopped(txq) call.
 310          */
 311         smp_mb();
 312
 313         if (unlikely(netif_tx_queue_stopped(txq))) {
 314                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 315                  * while it's empty. This could have happen if rx_action() gets
 316                  * suspended in bnx2x_tx_int() after the condition before
 317                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 318                  *
 319                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 320                  * sends some packets consuming the whole queue again->
 321                  * stops the queue
 322                  */
 323
 324                 __netif_tx_lock(txq, smp_processor_id());
 325
 326                 if ((netif_tx_queue_stopped(txq)) &&
 327                     (bp->state == BNX2X_STATE_OPEN) &&
 328                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 329                         netif_tx_wake_queue(txq);
 330
 331                 __netif_tx_unlock(txq);
 332         }
 333         return 0;
 334 }
 335
 336 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 337                                              u16 idx)
 338 {
 339         u16 last_max = fp->last_max_sge;
 340
 341         if (SUB_S16(idx, last_max) > 0)
 342                 fp->last_max_sge = idx;
 343 }
 344
 345 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 346                                          u16 sge_len,
 347                                          struct eth_end_agg_rx_cqe *cqe)
 348 {
 349         struct bnx2x *bp = fp->bp;
 350         u16 last_max, last_elem, first_elem;
 351         u16 delta = 0;
 352         u16 i;
 353
 354         if (!sge_len)
 355                 return;
 356
 357         /* First mark all used pages */
 358         for (i = 0; i < sge_len; i++)
 359                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 360                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 361
 362         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 363            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 364
 365         /* Here we assume that the last SGE index is the biggest */
 366         prefetch((void *)(fp->sge_mask));
 367         bnx2x_update_last_max_sge(fp,
 368                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 369
 370         last_max = RX_SGE(fp->last_max_sge);
 371         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 372         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 373
 374         /* If ring is not full */
 375         if (last_elem + 1 != first_elem)
 376                 last_elem++;
 377
 378         /* Now update the prod */
 379         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 380                 if (likely(fp->sge_mask[i]))
 381                         break;
 382
 383                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 384                 delta += BIT_VEC64_ELEM_SZ;
 385         }
 386
 387         if (delta > 0) {
 388                 fp->rx_sge_prod += delta;
 389                 /* clear page-end entries */
 390                 bnx2x_clear_sge_mask_next_elems(fp);
 391         }
 392
 393         DP(NETIF_MSG_RX_STATUS,
 394            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 395            fp->last_max_sge, fp->rx_sge_prod);
 396 }
 397
 398 /* Get Toeplitz hash value in the skb using the value from the
 399  * CQE (calculated by HW).
 400  */
 401 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 402                             const struct eth_fast_path_rx_cqe *cqe,
 403                             enum pkt_hash_types *rxhash_type)
 404 {
 405         /* Get Toeplitz hash from CQE */
 406         if ((bp->dev->features & NETIF_F_RXHASH) &&
 407             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 408                 enum eth_rss_hash_type htype;
 409
 410                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 411                 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
 412                                 (htype == TCP_IPV6_HASH_TYPE)) ?
 413                                PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
 414
 415                 return le32_to_cpu(cqe->rss_hash_result);
 416         }
 417         *rxhash_type = PKT_HASH_TYPE_NONE;
 418         return 0;
 419 }
 420
 421 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 422                             u16 cons, u16 prod,
 423                             struct eth_fast_path_rx_cqe *cqe)
 424 {
 425         struct bnx2x *bp = fp->bp;
 426         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 427         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 428         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 429         dma_addr_t mapping;
 430         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 431         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 432
 433         /* print error if current state != stop */
 434         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 435                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 436
 437         /* Try to map an empty data buffer from the aggregation info  */
 438         mapping = dma_map_single(&bp->pdev->dev,
 439                                  first_buf->data + NET_SKB_PAD,
 440                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 441         /*
 442          *  ...if it fails - move the skb from the consumer to the producer
 443          *  and set the current aggregation state as ERROR to drop it
 444          *  when TPA_STOP arrives.
 445          */
 446
 447         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 448                 /* Move the BD from the consumer to the producer */
 449                 bnx2x_reuse_rx_data(fp, cons, prod);
 450                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 451                 return;
 452         }
 453
 454         /* move empty data from pool to prod */
 455         prod_rx_buf->data = first_buf->data;
 456         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 457         /* point prod_bd to new data */
 458         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 459         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 460
 461         /* move partial skb from cons to pool (don't unmap yet) */
 462         *first_buf = *cons_rx_buf;
 463
 464         /* mark bin state as START */
 465         tpa_info->parsing_flags =
 466                 le16_to_cpu(cqe->pars_flags.flags);
 467         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 468         tpa_info->tpa_state = BNX2X_TPA_START;
 469         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 470         tpa_info->placement_offset = cqe->placement_offset;
 471         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
 472         if (fp->mode == TPA_MODE_GRO) {
 473                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 474                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 475                 tpa_info->gro_size = gro_size;
 476         }
 477
 478 #ifdef BNX2X_STOP_ON_ERROR
 479         fp->tpa_queue_used |= (1 << queue);
 480 #ifdef _ASM_GENERIC_INT_L64_H
 481         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 482 #else
 483         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 484 #endif
 485            fp->tpa_queue_used);
 486 #endif
 487 }
 488
 489 /* Timestamp option length allowed for TPA aggregation:
 490  *
 491  *              nop nop kind length echo val
 492  */
 493 #define TPA_TSTAMP_OPT_LEN      12
 494 /**
 495  * bnx2x_set_gro_params - compute GRO values
 496  *
 497  * @skb:                packet skb
 498  * @parsing_flags:      parsing flags from the START CQE
 499  * @len_on_bd:          total length of the first packet for the
 500  *                      aggregation.
 501  * @pkt_len:            length of all segments
 502  *
 503  * Approximate value of the MSS for this aggregation calculated using
 504  * the first packet of it.
 505  * Compute number of aggregated segments, and gso_type.
 506  */
 507 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 508                                  u16 len_on_bd, unsigned int pkt_len,
 509                                  u16 num_of_coalesced_segs)
 510 {
 511         /* TPA aggregation won't have either IP options or TCP options
 512          * other than timestamp or IPv6 extension headers.
 513          */
 514         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 515
 516         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 517             PRS_FLAG_OVERETH_IPV6) {
 518                 hdrs_len += sizeof(struct ipv6hdr);
 519                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 520         } else {
 521                 hdrs_len += sizeof(struct iphdr);
 522                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 523         }
 524
 525         /* Check if there was a TCP timestamp, if there is it's will
 526          * always be 12 bytes length: nop nop kind length echo val.
 527          *
 528          * Otherwise FW would close the aggregation.
 529          */
 530         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 531                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 532
 533         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 534
 535         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 536          * to skb_shinfo(skb)->gso_segs
 537          */
 538         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 539 }
 540
 541 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 542                               u16 index, gfp_t gfp_mask)
 543 {
 544         struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 545         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 546         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 547         dma_addr_t mapping;
 548
 549         if (unlikely(page == NULL)) {
 550                 BNX2X_ERR("Can't alloc sge\n");
 551                 return -ENOMEM;
 552         }
 553
 554         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 555                                SGE_PAGES, DMA_FROM_DEVICE);
 556         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 557                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 558                 BNX2X_ERR("Can't map sge\n");
 559                 return -ENOMEM;
 560         }
 561
 562         sw_buf->page = page;
 563         dma_unmap_addr_set(sw_buf, mapping, mapping);
 564
 565         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 566         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 567
 568         return 0;
 569 }
 570
 571 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 572                                struct bnx2x_agg_info *tpa_info,
 573                                u16 pages,
 574                                struct sk_buff *skb,
 575                                struct eth_end_agg_rx_cqe *cqe,
 576                                u16 cqe_idx)
 577 {
 578         struct sw_rx_page *rx_pg, old_rx_pg;
 579         u32 i, frag_len, frag_size;
 580         int err, j, frag_id = 0;
 581         u16 len_on_bd = tpa_info->len_on_bd;
 582         u16 full_page = 0, gro_size = 0;
 583
 584         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 585
 586         if (fp->mode == TPA_MODE_GRO) {
 587                 gro_size = tpa_info->gro_size;
 588                 full_page = tpa_info->full_page;
 589         }
 590
 591         /* This is needed in order to enable forwarding support */
 592         if (frag_size)
 593                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 594                                      le16_to_cpu(cqe->pkt_len),
 595                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 596
 597 #ifdef BNX2X_STOP_ON_ERROR
 598         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 599                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 600                           pages, cqe_idx);
 601                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 602                 bnx2x_panic();
 603                 return -EINVAL;
 604         }
 605 #endif
 606
 607         /* Run through the SGL and compose the fragmented skb */
 608         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 609                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 610
 611                 /* FW gives the indices of the SGE as if the ring is an array
 612                    (meaning that "next" element will consume 2 indices) */
 613                 if (fp->mode == TPA_MODE_GRO)
 614                         frag_len = min_t(u32, frag_size, (u32)full_page);
 615                 else /* LRO */
 616                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 617
 618                 rx_pg = &fp->rx_page_ring[sge_idx];
 619                 old_rx_pg = *rx_pg;
 620
 621                 /* If we fail to allocate a substitute page, we simply stop
 622                    where we are and drop the whole packet */
 623                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 624                 if (unlikely(err)) {
 625                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 626                         return err;
 627                 }
 628
 629                 /* Unmap the page as we're going to pass it to the stack */
 630                 dma_unmap_page(&bp->pdev->dev,
 631                                dma_unmap_addr(&old_rx_pg, mapping),
 632                                SGE_PAGES, DMA_FROM_DEVICE);
 633                 /* Add one frag and update the appropriate fields in the skb */
 634                 if (fp->mode == TPA_MODE_LRO)
 635                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 636                 else { /* GRO */
 637                         int rem;
 638                         int offset = 0;
 639                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 640                                 int len = rem > gro_size ? gro_size : rem;
 641                                 skb_fill_page_desc(skb, frag_id++,
 642                                                    old_rx_pg.page, offset, len);
 643                                 if (offset)
 644                                         get_page(old_rx_pg.page);
 645                                 offset += len;
 646                         }
 647                 }
 648
 649                 skb->data_len += frag_len;
 650                 skb->truesize += SGE_PAGES;
 651                 skb->len += frag_len;
 652
 653                 frag_size -= frag_len;
 654         }
 655
 656         return 0;
 657 }
 658
 659 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 660 {
 661         if (fp->rx_frag_size)
 662                 put_page(virt_to_head_page(data));
 663         else
 664                 kfree(data);
 665 }
 666
 667 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 668 {
 669         if (fp->rx_frag_size) {
 670                 /* GFP_KERNEL allocations are used only during initialization */
 671                 if (unlikely(gfp_mask & __GFP_WAIT))
 672                         return (void *)__get_free_page(gfp_mask);
 673
 674                 return netdev_alloc_frag(fp->rx_frag_size);
 675         }
 676
 677         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 678 }
 679
 680 #ifdef CONFIG_INET
 681 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 682 {
 683         const struct iphdr *iph = ip_hdr(skb);
 684         struct tcphdr *th;
 685
 686         skb_set_transport_header(skb, sizeof(struct iphdr));
 687         th = tcp_hdr(skb);
 688
 689         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 690                                   iph->saddr, iph->daddr, 0);
 691 }
 692
 693 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 694 {
 695         struct ipv6hdr *iph = ipv6_hdr(skb);
 696         struct tcphdr *th;
 697
 698         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 699         th = tcp_hdr(skb);
 700
 701         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 702                                   &iph->saddr, &iph->daddr, 0);
 703 }
 704
 705 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 706                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 707 {
 708         skb_set_network_header(skb, 0);
 709         gro_func(bp, skb);
 710         tcp_gro_complete(skb);
 711 }
 712 #endif
 713
 714 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 715                                struct sk_buff *skb)
 716 {
 717 #ifdef CONFIG_INET
 718         if (skb_shinfo(skb)->gso_size) {
 719                 switch (be16_to_cpu(skb->protocol)) {
 720                 case ETH_P_IP:
 721                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 722                         break;
 723                 case ETH_P_IPV6:
 724                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 725                         break;
 726                 default:
 727                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 728                                   be16_to_cpu(skb->protocol));
 729                 }
 730         }
 731 #endif
 732         skb_record_rx_queue(skb, fp->rx_queue);
 733         napi_gro_receive(&fp->napi, skb);
 734 }
 735
 736 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 737                            struct bnx2x_agg_info *tpa_info,
 738                            u16 pages,
 739                            struct eth_end_agg_rx_cqe *cqe,
 740                            u16 cqe_idx)
 741 {
 742         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 743         u8 pad = tpa_info->placement_offset;
 744         u16 len = tpa_info->len_on_bd;
 745         struct sk_buff *skb = NULL;
 746         u8 *new_data, *data = rx_buf->data;
 747         u8 old_tpa_state = tpa_info->tpa_state;
 748
 749         tpa_info->tpa_state = BNX2X_TPA_STOP;
 750
 751         /* If we there was an error during the handling of the TPA_START -
 752          * drop this aggregation.
 753          */
 754         if (old_tpa_state == BNX2X_TPA_ERROR)
 755                 goto drop;
 756
 757         /* Try to allocate the new data */
 758         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 759         /* Unmap skb in the pool anyway, as we are going to change
 760            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 761            fails. */
 762         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 763                          fp->rx_buf_size, DMA_FROM_DEVICE);
 764         if (likely(new_data))
 765                 skb = build_skb(data, fp->rx_frag_size);
 766
 767         if (likely(skb)) {
 768 #ifdef BNX2X_STOP_ON_ERROR
 769                 if (pad + len > fp->rx_buf_size) {
 770                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 771                                   pad, len, fp->rx_buf_size);
 772                         bnx2x_panic();
 773                         return;
 774                 }
 775 #endif
 776
 777                 skb_reserve(skb, pad + NET_SKB_PAD);
 778                 skb_put(skb, len);
 779                 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
 780
 781                 skb->protocol = eth_type_trans(skb, bp->dev);
 782                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 783
 784                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 785                                          skb, cqe, cqe_idx)) {
 786                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 787                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 788                         bnx2x_gro_receive(bp, fp, skb);
 789                 } else {
 790                         DP(NETIF_MSG_RX_STATUS,
 791                            "Failed to allocate new pages - dropping packet!\n");
 792                         dev_kfree_skb_any(skb);
 793                 }
 794
 795                 /* put new data in bin */
 796                 rx_buf->data = new_data;
 797
 798                 return;
 799         }
 800         bnx2x_frag_free(fp, new_data);
 801 drop:
 802         /* drop the packet and keep the buffer in the bin */
 803         DP(NETIF_MSG_RX_STATUS,
 804            "Failed to allocate or map a new skb - dropping packet!\n");
 805         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 806 }
 807
 808 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 809                                u16 index, gfp_t gfp_mask)
 810 {
 811         u8 *data;
 812         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 813         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 814         dma_addr_t mapping;
 815
 816         data = bnx2x_frag_alloc(fp, gfp_mask);
 817         if (unlikely(data == NULL))
 818                 return -ENOMEM;
 819
 820         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 821                                  fp->rx_buf_size,
 822                                  DMA_FROM_DEVICE);
 823         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 824                 bnx2x_frag_free(fp, data);
 825                 BNX2X_ERR("Can't map rx data\n");
 826                 return -ENOMEM;
 827         }
 828
 829         rx_buf->data = data;
 830         dma_unmap_addr_set(rx_buf, mapping, mapping);
 831
 832         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 833         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 834
 835         return 0;
 836 }
 837
 838 static
 839 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 840                                  struct bnx2x_fastpath *fp,
 841                                  struct bnx2x_eth_q_stats *qstats)
 842 {
 843         /* Do nothing if no L4 csum validation was done.
 844          * We do not check whether IP csum was validated. For IPv4 we assume
 845          * that if the card got as far as validating the L4 csum, it also
 846          * validated the IP csum. IPv6 has no IP csum.
 847          */
 848         if (cqe->fast_path_cqe.status_flags &
 849             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 850                 return;
 851
 852         /* If L4 validation was done, check if an error was found. */
 853
 854         if (cqe->fast_path_cqe.type_error_flags &
 855             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 856              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 857                 qstats->hw_csum_err++;
 858         else
 859                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 860 }
 861
 862 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 863 {
 864         struct bnx2x *bp = fp->bp;
 865         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 866         u16 sw_comp_cons, sw_comp_prod;
 867         int rx_pkt = 0;
 868         union eth_rx_cqe *cqe;
 869         struct eth_fast_path_rx_cqe *cqe_fp;
 870
 871 #ifdef BNX2X_STOP_ON_ERROR
 872         if (unlikely(bp->panic))
 873                 return 0;
 874 #endif
 875         if (budget <= 0)
 876                 return rx_pkt;
 877
 878         bd_cons = fp->rx_bd_cons;
 879         bd_prod = fp->rx_bd_prod;
 880         bd_prod_fw = bd_prod;
 881         sw_comp_cons = fp->rx_comp_cons;
 882         sw_comp_prod = fp->rx_comp_prod;
 883
 884         comp_ring_cons = RCQ_BD(sw_comp_cons);
 885         cqe = &fp->rx_comp_ring[comp_ring_cons];
 886         cqe_fp = &cqe->fast_path_cqe;
 887
 888         DP(NETIF_MSG_RX_STATUS,
 889            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 890
 891         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 892                 struct sw_rx_bd *rx_buf = NULL;
 893                 struct sk_buff *skb;
 894                 u8 cqe_fp_flags;
 895                 enum eth_rx_cqe_type cqe_fp_type;
 896                 u16 len, pad, queue;
 897                 u8 *data;
 898                 u32 rxhash;
 899                 enum pkt_hash_types rxhash_type;
 900
 901 #ifdef BNX2X_STOP_ON_ERROR
 902                 if (unlikely(bp->panic))
 903                         return 0;
 904 #endif
 905
 906                 bd_prod = RX_BD(bd_prod);
 907                 bd_cons = RX_BD(bd_cons);
 908
 909                 /* A rmb() is required to ensure that the CQE is not read
 910                  * before it is written by the adapter DMA.  PCI ordering
 911                  * rules will make sure the other fields are written before
 912                  * the marker at the end of struct eth_fast_path_rx_cqe
 913                  * but without rmb() a weakly ordered processor can process
 914                  * stale data.  Without the barrier TPA state-machine might
 915                  * enter inconsistent state and kernel stack might be
 916                  * provided with incorrect packet description - these lead
 917                  * to various kernel crashed.
 918                  */
 919                 rmb();
 920
 921                 cqe_fp_flags = cqe_fp->type_error_flags;
 922                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 923
 924                 DP(NETIF_MSG_RX_STATUS,
 925                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 926                    CQE_TYPE(cqe_fp_flags),
 927                    cqe_fp_flags, cqe_fp->status_flags,
 928                    le32_to_cpu(cqe_fp->rss_hash_result),
 929                    le16_to_cpu(cqe_fp->vlan_tag),
 930                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 931
 932                 /* is this a slowpath msg? */
 933                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 934                         bnx2x_sp_event(fp, cqe);
 935                         goto next_cqe;
 936                 }
 937
 938                 rx_buf = &fp->rx_buf_ring[bd_cons];
 939                 data = rx_buf->data;
 940
 941                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 942                         struct bnx2x_agg_info *tpa_info;
 943                         u16 frag_size, pages;
 944 #ifdef BNX2X_STOP_ON_ERROR
 945                         /* sanity check */
 946                         if (fp->disable_tpa &&
 947                             (CQE_TYPE_START(cqe_fp_type) ||
 948                              CQE_TYPE_STOP(cqe_fp_type)))
 949                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 950                                           CQE_TYPE(cqe_fp_type));
 951 #endif
 952
 953                         if (CQE_TYPE_START(cqe_fp_type)) {
 954                                 u16 queue = cqe_fp->queue_index;
 955                                 DP(NETIF_MSG_RX_STATUS,
 956                                    "calling tpa_start on queue %d\n",
 957                                    queue);
 958
 959                                 bnx2x_tpa_start(fp, queue,
 960                                                 bd_cons, bd_prod,
 961                                                 cqe_fp);
 962
 963                                 goto next_rx;
 964                         }
 965                         queue = cqe->end_agg_cqe.queue_index;
 966                         tpa_info = &fp->tpa_info[queue];
 967                         DP(NETIF_MSG_RX_STATUS,
 968                            "calling tpa_stop on queue %d\n",
 969                            queue);
 970
 971                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 972                                     tpa_info->len_on_bd;
 973
 974                         if (fp->mode == TPA_MODE_GRO)
 975                                 pages = (frag_size + tpa_info->full_page - 1) /
 976                                          tpa_info->full_page;
 977                         else
 978                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 979                                         SGE_PAGE_SHIFT;
 980
 981                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 982                                        &cqe->end_agg_cqe, comp_ring_cons);
 983 #ifdef BNX2X_STOP_ON_ERROR
 984                         if (bp->panic)
 985                                 return 0;
 986 #endif
 987
 988                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 989                         goto next_cqe;
 990                 }
 991                 /* non TPA */
 992                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 993                 pad = cqe_fp->placement_offset;
 994                 dma_sync_single_for_cpu(&bp->pdev->dev,
 995                                         dma_unmap_addr(rx_buf, mapping),
 996                                         pad + RX_COPY_THRESH,
 997                                         DMA_FROM_DEVICE);
 998                 pad += NET_SKB_PAD;
 999                 prefetch(data + pad); /* speedup eth_type_trans() */
1000                 /* is this an error packet? */
1001                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1002                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1003                            "ERROR  flags %x  rx packet %u\n",
1004                            cqe_fp_flags, sw_comp_cons);
1005                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1006                         goto reuse_rx;
1007                 }
1008
1009                 /* Since we don't have a jumbo ring
1010                  * copy small packets if mtu > 1500
1011                  */
1012                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1013                     (len <= RX_COPY_THRESH)) {
1014                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
1015                         if (skb == NULL) {
1016                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1017                                    "ERROR  packet dropped because of alloc failure\n");
1018                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1019                                 goto reuse_rx;
1020                         }
1021                         memcpy(skb->data, data + pad, len);
1022                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1023                 } else {
1024                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1025                                                        GFP_ATOMIC) == 0)) {
1026                                 dma_unmap_single(&bp->pdev->dev,
1027                                                  dma_unmap_addr(rx_buf, mapping),
1028                                                  fp->rx_buf_size,
1029                                                  DMA_FROM_DEVICE);
1030                                 skb = build_skb(data, fp->rx_frag_size);
1031                                 if (unlikely(!skb)) {
1032                                         bnx2x_frag_free(fp, data);
1033                                         bnx2x_fp_qstats(bp, fp)->
1034                                                         rx_skb_alloc_failed++;
1035                                         goto next_rx;
1036                                 }
1037                                 skb_reserve(skb, pad);
1038                         } else {
1039                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1040                                    "ERROR  packet dropped because of alloc failure\n");
1041                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1042 reuse_rx:
1043                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1044                                 goto next_rx;
1045                         }
1046                 }
1047
1048                 skb_put(skb, len);
1049                 skb->protocol = eth_type_trans(skb, bp->dev);
1050
1051                 /* Set Toeplitz hash for a none-LRO skb */
1052                 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1053                 skb_set_hash(skb, rxhash, rxhash_type);
1054
1055                 skb_checksum_none_assert(skb);
1056
1057                 if (bp->dev->features & NETIF_F_RXCSUM)
1058                         bnx2x_csum_validate(skb, cqe, fp,
1059                                             bnx2x_fp_qstats(bp, fp));
1060
1061                 skb_record_rx_queue(skb, fp->rx_queue);
1062
1063                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1064                     PARSING_FLAGS_VLAN)
1065                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1066                                                le16_to_cpu(cqe_fp->vlan_tag));
1067
1068                 skb_mark_napi_id(skb, &fp->napi);
1069
1070                 if (bnx2x_fp_ll_polling(fp))
1071                         netif_receive_skb(skb);
1072                 else
1073                         napi_gro_receive(&fp->napi, skb);
1074 next_rx:
1075                 rx_buf->data = NULL;
1076
1077                 bd_cons = NEXT_RX_IDX(bd_cons);
1078                 bd_prod = NEXT_RX_IDX(bd_prod);
1079                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1080                 rx_pkt++;
1081 next_cqe:
1082                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1083                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1084
1085                 /* mark CQE as free */
1086                 BNX2X_SEED_CQE(cqe_fp);
1087
1088                 if (rx_pkt == budget)
1089                         break;
1090
1091                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1092                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1093                 cqe_fp = &cqe->fast_path_cqe;
1094         } /* while */
1095
1096         fp->rx_bd_cons = bd_cons;
1097         fp->rx_bd_prod = bd_prod_fw;
1098         fp->rx_comp_cons = sw_comp_cons;
1099         fp->rx_comp_prod = sw_comp_prod;
1100
1101         /* Update producers */
1102         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1103                              fp->rx_sge_prod);
1104
1105         fp->rx_pkt += rx_pkt;
1106         fp->rx_calls++;
1107
1108         return rx_pkt;
1109 }
1110
1111 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1112 {
1113         struct bnx2x_fastpath *fp = fp_cookie;
1114         struct bnx2x *bp = fp->bp;
1115         u8 cos;
1116
1117         DP(NETIF_MSG_INTR,
1118            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1119            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1120
1121         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1122
1123 #ifdef BNX2X_STOP_ON_ERROR
1124         if (unlikely(bp->panic))
1125                 return IRQ_HANDLED;
1126 #endif
1127
1128         /* Handle Rx and Tx according to MSI-X vector */
1129         for_each_cos_in_tx_queue(fp, cos)
1130                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1131
1132         prefetch(&fp->sb_running_index[SM_RX_ID]);
1133         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1134
1135         return IRQ_HANDLED;
1136 }
1137
1138 /* HW Lock for shared dual port PHYs */
1139 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1140 {
1141         mutex_lock(&bp->port.phy_mutex);
1142
1143         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1144 }
1145
1146 void bnx2x_release_phy_lock(struct bnx2x *bp)
1147 {
1148         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1149
1150         mutex_unlock(&bp->port.phy_mutex);
1151 }
1152
1153 /* calculates MF speed according to current linespeed and MF configuration */
1154 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1155 {
1156         u16 line_speed = bp->link_vars.line_speed;
1157         if (IS_MF(bp)) {
1158                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1159                                                    bp->mf_config[BP_VN(bp)]);
1160
1161                 /* Calculate the current MAX line speed limit for the MF
1162                  * devices
1163                  */
1164                 if (IS_MF_SI(bp))
1165                         line_speed = (line_speed * maxCfg) / 100;
1166                 else { /* SD mode */
1167                         u16 vn_max_rate = maxCfg * 100;
1168
1169                         if (vn_max_rate < line_speed)
1170                                 line_speed = vn_max_rate;
1171                 }
1172         }
1173
1174         return line_speed;
1175 }
1176
1177 /**
1178  * bnx2x_fill_report_data - fill link report data to report
1179  *
1180  * @bp:         driver handle
1181  * @data:       link state to update
1182  *
1183  * It uses a none-atomic bit operations because is called under the mutex.
1184  */
1185 static void bnx2x_fill_report_data(struct bnx2x *bp,
1186                                    struct bnx2x_link_report_data *data)
1187 {
1188         u16 line_speed = bnx2x_get_mf_speed(bp);
1189
1190         memset(data, 0, sizeof(*data));
1191
1192         /* Fill the report data: effective line speed */
1193         data->line_speed = line_speed;
1194
1195         /* Link is down */
1196         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1197                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1198                           &data->link_report_flags);
1199
1200         /* Full DUPLEX */
1201         if (bp->link_vars.duplex == DUPLEX_FULL)
1202                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1203
1204         /* Rx Flow Control is ON */
1205         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1206                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1207
1208         /* Tx Flow Control is ON */
1209         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1210                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1211 }
1212
1213 /**
1214  * bnx2x_link_report - report link status to OS.
1215  *
1216  * @bp:         driver handle
1217  *
1218  * Calls the __bnx2x_link_report() under the same locking scheme
1219  * as a link/PHY state managing code to ensure a consistent link
1220  * reporting.
1221  */
1222
1223 void bnx2x_link_report(struct bnx2x *bp)
1224 {
1225         bnx2x_acquire_phy_lock(bp);
1226         __bnx2x_link_report(bp);
1227         bnx2x_release_phy_lock(bp);
1228 }
1229
1230 /**
1231  * __bnx2x_link_report - report link status to OS.
1232  *
1233  * @bp:         driver handle
1234  *
1235  * None atomic implementation.
1236  * Should be called under the phy_lock.
1237  */
1238 void __bnx2x_link_report(struct bnx2x *bp)
1239 {
1240         struct bnx2x_link_report_data cur_data;
1241
1242         /* reread mf_cfg */
1243         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1244                 bnx2x_read_mf_cfg(bp);
1245
1246         /* Read the current link report info */
1247         bnx2x_fill_report_data(bp, &cur_data);
1248
1249         /* Don't report link down or exactly the same link status twice */
1250         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1251             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1252                       &bp->last_reported_link.link_report_flags) &&
1253              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1254                       &cur_data.link_report_flags)))
1255                 return;
1256
1257         bp->link_cnt++;
1258
1259         /* We are going to report a new link parameters now -
1260          * remember the current data for the next time.
1261          */
1262         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1263
1264         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1265                      &cur_data.link_report_flags)) {
1266                 netif_carrier_off(bp->dev);
1267                 netdev_err(bp->dev, "NIC Link is Down\n");
1268                 return;
1269         } else {
1270                 const char *duplex;
1271                 const char *flow;
1272
1273                 netif_carrier_on(bp->dev);
1274
1275                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1276                                        &cur_data.link_report_flags))
1277                         duplex = "full";
1278                 else
1279                         duplex = "half";
1280
1281                 /* Handle the FC at the end so that only these flags would be
1282                  * possibly set. This way we may easily check if there is no FC
1283                  * enabled.
1284                  */
1285                 if (cur_data.link_report_flags) {
1286                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1287                                      &cur_data.link_report_flags)) {
1288                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1289                                      &cur_data.link_report_flags))
1290                                         flow = "ON - receive & transmit";
1291                                 else
1292                                         flow = "ON - receive";
1293                         } else {
1294                                 flow = "ON - transmit";
1295                         }
1296                 } else {
1297                         flow = "none";
1298                 }
1299                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1300                             cur_data.line_speed, duplex, flow);
1301         }
1302 }
1303
1304 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1305 {
1306         int i;
1307
1308         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1309                 struct eth_rx_sge *sge;
1310
1311                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1312                 sge->addr_hi =
1313                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1314                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1315
1316                 sge->addr_lo =
1317                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1318                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1319         }
1320 }
1321
1322 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1323                                 struct bnx2x_fastpath *fp, int last)
1324 {
1325         int i;
1326
1327         for (i = 0; i < last; i++) {
1328                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1329                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1330                 u8 *data = first_buf->data;
1331
1332                 if (data == NULL) {
1333                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1334                         continue;
1335                 }
1336                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1337                         dma_unmap_single(&bp->pdev->dev,
1338                                          dma_unmap_addr(first_buf, mapping),
1339                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1340                 bnx2x_frag_free(fp, data);
1341                 first_buf->data = NULL;
1342         }
1343 }
1344
1345 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1346 {
1347         int j;
1348
1349         for_each_rx_queue_cnic(bp, j) {
1350                 struct bnx2x_fastpath *fp = &bp->fp[j];
1351
1352                 fp->rx_bd_cons = 0;
1353
1354                 /* Activate BD ring */
1355                 /* Warning!
1356                  * this will generate an interrupt (to the TSTORM)
1357                  * must only be done after chip is initialized
1358                  */
1359                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1360                                      fp->rx_sge_prod);
1361         }
1362 }
1363
1364 void bnx2x_init_rx_rings(struct bnx2x *bp)
1365 {
1366         int func = BP_FUNC(bp);
1367         u16 ring_prod;
1368         int i, j;
1369
1370         /* Allocate TPA resources */
1371         for_each_eth_queue(bp, j) {
1372                 struct bnx2x_fastpath *fp = &bp->fp[j];
1373
1374                 DP(NETIF_MSG_IFUP,
1375                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1376
1377                 if (!fp->disable_tpa) {
1378                         /* Fill the per-aggregation pool */
1379                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1380                                 struct bnx2x_agg_info *tpa_info =
1381                                         &fp->tpa_info[i];
1382                                 struct sw_rx_bd *first_buf =
1383                                         &tpa_info->first_buf;
1384
1385                                 first_buf->data =
1386                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1387                                 if (!first_buf->data) {
1388                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1389                                                   j);
1390                                         bnx2x_free_tpa_pool(bp, fp, i);
1391                                         fp->disable_tpa = 1;
1392                                         break;
1393                                 }
1394                                 dma_unmap_addr_set(first_buf, mapping, 0);
1395                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1396                         }
1397
1398                         /* "next page" elements initialization */
1399                         bnx2x_set_next_page_sgl(fp);
1400
1401                         /* set SGEs bit mask */
1402                         bnx2x_init_sge_ring_bit_mask(fp);
1403
1404                         /* Allocate SGEs and initialize the ring elements */
1405                         for (i = 0, ring_prod = 0;
1406                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1407
1408                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1409                                                        GFP_KERNEL) < 0) {
1410                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1411                                                   i);
1412                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1413                                                   j);
1414                                         /* Cleanup already allocated elements */
1415                                         bnx2x_free_rx_sge_range(bp, fp,
1416                                                                 ring_prod);
1417                                         bnx2x_free_tpa_pool(bp, fp,
1418                                                             MAX_AGG_QS(bp));
1419                                         fp->disable_tpa = 1;
1420                                         ring_prod = 0;
1421                                         break;
1422                                 }
1423                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1424                         }
1425
1426                         fp->rx_sge_prod = ring_prod;
1427                 }
1428         }
1429
1430         for_each_eth_queue(bp, j) {
1431                 struct bnx2x_fastpath *fp = &bp->fp[j];
1432
1433                 fp->rx_bd_cons = 0;
1434
1435                 /* Activate BD ring */
1436                 /* Warning!
1437                  * this will generate an interrupt (to the TSTORM)
1438                  * must only be done after chip is initialized
1439                  */
1440                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1441                                      fp->rx_sge_prod);
1442
1443                 if (j != 0)
1444                         continue;
1445
1446                 if (CHIP_IS_E1(bp)) {
1447                         REG_WR(bp, BAR_USTRORM_INTMEM +
1448                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1449                                U64_LO(fp->rx_comp_mapping));
1450                         REG_WR(bp, BAR_USTRORM_INTMEM +
1451                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1452                                U64_HI(fp->rx_comp_mapping));
1453                 }
1454         }
1455 }
1456
1457 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1458 {
1459         u8 cos;
1460         struct bnx2x *bp = fp->bp;
1461
1462         for_each_cos_in_tx_queue(fp, cos) {
1463                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1464                 unsigned pkts_compl = 0, bytes_compl = 0;
1465
1466                 u16 sw_prod = txdata->tx_pkt_prod;
1467                 u16 sw_cons = txdata->tx_pkt_cons;
1468
1469                 while (sw_cons != sw_prod) {
1470                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1471                                           &pkts_compl, &bytes_compl);
1472                         sw_cons++;
1473                 }
1474
1475                 netdev_tx_reset_queue(
1476                         netdev_get_tx_queue(bp->dev,
1477                                             txdata->txq_index));
1478         }
1479 }
1480
1481 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1482 {
1483         int i;
1484
1485         for_each_tx_queue_cnic(bp, i) {
1486                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1487         }
1488 }
1489
1490 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1491 {
1492         int i;
1493
1494         for_each_eth_queue(bp, i) {
1495                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1496         }
1497 }
1498
1499 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1500 {
1501         struct bnx2x *bp = fp->bp;
1502         int i;
1503
1504         /* ring wasn't allocated */
1505         if (fp->rx_buf_ring == NULL)
1506                 return;
1507
1508         for (i = 0; i < NUM_RX_BD; i++) {
1509                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1510                 u8 *data = rx_buf->data;
1511
1512                 if (data == NULL)
1513                         continue;
1514                 dma_unmap_single(&bp->pdev->dev,
1515                                  dma_unmap_addr(rx_buf, mapping),
1516                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1517
1518                 rx_buf->data = NULL;
1519                 bnx2x_frag_free(fp, data);
1520         }
1521 }
1522
1523 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1524 {
1525         int j;
1526
1527         for_each_rx_queue_cnic(bp, j) {
1528                 bnx2x_free_rx_bds(&bp->fp[j]);
1529         }
1530 }
1531
1532 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1533 {
1534         int j;
1535
1536         for_each_eth_queue(bp, j) {
1537                 struct bnx2x_fastpath *fp = &bp->fp[j];
1538
1539                 bnx2x_free_rx_bds(fp);
1540
1541                 if (!fp->disable_tpa)
1542                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1543         }
1544 }
1545
1546 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1547 {
1548         bnx2x_free_tx_skbs_cnic(bp);
1549         bnx2x_free_rx_skbs_cnic(bp);
1550 }
1551
1552 void bnx2x_free_skbs(struct bnx2x *bp)
1553 {
1554         bnx2x_free_tx_skbs(bp);
1555         bnx2x_free_rx_skbs(bp);
1556 }
1557
1558 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1559 {
1560         /* load old values */
1561         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1562
1563         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1564                 /* leave all but MAX value */
1565                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1566
1567                 /* set new MAX value */
1568                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1569                                 & FUNC_MF_CFG_MAX_BW_MASK;
1570
1571                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1572         }
1573 }
1574
1575 /**
1576  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1577  *
1578  * @bp:         driver handle
1579  * @nvecs:      number of vectors to be released
1580  */
1581 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1582 {
1583         int i, offset = 0;
1584
1585         if (nvecs == offset)
1586                 return;
1587
1588         /* VFs don't have a default SB */
1589         if (IS_PF(bp)) {
1590                 free_irq(bp->msix_table[offset].vector, bp->dev);
1591                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1592                    bp->msix_table[offset].vector);
1593                 offset++;
1594         }
1595
1596         if (CNIC_SUPPORT(bp)) {
1597                 if (nvecs == offset)
1598                         return;
1599                 offset++;
1600         }
1601
1602         for_each_eth_queue(bp, i) {
1603                 if (nvecs == offset)
1604                         return;
1605                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1606                    i, bp->msix_table[offset].vector);
1607
1608                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1609         }
1610 }
1611
1612 void bnx2x_free_irq(struct bnx2x *bp)
1613 {
1614         if (bp->flags & USING_MSIX_FLAG &&
1615             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1616                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1617
1618                 /* vfs don't have a default status block */
1619                 if (IS_PF(bp))
1620                         nvecs++;
1621
1622                 bnx2x_free_msix_irqs(bp, nvecs);
1623         } else {
1624                 free_irq(bp->dev->irq, bp->dev);
1625         }
1626 }
1627
1628 int bnx2x_enable_msix(struct bnx2x *bp)
1629 {
1630         int msix_vec = 0, i, rc;
1631
1632         /* VFs don't have a default status block */
1633         if (IS_PF(bp)) {
1634                 bp->msix_table[msix_vec].entry = msix_vec;
1635                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1636                                bp->msix_table[0].entry);
1637                 msix_vec++;
1638         }
1639
1640         /* Cnic requires an msix vector for itself */
1641         if (CNIC_SUPPORT(bp)) {
1642                 bp->msix_table[msix_vec].entry = msix_vec;
1643                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1644                                msix_vec, bp->msix_table[msix_vec].entry);
1645                 msix_vec++;
1646         }
1647
1648         /* We need separate vectors for ETH queues only (not FCoE) */
1649         for_each_eth_queue(bp, i) {
1650                 bp->msix_table[msix_vec].entry = msix_vec;
1651                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1652                                msix_vec, msix_vec, i);
1653                 msix_vec++;
1654         }
1655
1656         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1657            msix_vec);
1658
1659         rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1660                                    BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1661         /*
1662          * reconfigure number of tx/rx queues according to available
1663          * MSI-X vectors
1664          */
1665         if (rc == -ENOSPC) {
1666                 /* Get by with single vector */
1667                 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1668                 if (rc < 0) {
1669                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1670                                        rc);
1671                         goto no_msix;
1672                 }
1673
1674                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1675                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1676
1677                 BNX2X_DEV_INFO("set number of queues to 1\n");
1678                 bp->num_ethernet_queues = 1;
1679                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1680         } else if (rc < 0) {
1681                 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1682                 goto no_msix;
1683         } else if (rc < msix_vec) {
1684                 /* how less vectors we will have? */
1685                 int diff = msix_vec - rc;
1686
1687                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1688
1689                 /*
1690                  * decrease number of queues by number of unallocated entries
1691                  */
1692                 bp->num_ethernet_queues -= diff;
1693                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1694
1695                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1696                                bp->num_queues);
1697         }
1698
1699         bp->flags |= USING_MSIX_FLAG;
1700
1701         return 0;
1702
1703 no_msix:
1704         /* fall to INTx if not enough memory */
1705         if (rc == -ENOMEM)
1706                 bp->flags |= DISABLE_MSI_FLAG;
1707
1708         return rc;
1709 }
1710
1711 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1712 {
1713         int i, rc, offset = 0;
1714
1715         /* no default status block for vf */
1716         if (IS_PF(bp)) {
1717                 rc = request_irq(bp->msix_table[offset++].vector,
1718                                  bnx2x_msix_sp_int, 0,
1719                                  bp->dev->name, bp->dev);
1720                 if (rc) {
1721                         BNX2X_ERR("request sp irq failed\n");
1722                         return -EBUSY;
1723                 }
1724         }
1725
1726         if (CNIC_SUPPORT(bp))
1727                 offset++;
1728
1729         for_each_eth_queue(bp, i) {
1730                 struct bnx2x_fastpath *fp = &bp->fp[i];
1731                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1732                          bp->dev->name, i);
1733
1734                 rc = request_irq(bp->msix_table[offset].vector,
1735                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1736                 if (rc) {
1737                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1738                               bp->msix_table[offset].vector, rc);
1739                         bnx2x_free_msix_irqs(bp, offset);
1740                         return -EBUSY;
1741                 }
1742
1743                 offset++;
1744         }
1745
1746         i = BNX2X_NUM_ETH_QUEUES(bp);
1747         if (IS_PF(bp)) {
1748                 offset = 1 + CNIC_SUPPORT(bp);
1749                 netdev_info(bp->dev,
1750                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1751                             bp->msix_table[0].vector,
1752                             0, bp->msix_table[offset].vector,
1753                             i - 1, bp->msix_table[offset + i - 1].vector);
1754         } else {
1755                 offset = CNIC_SUPPORT(bp);
1756                 netdev_info(bp->dev,
1757                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1758                             0, bp->msix_table[offset].vector,
1759                             i - 1, bp->msix_table[offset + i - 1].vector);
1760         }
1761         return 0;
1762 }
1763
1764 int bnx2x_enable_msi(struct bnx2x *bp)
1765 {
1766         int rc;
1767
1768         rc = pci_enable_msi(bp->pdev);
1769         if (rc) {
1770                 BNX2X_DEV_INFO("MSI is not attainable\n");
1771                 return -1;
1772         }
1773         bp->flags |= USING_MSI_FLAG;
1774
1775         return 0;
1776 }
1777
1778 static int bnx2x_req_irq(struct bnx2x *bp)
1779 {
1780         unsigned long flags;
1781         unsigned int irq;
1782
1783         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1784                 flags = 0;
1785         else
1786                 flags = IRQF_SHARED;
1787
1788         if (bp->flags & USING_MSIX_FLAG)
1789                 irq = bp->msix_table[0].vector;
1790         else
1791                 irq = bp->pdev->irq;
1792
1793         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1794 }
1795
1796 static int bnx2x_setup_irqs(struct bnx2x *bp)
1797 {
1798         int rc = 0;
1799         if (bp->flags & USING_MSIX_FLAG &&
1800             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1801                 rc = bnx2x_req_msix_irqs(bp);
1802                 if (rc)
1803                         return rc;
1804         } else {
1805                 rc = bnx2x_req_irq(bp);
1806                 if (rc) {
1807                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1808                         return rc;
1809                 }
1810                 if (bp->flags & USING_MSI_FLAG) {
1811                         bp->dev->irq = bp->pdev->irq;
1812                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1813                                     bp->dev->irq);
1814                 }
1815                 if (bp->flags & USING_MSIX_FLAG) {
1816                         bp->dev->irq = bp->msix_table[0].vector;
1817                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1818                                     bp->dev->irq);
1819                 }
1820         }
1821
1822         return 0;
1823 }
1824
1825 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1826 {
1827         int i;
1828
1829         for_each_rx_queue_cnic(bp, i) {
1830                 bnx2x_fp_init_lock(&bp->fp[i]);
1831                 napi_enable(&bnx2x_fp(bp, i, napi));
1832         }
1833 }
1834
1835 static void bnx2x_napi_enable(struct bnx2x *bp)
1836 {
1837         int i;
1838
1839         for_each_eth_queue(bp, i) {
1840                 bnx2x_fp_init_lock(&bp->fp[i]);
1841                 napi_enable(&bnx2x_fp(bp, i, napi));
1842         }
1843 }
1844
1845 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1846 {
1847         int i;
1848
1849         for_each_rx_queue_cnic(bp, i) {
1850                 napi_disable(&bnx2x_fp(bp, i, napi));
1851                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1852                         usleep_range(1000, 2000);
1853         }
1854 }
1855
1856 static void bnx2x_napi_disable(struct bnx2x *bp)
1857 {
1858         int i;
1859
1860         for_each_eth_queue(bp, i) {
1861                 napi_disable(&bnx2x_fp(bp, i, napi));
1862                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1863                         usleep_range(1000, 2000);
1864         }
1865 }
1866
1867 void bnx2x_netif_start(struct bnx2x *bp)
1868 {
1869         if (netif_running(bp->dev)) {
1870                 bnx2x_napi_enable(bp);
1871                 if (CNIC_LOADED(bp))
1872                         bnx2x_napi_enable_cnic(bp);
1873                 bnx2x_int_enable(bp);
1874                 if (bp->state == BNX2X_STATE_OPEN)
1875                         netif_tx_wake_all_queues(bp->dev);
1876         }
1877 }
1878
1879 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1880 {
1881         bnx2x_int_disable_sync(bp, disable_hw);
1882         bnx2x_napi_disable(bp);
1883         if (CNIC_LOADED(bp))
1884                 bnx2x_napi_disable_cnic(bp);
1885 }
1886
1887 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1888                        void *accel_priv, select_queue_fallback_t fallback)
1889 {
1890         struct bnx2x *bp = netdev_priv(dev);
1891
1892         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1893                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1894                 u16 ether_type = ntohs(hdr->h_proto);
1895
1896                 /* Skip VLAN tag if present */
1897                 if (ether_type == ETH_P_8021Q) {
1898                         struct vlan_ethhdr *vhdr =
1899                                 (struct vlan_ethhdr *)skb->data;
1900
1901                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1902                 }
1903
1904                 /* If ethertype is FCoE or FIP - use FCoE ring */
1905                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1906                         return bnx2x_fcoe_tx(bp, txq_index);
1907         }
1908
1909         /* select a non-FCoE queue */
1910         return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1911 }
1912
1913 void bnx2x_set_num_queues(struct bnx2x *bp)
1914 {
1915         /* RSS queues */
1916         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1917
1918         /* override in STORAGE SD modes */
1919         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1920                 bp->num_ethernet_queues = 1;
1921
1922         /* Add special queues */
1923         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1924         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1925
1926         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1927 }
1928
1929 /**
1930  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1931  *
1932  * @bp:         Driver handle
1933  *
1934  * We currently support for at most 16 Tx queues for each CoS thus we will
1935  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1936  * bp->max_cos.
1937  *
1938  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1939  * index after all ETH L2 indices.
1940  *
1941  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1942  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1943  * 16..31,...) with indices that are not coupled with any real Tx queue.
1944  *
1945  * The proper configuration of skb->queue_mapping is handled by
1946  * bnx2x_select_queue() and __skb_tx_hash().
1947  *
1948  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1949  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1950  */
1951 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1952 {
1953         int rc, tx, rx;
1954
1955         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1956         rx = BNX2X_NUM_ETH_QUEUES(bp);
1957
1958 /* account for fcoe queue */
1959         if (include_cnic && !NO_FCOE(bp)) {
1960                 rx++;
1961                 tx++;
1962         }
1963
1964         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1965         if (rc) {
1966                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1967                 return rc;
1968         }
1969         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1970         if (rc) {
1971                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1972                 return rc;
1973         }
1974
1975         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1976                           tx, rx);
1977
1978         return rc;
1979 }
1980
1981 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1982 {
1983         int i;
1984
1985         for_each_queue(bp, i) {
1986                 struct bnx2x_fastpath *fp = &bp->fp[i];
1987                 u32 mtu;
1988
1989                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1990                 if (IS_FCOE_IDX(i))
1991                         /*
1992                          * Although there are no IP frames expected to arrive to
1993                          * this ring we still want to add an
1994                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1995                          * overrun attack.
1996                          */
1997                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1998                 else
1999                         mtu = bp->dev->mtu;
2000                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2001                                   IP_HEADER_ALIGNMENT_PADDING +
2002                                   ETH_OVREHEAD +
2003                                   mtu +
2004                                   BNX2X_FW_RX_ALIGN_END;
2005                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2006                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2007                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2008                 else
2009                         fp->rx_frag_size = 0;
2010         }
2011 }
2012
2013 static int bnx2x_init_rss(struct bnx2x *bp)
2014 {
2015         int i;
2016         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2017
2018         /* Prepare the initial contents for the indirection table if RSS is
2019          * enabled
2020          */
2021         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2022                 bp->rss_conf_obj.ind_table[i] =
2023                         bp->fp->cl_id +
2024                         ethtool_rxfh_indir_default(i, num_eth_queues);
2025
2026         /*
2027          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2028          * per-port, so if explicit configuration is needed , do it only
2029          * for a PMF.
2030          *
2031          * For 57712 and newer on the other hand it's a per-function
2032          * configuration.
2033          */
2034         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2035 }
2036
2037 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2038               bool config_hash, bool enable)
2039 {
2040         struct bnx2x_config_rss_params params = {NULL};
2041
2042         /* Although RSS is meaningless when there is a single HW queue we
2043          * still need it enabled in order to have HW Rx hash generated.
2044          *
2045          * if (!is_eth_multi(bp))
2046          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2047          */
2048
2049         params.rss_obj = rss_obj;
2050
2051         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2052
2053         if (enable) {
2054                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2055
2056                 /* RSS configuration */
2057                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2058                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2059                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2060                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2061                 if (rss_obj->udp_rss_v4)
2062                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2063                 if (rss_obj->udp_rss_v6)
2064                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2065         } else {
2066                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2067         }
2068
2069         /* Hash bits */
2070         params.rss_result_mask = MULTI_MASK;
2071
2072         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2073
2074         if (config_hash) {
2075                 /* RSS keys */
2076                 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2077                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2078         }
2079
2080         if (IS_PF(bp))
2081                 return bnx2x_config_rss(bp, &params);
2082         else
2083                 return bnx2x_vfpf_config_rss(bp, &params);
2084 }
2085
2086 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2087 {
2088         struct bnx2x_func_state_params func_params = {NULL};
2089
2090         /* Prepare parameters for function state transitions */
2091         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2092
2093         func_params.f_obj = &bp->func_obj;
2094         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2095
2096         func_params.params.hw_init.load_phase = load_code;
2097
2098         return bnx2x_func_state_change(bp, &func_params);
2099 }
2100
2101 /*
2102  * Cleans the object that have internal lists without sending
2103  * ramrods. Should be run when interrupts are disabled.
2104  */
2105 void bnx2x_squeeze_objects(struct bnx2x *bp)
2106 {
2107         int rc;
2108         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2109         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2110         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2111
2112         /***************** Cleanup MACs' object first *************************/
2113
2114         /* Wait for completion of requested */
2115         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2116         /* Perform a dry cleanup */
2117         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2118
2119         /* Clean ETH primary MAC */
2120         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2121         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2122                                  &ramrod_flags);
2123         if (rc != 0)
2124                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2125
2126         /* Cleanup UC list */
2127         vlan_mac_flags = 0;
2128         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2129         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2130                                  &ramrod_flags);
2131         if (rc != 0)
2132                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2133
2134         /***************** Now clean mcast object *****************************/
2135         rparam.mcast_obj = &bp->mcast_obj;
2136         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2137
2138         /* Add a DEL command... - Since we're doing a driver cleanup only,
2139          * we take a lock surrounding both the initial send and the CONTs,
2140          * as we don't want a true completion to disrupt us in the middle.
2141          */
2142         netif_addr_lock_bh(bp->dev);
2143         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2144         if (rc < 0)
2145                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2146                           rc);
2147
2148         /* ...and wait until all pending commands are cleared */
2149         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2150         while (rc != 0) {
2151                 if (rc < 0) {
2152                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2153                                   rc);
2154                         netif_addr_unlock_bh(bp->dev);
2155                         return;
2156                 }
2157
2158                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2159         }
2160         netif_addr_unlock_bh(bp->dev);
2161 }
2162
2163 #ifndef BNX2X_STOP_ON_ERROR
2164 #define LOAD_ERROR_EXIT(bp, label) \
2165         do { \
2166                 (bp)->state = BNX2X_STATE_ERROR; \
2167                 goto label; \
2168         } while (0)
2169
2170 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2171         do { \
2172                 bp->cnic_loaded = false; \
2173                 goto label; \
2174         } while (0)
2175 #else /*BNX2X_STOP_ON_ERROR*/
2176 #define LOAD_ERROR_EXIT(bp, label) \
2177         do { \
2178                 (bp)->state = BNX2X_STATE_ERROR; \
2179                 (bp)->panic = 1; \
2180                 return -EBUSY; \
2181         } while (0)
2182 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2183         do { \
2184                 bp->cnic_loaded = false; \
2185                 (bp)->panic = 1; \
2186                 return -EBUSY; \
2187         } while (0)
2188 #endif /*BNX2X_STOP_ON_ERROR*/
2189
2190 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2191 {
2192         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2193                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2194         return;
2195 }
2196
2197 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2198 {
2199         int num_groups, vf_headroom = 0;
2200         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2201
2202         /* number of queues for statistics is number of eth queues + FCoE */
2203         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2204
2205         /* Total number of FW statistics requests =
2206          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2207          * and fcoe l2 queue) stats + num of queues (which includes another 1
2208          * for fcoe l2 queue if applicable)
2209          */
2210         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2211
2212         /* vf stats appear in the request list, but their data is allocated by
2213          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2214          * it is used to determine where to place the vf stats queries in the
2215          * request struct
2216          */
2217         if (IS_SRIOV(bp))
2218                 vf_headroom = bnx2x_vf_headroom(bp);
2219
2220         /* Request is built from stats_query_header and an array of
2221          * stats_query_cmd_group each of which contains
2222          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2223          * configured in the stats_query_header.
2224          */
2225         num_groups =
2226                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2227                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2228                  1 : 0));
2229
2230         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2231            bp->fw_stats_num, vf_headroom, num_groups);
2232         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2233                 num_groups * sizeof(struct stats_query_cmd_group);
2234
2235         /* Data for statistics requests + stats_counter
2236          * stats_counter holds per-STORM counters that are incremented
2237          * when STORM has finished with the current request.
2238          * memory for FCoE offloaded statistics are counted anyway,
2239          * even if they will not be sent.
2240          * VF stats are not accounted for here as the data of VF stats is stored
2241          * in memory allocated by the VF, not here.
2242          */
2243         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2244                 sizeof(struct per_pf_stats) +
2245                 sizeof(struct fcoe_statistics_params) +
2246                 sizeof(struct per_queue_stats) * num_queue_stats +
2247                 sizeof(struct stats_counter);
2248
2249         bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2250                                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2251         if (!bp->fw_stats)
2252                 goto alloc_mem_err;
2253
2254         /* Set shortcuts */
2255         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2256         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2257         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2258                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2259         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2260                 bp->fw_stats_req_sz;
2261
2262         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2263            U64_HI(bp->fw_stats_req_mapping),
2264            U64_LO(bp->fw_stats_req_mapping));
2265         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2266            U64_HI(bp->fw_stats_data_mapping),
2267            U64_LO(bp->fw_stats_data_mapping));
2268         return 0;
2269
2270 alloc_mem_err:
2271         bnx2x_free_fw_stats_mem(bp);
2272         BNX2X_ERR("Can't allocate FW stats memory\n");
2273         return -ENOMEM;
2274 }
2275
2276 /* send load request to mcp and analyze response */
2277 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2278 {
2279         u32 param;
2280
2281         /* init fw_seq */
2282         bp->fw_seq =
2283                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2284                  DRV_MSG_SEQ_NUMBER_MASK);
2285         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2286
2287         /* Get current FW pulse sequence */
2288         bp->fw_drv_pulse_wr_seq =
2289                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2290                  DRV_PULSE_SEQ_MASK);
2291         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2292
2293         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2294
2295         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2296                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2297
2298         /* load request */
2299         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2300
2301         /* if mcp fails to respond we must abort */
2302         if (!(*load_code)) {
2303                 BNX2X_ERR("MCP response failure, aborting\n");
2304                 return -EBUSY;
2305         }
2306
2307         /* If mcp refused (e.g. other port is in diagnostic mode) we
2308          * must abort
2309          */
2310         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2311                 BNX2X_ERR("MCP refused load request, aborting\n");
2312                 return -EBUSY;
2313         }
2314         return 0;
2315 }
2316
2317 /* check whether another PF has already loaded FW to chip. In
2318  * virtualized environments a pf from another VM may have already
2319  * initialized the device including loading FW
2320  */
2321 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2322 {
2323         /* is another pf loaded on this engine? */
2324         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2325             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2326                 /* build my FW version dword */
2327                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2328                         (BCM_5710_FW_MINOR_VERSION << 8) +
2329                         (BCM_5710_FW_REVISION_VERSION << 16) +
2330                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2331
2332                 /* read loaded FW from chip */
2333                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2334
2335                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2336                    loaded_fw, my_fw);
2337
2338                 /* abort nic load if version mismatch */
2339                 if (my_fw != loaded_fw) {
2340                         if (print_err)
2341                                 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2342                                           loaded_fw, my_fw);
2343                         else
2344                                 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2345                                                loaded_fw, my_fw);
2346                         return -EBUSY;
2347                 }
2348         }
2349         return 0;
2350 }
2351
2352 /* returns the "mcp load_code" according to global load_count array */
2353 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2354 {
2355         int path = BP_PATH(bp);
2356
2357         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2358            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2359            bnx2x_load_count[path][2]);
2360         bnx2x_load_count[path][0]++;
2361         bnx2x_load_count[path][1 + port]++;
2362         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2363            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2364            bnx2x_load_count[path][2]);
2365         if (bnx2x_load_count[path][0] == 1)
2366                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2367         else if (bnx2x_load_count[path][1 + port] == 1)
2368                 return FW_MSG_CODE_DRV_LOAD_PORT;
2369         else
2370                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2371 }
2372
2373 /* mark PMF if applicable */
2374 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2375 {
2376         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2377             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2378             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2379                 bp->port.pmf = 1;
2380                 /* We need the barrier to ensure the ordering between the
2381                  * writing to bp->port.pmf here and reading it from the
2382                  * bnx2x_periodic_task().
2383                  */
2384                 smp_mb();
2385         } else {
2386                 bp->port.pmf = 0;
2387         }
2388
2389         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2390 }
2391
2392 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2393 {
2394         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2395              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2396             (bp->common.shmem2_base)) {
2397                 if (SHMEM2_HAS(bp, dcc_support))
2398                         SHMEM2_WR(bp, dcc_support,
2399                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2400                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2401                 if (SHMEM2_HAS(bp, afex_driver_support))
2402                         SHMEM2_WR(bp, afex_driver_support,
2403                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2404         }
2405
2406         /* Set AFEX default VLAN tag to an invalid value */
2407         bp->afex_def_vlan_tag = -1;
2408 }
2409
2410 /**
2411  * bnx2x_bz_fp - zero content of the fastpath structure.
2412  *
2413  * @bp:         driver handle
2414  * @index:      fastpath index to be zeroed
2415  *
2416  * Makes sure the contents of the bp->fp[index].napi is kept
2417  * intact.
2418  */
2419 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2420 {
2421         struct bnx2x_fastpath *fp = &bp->fp[index];
2422         int cos;
2423         struct napi_struct orig_napi = fp->napi;
2424         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2425
2426         /* bzero bnx2x_fastpath contents */
2427         if (fp->tpa_info)
2428                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2429                        sizeof(struct bnx2x_agg_info));
2430         memset(fp, 0, sizeof(*fp));
2431
2432         /* Restore the NAPI object as it has been already initialized */
2433         fp->napi = orig_napi;
2434         fp->tpa_info = orig_tpa_info;
2435         fp->bp = bp;
2436         fp->index = index;
2437         if (IS_ETH_FP(fp))
2438                 fp->max_cos = bp->max_cos;
2439         else
2440                 /* Special queues support only one CoS */
2441                 fp->max_cos = 1;
2442
2443         /* Init txdata pointers */
2444         if (IS_FCOE_FP(fp))
2445                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2446         if (IS_ETH_FP(fp))
2447                 for_each_cos_in_tx_queue(fp, cos)
2448                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2449                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2450
2451         /* set the tpa flag for each queue. The tpa flag determines the queue
2452          * minimal size so it must be set prior to queue memory allocation
2453          */
2454         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2455                                   (bp->flags & GRO_ENABLE_FLAG &&
2456                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2457         if (bp->flags & TPA_ENABLE_FLAG)
2458                 fp->mode = TPA_MODE_LRO;
2459         else if (bp->flags & GRO_ENABLE_FLAG)
2460                 fp->mode = TPA_MODE_GRO;
2461
2462         /* We don't want TPA on an FCoE L2 ring */
2463         if (IS_FCOE_FP(fp))
2464                 fp->disable_tpa = 1;
2465 }
2466
2467 int bnx2x_load_cnic(struct bnx2x *bp)
2468 {
2469         int i, rc, port = BP_PORT(bp);
2470
2471         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2472
2473         mutex_init(&bp->cnic_mutex);
2474
2475         if (IS_PF(bp)) {
2476                 rc = bnx2x_alloc_mem_cnic(bp);
2477                 if (rc) {
2478                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2479                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2480                 }
2481         }
2482
2483         rc = bnx2x_alloc_fp_mem_cnic(bp);
2484         if (rc) {
2485                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2486                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2487         }
2488
2489         /* Update the number of queues with the cnic queues */
2490         rc = bnx2x_set_real_num_queues(bp, 1);
2491         if (rc) {
2492                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2493                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2494         }
2495
2496         /* Add all CNIC NAPI objects */
2497         bnx2x_add_all_napi_cnic(bp);
2498         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2499         bnx2x_napi_enable_cnic(bp);
2500
2501         rc = bnx2x_init_hw_func_cnic(bp);
2502         if (rc)
2503                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2504
2505         bnx2x_nic_init_cnic(bp);
2506
2507         if (IS_PF(bp)) {
2508                 /* Enable Timer scan */
2509                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2510
2511                 /* setup cnic queues */
2512                 for_each_cnic_queue(bp, i) {
2513                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2514                         if (rc) {
2515                                 BNX2X_ERR("Queue setup failed\n");
2516                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2517                         }
2518                 }
2519         }
2520
2521         /* Initialize Rx filter. */
2522         bnx2x_set_rx_mode_inner(bp);
2523
2524         /* re-read iscsi info */
2525         bnx2x_get_iscsi_info(bp);
2526         bnx2x_setup_cnic_irq_info(bp);
2527         bnx2x_setup_cnic_info(bp);
2528         bp->cnic_loaded = true;
2529         if (bp->state == BNX2X_STATE_OPEN)
2530                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2531
2532         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2533
2534         return 0;
2535
2536 #ifndef BNX2X_STOP_ON_ERROR
2537 load_error_cnic2:
2538         /* Disable Timer scan */
2539         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2540
2541 load_error_cnic1:
2542         bnx2x_napi_disable_cnic(bp);
2543         /* Update the number of queues without the cnic queues */
2544         if (bnx2x_set_real_num_queues(bp, 0))
2545                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2546 load_error_cnic0:
2547         BNX2X_ERR("CNIC-related load failed\n");
2548         bnx2x_free_fp_mem_cnic(bp);
2549         bnx2x_free_mem_cnic(bp);
2550         return rc;
2551 #endif /* ! BNX2X_STOP_ON_ERROR */
2552 }
2553
2554 /* must be called with rtnl_lock */
2555 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2556 {
2557         int port = BP_PORT(bp);
2558         int i, rc = 0, load_code = 0;
2559
2560         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2561         DP(NETIF_MSG_IFUP,
2562            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2563
2564 #ifdef BNX2X_STOP_ON_ERROR
2565         if (unlikely(bp->panic)) {
2566                 BNX2X_ERR("Can't load NIC when there is panic\n");
2567                 return -EPERM;
2568         }
2569 #endif
2570
2571         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2572
2573         /* zero the structure w/o any lock, before SP handler is initialized */
2574         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2575         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2576                 &bp->last_reported_link.link_report_flags);
2577
2578         if (IS_PF(bp))
2579                 /* must be called before memory allocation and HW init */
2580                 bnx2x_ilt_set_info(bp);
2581
2582         /*
2583          * Zero fastpath structures preserving invariants like napi, which are
2584          * allocated only once, fp index, max_cos, bp pointer.
2585          * Also set fp->disable_tpa and txdata_ptr.
2586          */
2587         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2588         for_each_queue(bp, i)
2589                 bnx2x_bz_fp(bp, i);
2590         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2591                                   bp->num_cnic_queues) *
2592                                   sizeof(struct bnx2x_fp_txdata));
2593
2594         bp->fcoe_init = false;
2595
2596         /* Set the receive queues buffer size */
2597         bnx2x_set_rx_buf_size(bp);
2598
2599         if (IS_PF(bp)) {
2600                 rc = bnx2x_alloc_mem(bp);
2601                 if (rc) {
2602                         BNX2X_ERR("Unable to allocate bp memory\n");
2603                         return rc;
2604                 }
2605         }
2606
2607         /* need to be done after alloc mem, since it's self adjusting to amount
2608          * of memory available for RSS queues
2609          */
2610         rc = bnx2x_alloc_fp_mem(bp);
2611         if (rc) {
2612                 BNX2X_ERR("Unable to allocate memory for fps\n");
2613                 LOAD_ERROR_EXIT(bp, load_error0);
2614         }
2615
2616         /* Allocated memory for FW statistics  */
2617         if (bnx2x_alloc_fw_stats_mem(bp))
2618                 LOAD_ERROR_EXIT(bp, load_error0);
2619
2620         /* request pf to initialize status blocks */
2621         if (IS_VF(bp)) {
2622                 rc = bnx2x_vfpf_init(bp);
2623                 if (rc)
2624                         LOAD_ERROR_EXIT(bp, load_error0);
2625         }
2626
2627         /* As long as bnx2x_alloc_mem() may possibly update
2628          * bp->num_queues, bnx2x_set_real_num_queues() should always
2629          * come after it. At this stage cnic queues are not counted.
2630          */
2631         rc = bnx2x_set_real_num_queues(bp, 0);
2632         if (rc) {
2633                 BNX2X_ERR("Unable to set real_num_queues\n");
2634                 LOAD_ERROR_EXIT(bp, load_error0);
2635         }
2636
2637         /* configure multi cos mappings in kernel.
2638          * this configuration may be overridden by a multi class queue
2639          * discipline or by a dcbx negotiation result.
2640          */
2641         bnx2x_setup_tc(bp->dev, bp->max_cos);
2642
2643         /* Add all NAPI objects */
2644         bnx2x_add_all_napi(bp);
2645         DP(NETIF_MSG_IFUP, "napi added\n");
2646         bnx2x_napi_enable(bp);
2647
2648         if (IS_PF(bp)) {
2649                 /* set pf load just before approaching the MCP */
2650                 bnx2x_set_pf_load(bp);
2651
2652                 /* if mcp exists send load request and analyze response */
2653                 if (!BP_NOMCP(bp)) {
2654                         /* attempt to load pf */
2655                         rc = bnx2x_nic_load_request(bp, &load_code);
2656                         if (rc)
2657                                 LOAD_ERROR_EXIT(bp, load_error1);
2658
2659                         /* what did mcp say? */
2660                         rc = bnx2x_compare_fw_ver(bp, load_code, true);
2661                         if (rc) {
2662                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2663                                 LOAD_ERROR_EXIT(bp, load_error2);
2664                         }
2665                 } else {
2666                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2667                 }
2668
2669                 /* mark pmf if applicable */
2670                 bnx2x_nic_load_pmf(bp, load_code);
2671
2672                 /* Init Function state controlling object */
2673                 bnx2x__init_func_obj(bp);
2674
2675                 /* Initialize HW */
2676                 rc = bnx2x_init_hw(bp, load_code);
2677                 if (rc) {
2678                         BNX2X_ERR("HW init failed, aborting\n");
2679                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2680                         LOAD_ERROR_EXIT(bp, load_error2);
2681                 }
2682         }
2683
2684         bnx2x_pre_irq_nic_init(bp);
2685
2686         /* Connect to IRQs */
2687         rc = bnx2x_setup_irqs(bp);
2688         if (rc) {
2689                 BNX2X_ERR("setup irqs failed\n");
2690                 if (IS_PF(bp))
2691                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2692                 LOAD_ERROR_EXIT(bp, load_error2);
2693         }
2694
2695         /* Init per-function objects */
2696         if (IS_PF(bp)) {
2697                 /* Setup NIC internals and enable interrupts */
2698                 bnx2x_post_irq_nic_init(bp, load_code);
2699
2700                 bnx2x_init_bp_objs(bp);
2701                 bnx2x_iov_nic_init(bp);
2702
2703                 /* Set AFEX default VLAN tag to an invalid value */
2704                 bp->afex_def_vlan_tag = -1;
2705                 bnx2x_nic_load_afex_dcc(bp, load_code);
2706                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2707                 rc = bnx2x_func_start(bp);
2708                 if (rc) {
2709                         BNX2X_ERR("Function start failed!\n");
2710                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2711
2712                         LOAD_ERROR_EXIT(bp, load_error3);
2713                 }
2714
2715                 /* Send LOAD_DONE command to MCP */
2716                 if (!BP_NOMCP(bp)) {
2717                         load_code = bnx2x_fw_command(bp,
2718                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2719                         if (!load_code) {
2720                                 BNX2X_ERR("MCP response failure, aborting\n");
2721                                 rc = -EBUSY;
2722                                 LOAD_ERROR_EXIT(bp, load_error3);
2723                         }
2724                 }
2725
2726                 /* initialize FW coalescing state machines in RAM */
2727                 bnx2x_update_coalesce(bp);
2728         }
2729
2730         /* setup the leading queue */
2731         rc = bnx2x_setup_leading(bp);
2732         if (rc) {
2733                 BNX2X_ERR("Setup leading failed!\n");
2734                 LOAD_ERROR_EXIT(bp, load_error3);
2735         }
2736
2737         /* set up the rest of the queues */
2738         for_each_nondefault_eth_queue(bp, i) {
2739                 if (IS_PF(bp))
2740                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2741                 else /* VF */
2742                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2743                 if (rc) {
2744                         BNX2X_ERR("Queue %d setup failed\n", i);
2745                         LOAD_ERROR_EXIT(bp, load_error3);
2746                 }
2747         }
2748
2749         /* setup rss */
2750         rc = bnx2x_init_rss(bp);
2751         if (rc) {
2752                 BNX2X_ERR("PF RSS init failed\n");
2753                 LOAD_ERROR_EXIT(bp, load_error3);
2754         }
2755
2756         /* Now when Clients are configured we are ready to work */
2757         bp->state = BNX2X_STATE_OPEN;
2758
2759         /* Configure a ucast MAC */
2760         if (IS_PF(bp))
2761                 rc = bnx2x_set_eth_mac(bp, true);
2762         else /* vf */
2763                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2764                                            true);
2765         if (rc) {
2766                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2767                 LOAD_ERROR_EXIT(bp, load_error3);
2768         }
2769
2770         if (IS_PF(bp) && bp->pending_max) {
2771                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2772                 bp->pending_max = 0;
2773         }
2774
2775         if (bp->port.pmf) {
2776                 rc = bnx2x_initial_phy_init(bp, load_mode);
2777                 if (rc)
2778                         LOAD_ERROR_EXIT(bp, load_error3);
2779         }
2780         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2781
2782         /* Start fast path */
2783
2784         /* Initialize Rx filter. */
2785         bnx2x_set_rx_mode_inner(bp);
2786
2787         /* Start the Tx */
2788         switch (load_mode) {
2789         case LOAD_NORMAL:
2790                 /* Tx queue should be only re-enabled */
2791                 netif_tx_wake_all_queues(bp->dev);
2792                 break;
2793
2794         case LOAD_OPEN:
2795                 netif_tx_start_all_queues(bp->dev);
2796                 smp_mb__after_atomic();
2797                 break;
2798
2799         case LOAD_DIAG:
2800         case LOAD_LOOPBACK_EXT:
2801                 bp->state = BNX2X_STATE_DIAG;
2802                 break;
2803
2804         default:
2805                 break;
2806         }
2807
2808         if (bp->port.pmf)
2809                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2810         else
2811                 bnx2x__link_status_update(bp);
2812
2813         /* start the timer */
2814         mod_timer(&bp->timer, jiffies + bp->current_interval);
2815
2816         if (CNIC_ENABLED(bp))
2817                 bnx2x_load_cnic(bp);
2818
2819         if (IS_PF(bp))
2820                 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2821
2822         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2823                 /* mark driver is loaded in shmem2 */
2824                 u32 val;
2825                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2826                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2827                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2828                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2829         }
2830
2831         /* Wait for all pending SP commands to complete */
2832         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2833                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2834                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2835                 return -EBUSY;
2836         }
2837
2838         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2839         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2840                 bnx2x_dcbx_init(bp, false);
2841
2842         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2843
2844         return 0;
2845
2846 #ifndef BNX2X_STOP_ON_ERROR
2847 load_error3:
2848         if (IS_PF(bp)) {
2849                 bnx2x_int_disable_sync(bp, 1);
2850
2851                 /* Clean queueable objects */
2852                 bnx2x_squeeze_objects(bp);
2853         }
2854
2855         /* Free SKBs, SGEs, TPA pool and driver internals */
2856         bnx2x_free_skbs(bp);
2857         for_each_rx_queue(bp, i)
2858                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2859
2860         /* Release IRQs */
2861         bnx2x_free_irq(bp);
2862 load_error2:
2863         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2864                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2865                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2866         }
2867
2868         bp->port.pmf = 0;
2869 load_error1:
2870         bnx2x_napi_disable(bp);
2871         bnx2x_del_all_napi(bp);
2872
2873         /* clear pf_load status, as it was already set */
2874         if (IS_PF(bp))
2875                 bnx2x_clear_pf_load(bp);
2876 load_error0:
2877         bnx2x_free_fw_stats_mem(bp);
2878         bnx2x_free_fp_mem(bp);
2879         bnx2x_free_mem(bp);
2880
2881         return rc;
2882 #endif /* ! BNX2X_STOP_ON_ERROR */
2883 }
2884
2885 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2886 {
2887         u8 rc = 0, cos, i;
2888
2889         /* Wait until tx fastpath tasks complete */
2890         for_each_tx_queue(bp, i) {
2891                 struct bnx2x_fastpath *fp = &bp->fp[i];
2892
2893                 for_each_cos_in_tx_queue(fp, cos)
2894                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2895                 if (rc)
2896                         return rc;
2897         }
2898         return 0;
2899 }
2900
2901 /* must be called with rtnl_lock */
2902 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2903 {
2904         int i;
2905         bool global = false;
2906
2907         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2908
2909         /* mark driver is unloaded in shmem2 */
2910         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2911                 u32 val;
2912                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2913                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2914                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2915         }
2916
2917         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2918             (bp->state == BNX2X_STATE_CLOSED ||
2919              bp->state == BNX2X_STATE_ERROR)) {
2920                 /* We can get here if the driver has been unloaded
2921                  * during parity error recovery and is either waiting for a
2922                  * leader to complete or for other functions to unload and
2923                  * then ifdown has been issued. In this case we want to
2924                  * unload and let other functions to complete a recovery
2925                  * process.
2926                  */
2927                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2928                 bp->is_leader = 0;
2929                 bnx2x_release_leader_lock(bp);
2930                 smp_mb();
2931
2932                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2933                 BNX2X_ERR("Can't unload in closed or error state\n");
2934                 return -EINVAL;
2935         }
2936
2937         /* Nothing to do during unload if previous bnx2x_nic_load()
2938          * have not completed successfully - all resources are released.
2939          *
2940          * we can get here only after unsuccessful ndo_* callback, during which
2941          * dev->IFF_UP flag is still on.
2942          */
2943         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2944                 return 0;
2945
2946         /* It's important to set the bp->state to the value different from
2947          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2948          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2949          */
2950         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2951         smp_mb();
2952
2953         /* indicate to VFs that the PF is going down */
2954         bnx2x_iov_channel_down(bp);
2955
2956         if (CNIC_LOADED(bp))
2957                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2958
2959         /* Stop Tx */
2960         bnx2x_tx_disable(bp);
2961         netdev_reset_tc(bp->dev);
2962
2963         bp->rx_mode = BNX2X_RX_MODE_NONE;
2964
2965         del_timer_sync(&bp->timer);
2966
2967         if (IS_PF(bp)) {
2968                 /* Set ALWAYS_ALIVE bit in shmem */
2969                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2970                 bnx2x_drv_pulse(bp);
2971                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2972                 bnx2x_save_statistics(bp);
2973         }
2974
2975         /* wait till consumers catch up with producers in all queues */
2976         bnx2x_drain_tx_queues(bp);
2977
2978         /* if VF indicate to PF this function is going down (PF will delete sp
2979          * elements and clear initializations
2980          */
2981         if (IS_VF(bp))
2982                 bnx2x_vfpf_close_vf(bp);
2983         else if (unload_mode != UNLOAD_RECOVERY)
2984                 /* if this is a normal/close unload need to clean up chip*/
2985                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2986         else {
2987                 /* Send the UNLOAD_REQUEST to the MCP */
2988                 bnx2x_send_unload_req(bp, unload_mode);
2989
2990                 /* Prevent transactions to host from the functions on the
2991                  * engine that doesn't reset global blocks in case of global
2992                  * attention once global blocks are reset and gates are opened
2993                  * (the engine which leader will perform the recovery
2994                  * last).
2995                  */
2996                 if (!CHIP_IS_E1x(bp))
2997                         bnx2x_pf_disable(bp);
2998
2999                 /* Disable HW interrupts, NAPI */
3000                 bnx2x_netif_stop(bp, 1);
3001                 /* Delete all NAPI objects */
3002                 bnx2x_del_all_napi(bp);
3003                 if (CNIC_LOADED(bp))
3004                         bnx2x_del_all_napi_cnic(bp);
3005                 /* Release IRQs */
3006                 bnx2x_free_irq(bp);
3007
3008                 /* Report UNLOAD_DONE to MCP */
3009                 bnx2x_send_unload_done(bp, false);
3010         }
3011
3012         /*
3013          * At this stage no more interrupts will arrive so we may safely clean
3014          * the queueable objects here in case they failed to get cleaned so far.
3015          */
3016         if (IS_PF(bp))
3017                 bnx2x_squeeze_objects(bp);
3018
3019         /* There should be no more pending SP commands at this stage */
3020         bp->sp_state = 0;
3021
3022         bp->port.pmf = 0;
3023
3024         /* clear pending work in rtnl task */
3025         bp->sp_rtnl_state = 0;
3026         smp_mb();
3027
3028         /* Free SKBs, SGEs, TPA pool and driver internals */
3029         bnx2x_free_skbs(bp);
3030         if (CNIC_LOADED(bp))
3031                 bnx2x_free_skbs_cnic(bp);
3032         for_each_rx_queue(bp, i)
3033                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3034
3035         bnx2x_free_fp_mem(bp);
3036         if (CNIC_LOADED(bp))
3037                 bnx2x_free_fp_mem_cnic(bp);
3038
3039         if (IS_PF(bp)) {
3040                 if (CNIC_LOADED(bp))
3041                         bnx2x_free_mem_cnic(bp);
3042         }
3043         bnx2x_free_mem(bp);
3044
3045         bp->state = BNX2X_STATE_CLOSED;
3046         bp->cnic_loaded = false;
3047
3048         /* Clear driver version indication in shmem */
3049         if (IS_PF(bp))
3050                 bnx2x_update_mng_version(bp);
3051
3052         /* Check if there are pending parity attentions. If there are - set
3053          * RECOVERY_IN_PROGRESS.
3054          */
3055         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3056                 bnx2x_set_reset_in_progress(bp);
3057
3058                 /* Set RESET_IS_GLOBAL if needed */
3059                 if (global)
3060                         bnx2x_set_reset_global(bp);
3061         }
3062
3063         /* The last driver must disable a "close the gate" if there is no
3064          * parity attention or "process kill" pending.
3065          */
3066         if (IS_PF(bp) &&
3067             !bnx2x_clear_pf_load(bp) &&
3068             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3069                 bnx2x_disable_close_the_gate(bp);
3070
3071         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3072
3073         return 0;
3074 }
3075
3076 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3077 {
3078         u16 pmcsr;
3079
3080         /* If there is no power capability, silently succeed */
3081         if (!bp->pdev->pm_cap) {
3082                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3083                 return 0;
3084         }
3085
3086         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3087
3088         switch (state) {
3089         case PCI_D0:
3090                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3091                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3092                                        PCI_PM_CTRL_PME_STATUS));
3093
3094                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3095                         /* delay required during transition out of D3hot */
3096                         msleep(20);
3097                 break;
3098
3099         case PCI_D3hot:
3100                 /* If there are other clients above don't
3101                    shut down the power */
3102                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3103                         return 0;
3104                 /* Don't shut down the power for emulation and FPGA */
3105                 if (CHIP_REV_IS_SLOW(bp))
3106                         return 0;
3107
3108                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3109                 pmcsr |= 3;
3110
3111                 if (bp->wol)
3112                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3113
3114                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3115                                       pmcsr);
3116
3117                 /* No more memory access after this point until
3118                 * device is brought back to D0.
3119                 */
3120                 break;
3121
3122         default:
3123                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3124                 return -EINVAL;
3125         }
3126         return 0;
3127 }
3128
3129 /*
3130  * net_device service functions
3131  */
3132 static int bnx2x_poll(struct napi_struct *napi, int budget)
3133 {
3134         int work_done = 0;
3135         u8 cos;
3136         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3137                                                  napi);
3138         struct bnx2x *bp = fp->bp;
3139
3140         while (1) {
3141 #ifdef BNX2X_STOP_ON_ERROR
3142                 if (unlikely(bp->panic)) {
3143                         napi_complete(napi);
3144                         return 0;
3145                 }
3146 #endif
3147                 if (!bnx2x_fp_lock_napi(fp))
3148                         return work_done;
3149
3150                 for_each_cos_in_tx_queue(fp, cos)
3151                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3152                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3153
3154                 if (bnx2x_has_rx_work(fp)) {
3155                         work_done += bnx2x_rx_int(fp, budget - work_done);
3156
3157                         /* must not complete if we consumed full budget */
3158                         if (work_done >= budget) {
3159                                 bnx2x_fp_unlock_napi(fp);
3160                                 break;
3161                         }
3162                 }
3163
3164                 /* Fall out from the NAPI loop if needed */
3165                 if (!bnx2x_fp_unlock_napi(fp) &&
3166                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3167
3168                         /* No need to update SB for FCoE L2 ring as long as
3169                          * it's connected to the default SB and the SB
3170                          * has been updated when NAPI was scheduled.
3171                          */
3172                         if (IS_FCOE_FP(fp)) {
3173                                 napi_complete(napi);
3174                                 break;
3175                         }
3176                         bnx2x_update_fpsb_idx(fp);
3177                         /* bnx2x_has_rx_work() reads the status block,
3178                          * thus we need to ensure that status block indices
3179                          * have been actually read (bnx2x_update_fpsb_idx)
3180                          * prior to this check (bnx2x_has_rx_work) so that
3181                          * we won't write the "newer" value of the status block
3182                          * to IGU (if there was a DMA right after
3183                          * bnx2x_has_rx_work and if there is no rmb, the memory
3184                          * reading (bnx2x_update_fpsb_idx) may be postponed
3185                          * to right before bnx2x_ack_sb). In this case there
3186                          * will never be another interrupt until there is
3187                          * another update of the status block, while there
3188                          * is still unhandled work.
3189                          */
3190                         rmb();
3191
3192                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3193                                 napi_complete(napi);
3194                                 /* Re-enable interrupts */
3195                                 DP(NETIF_MSG_RX_STATUS,
3196                                    "Update index to %d\n", fp->fp_hc_idx);
3197                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3198                                              le16_to_cpu(fp->fp_hc_idx),
3199                                              IGU_INT_ENABLE, 1);
3200                                 break;
3201                         }
3202                 }
3203         }
3204
3205         return work_done;
3206 }
3207
3208 #ifdef CONFIG_NET_RX_BUSY_POLL
3209 /* must be called with local_bh_disable()d */
3210 int bnx2x_low_latency_recv(struct napi_struct *napi)
3211 {
3212         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3213                                                  napi);
3214         struct bnx2x *bp = fp->bp;
3215         int found = 0;
3216
3217         if ((bp->state == BNX2X_STATE_CLOSED) ||
3218             (bp->state == BNX2X_STATE_ERROR) ||
3219             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3220                 return LL_FLUSH_FAILED;
3221
3222         if (!bnx2x_fp_lock_poll(fp))
3223                 return LL_FLUSH_BUSY;
3224
3225         if (bnx2x_has_rx_work(fp))
3226                 found = bnx2x_rx_int(fp, 4);
3227
3228         bnx2x_fp_unlock_poll(fp);
3229
3230         return found;
3231 }
3232 #endif
3233
3234 /* we split the first BD into headers and data BDs
3235  * to ease the pain of our fellow microcode engineers
3236  * we use one mapping for both BDs
3237  */
3238 static u16 bnx2x_tx_split(struct bnx2x *bp,
3239                           struct bnx2x_fp_txdata *txdata,
3240                           struct sw_tx_bd *tx_buf,
3241                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3242                           u16 bd_prod)
3243 {
3244         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3245         struct eth_tx_bd *d_tx_bd;
3246         dma_addr_t mapping;
3247         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3248
3249         /* first fix first BD */
3250         h_tx_bd->nbytes = cpu_to_le16(hlen);
3251
3252         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3253            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3254
3255         /* now get a new data BD
3256          * (after the pbd) and fill it */
3257         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3258         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3259
3260         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3261                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3262
3263         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3264         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3265         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3266
3267         /* this marks the BD as one that has no individual mapping */
3268         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3269
3270         DP(NETIF_MSG_TX_QUEUED,
3271            "TSO split data size is %d (%x:%x)\n",
3272            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3273
3274         /* update tx_bd */
3275         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3276
3277         return bd_prod;
3278 }
3279
3280 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3281 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3282 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3283 {
3284         __sum16 tsum = (__force __sum16) csum;
3285
3286         if (fix > 0)
3287                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3288                                   csum_partial(t_header - fix, fix, 0)));
3289
3290         else if (fix < 0)
3291                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3292                                   csum_partial(t_header, -fix, 0)));
3293
3294         return bswab16(tsum);
3295 }
3296
3297 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3298 {
3299         u32 rc;
3300         __u8 prot = 0;
3301         __be16 protocol;
3302
3303         if (skb->ip_summed != CHECKSUM_PARTIAL)
3304                 return XMIT_PLAIN;
3305
3306         protocol = vlan_get_protocol(skb);
3307         if (protocol == htons(ETH_P_IPV6)) {
3308                 rc = XMIT_CSUM_V6;
3309                 prot = ipv6_hdr(skb)->nexthdr;
3310         } else {
3311                 rc = XMIT_CSUM_V4;
3312                 prot = ip_hdr(skb)->protocol;
3313         }
3314
3315         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3316                 if (inner_ip_hdr(skb)->version == 6) {
3317                         rc |= XMIT_CSUM_ENC_V6;
3318                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3319                                 rc |= XMIT_CSUM_TCP;
3320                 } else {
3321                         rc |= XMIT_CSUM_ENC_V4;
3322                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3323                                 rc |= XMIT_CSUM_TCP;
3324                 }
3325         }
3326         if (prot == IPPROTO_TCP)
3327                 rc |= XMIT_CSUM_TCP;
3328
3329         if (skb_is_gso(skb)) {
3330                 if (skb_is_gso_v6(skb)) {
3331                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3332                         if (rc & XMIT_CSUM_ENC)
3333                                 rc |= XMIT_GSO_ENC_V6;
3334                 } else {
3335                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3336                         if (rc & XMIT_CSUM_ENC)
3337                                 rc |= XMIT_GSO_ENC_V4;
3338                 }
3339         }
3340
3341         return rc;
3342 }
3343
3344 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3345 /* check if packet requires linearization (packet is too fragmented)
3346    no need to check fragmentation if page size > 8K (there will be no
3347    violation to FW restrictions) */
3348 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3349                              u32 xmit_type)
3350 {
3351         int to_copy = 0;
3352         int hlen = 0;
3353         int first_bd_sz = 0;
3354
3355         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3356         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3357
3358                 if (xmit_type & XMIT_GSO) {
3359                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3360                         /* Check if LSO packet needs to be copied:
3361                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3362                         int wnd_size = MAX_FETCH_BD - 3;
3363                         /* Number of windows to check */
3364                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3365                         int wnd_idx = 0;
3366                         int frag_idx = 0;
3367                         u32 wnd_sum = 0;
3368
3369                         /* Headers length */
3370                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3371                                 tcp_hdrlen(skb);
3372
3373                         /* Amount of data (w/o headers) on linear part of SKB*/
3374                         first_bd_sz = skb_headlen(skb) - hlen;
3375
3376                         wnd_sum  = first_bd_sz;
3377
3378                         /* Calculate the first sum - it's special */
3379                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3380                                 wnd_sum +=
3381                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3382
3383                         /* If there was data on linear skb data - check it */
3384                         if (first_bd_sz > 0) {
3385                                 if (unlikely(wnd_sum < lso_mss)) {
3386                                         to_copy = 1;
3387                                         goto exit_lbl;
3388                                 }
3389
3390                                 wnd_sum -= first_bd_sz;
3391                         }
3392
3393                         /* Others are easier: run through the frag list and
3394                            check all windows */
3395                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3396                                 wnd_sum +=
3397                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3398
3399                                 if (unlikely(wnd_sum < lso_mss)) {
3400                                         to_copy = 1;
3401                                         break;
3402                                 }
3403                                 wnd_sum -=
3404                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3405                         }
3406                 } else {
3407                         /* in non-LSO too fragmented packet should always
3408                            be linearized */
3409                         to_copy = 1;
3410                 }
3411         }
3412
3413 exit_lbl:
3414         if (unlikely(to_copy))
3415                 DP(NETIF_MSG_TX_QUEUED,
3416                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3417                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3418                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3419
3420         return to_copy;
3421 }
3422 #endif
3423
3424 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3425                                  u32 xmit_type)
3426 {
3427         struct ipv6hdr *ipv6;
3428
3429         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3430                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3431                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3432
3433         if (xmit_type & XMIT_GSO_ENC_V6)
3434                 ipv6 = inner_ipv6_hdr(skb);
3435         else if (xmit_type & XMIT_GSO_V6)
3436                 ipv6 = ipv6_hdr(skb);
3437         else
3438                 ipv6 = NULL;
3439
3440         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3441                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3442 }
3443
3444 /**
3445  * bnx2x_set_pbd_gso - update PBD in GSO case.
3446  *
3447  * @skb:        packet skb
3448  * @pbd:        parse BD
3449  * @xmit_type:  xmit flags
3450  */
3451 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3452                               struct eth_tx_parse_bd_e1x *pbd,
3453                               struct eth_tx_start_bd *tx_start_bd,
3454                               u32 xmit_type)
3455 {
3456         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3457         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3458         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3459
3460         if (xmit_type & XMIT_GSO_V4) {
3461                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3462                 pbd->tcp_pseudo_csum =
3463                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3464                                                    ip_hdr(skb)->daddr,
3465                                                    0, IPPROTO_TCP, 0));
3466
3467                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3468                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3469         } else {
3470                 pbd->tcp_pseudo_csum =
3471                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3472                                                  &ipv6_hdr(skb)->daddr,
3473                                                  0, IPPROTO_TCP, 0));
3474         }
3475
3476         pbd->global_data |=
3477                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3478 }
3479
3480 /**
3481  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3482  *
3483  * @bp:                 driver handle
3484  * @skb:                packet skb
3485  * @parsing_data:       data to be updated
3486  * @xmit_type:          xmit flags
3487  *
3488  * 57712/578xx related, when skb has encapsulation
3489  */
3490 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3491                                  u32 *parsing_data, u32 xmit_type)
3492 {
3493         *parsing_data |=
3494                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3495                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3496                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3497
3498         if (xmit_type & XMIT_CSUM_TCP) {
3499                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3500                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3501                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3502
3503                 return skb_inner_transport_header(skb) +
3504                         inner_tcp_hdrlen(skb) - skb->data;
3505         }
3506
3507         /* We support checksum offload for TCP and UDP only.
3508          * No need to pass the UDP header length - it's a constant.
3509          */
3510         return skb_inner_transport_header(skb) +
3511                 sizeof(struct udphdr) - skb->data;
3512 }
3513
3514 /**
3515  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3516  *
3517  * @bp:                 driver handle
3518  * @skb:                packet skb
3519  * @parsing_data:       data to be updated
3520  * @xmit_type:          xmit flags
3521  *
3522  * 57712/578xx related
3523  */
3524 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3525                                 u32 *parsing_data, u32 xmit_type)
3526 {
3527         *parsing_data |=
3528                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3529                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3530                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3531
3532         if (xmit_type & XMIT_CSUM_TCP) {
3533                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3534                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3535                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3536
3537                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3538         }
3539         /* We support checksum offload for TCP and UDP only.
3540          * No need to pass the UDP header length - it's a constant.
3541          */
3542         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3543 }
3544
3545 /* set FW indication according to inner or outer protocols if tunneled */
3546 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3547                                struct eth_tx_start_bd *tx_start_bd,
3548                                u32 xmit_type)
3549 {
3550         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3551
3552         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3553                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3554
3555         if (!(xmit_type & XMIT_CSUM_TCP))
3556                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3557 }
3558
3559 /**
3560  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3561  *
3562  * @bp:         driver handle
3563  * @skb:        packet skb
3564  * @pbd:        parse BD to be updated
3565  * @xmit_type:  xmit flags
3566  */
3567 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3568                              struct eth_tx_parse_bd_e1x *pbd,
3569                              u32 xmit_type)
3570 {
3571         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3572
3573         /* for now NS flag is not used in Linux */
3574         pbd->global_data =
3575                 cpu_to_le16(hlen |
3576                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3577                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3578
3579         pbd->ip_hlen_w = (skb_transport_header(skb) -
3580                         skb_network_header(skb)) >> 1;
3581
3582         hlen += pbd->ip_hlen_w;
3583
3584         /* We support checksum offload for TCP and UDP only */
3585         if (xmit_type & XMIT_CSUM_TCP)
3586                 hlen += tcp_hdrlen(skb) / 2;
3587         else
3588                 hlen += sizeof(struct udphdr) / 2;
3589
3590         pbd->total_hlen_w = cpu_to_le16(hlen);
3591         hlen = hlen*2;
3592
3593         if (xmit_type & XMIT_CSUM_TCP) {
3594                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3595
3596         } else {
3597                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3598
3599                 DP(NETIF_MSG_TX_QUEUED,
3600                    "hlen %d  fix %d  csum before fix %x\n",
3601                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3602
3603                 /* HW bug: fixup the CSUM */
3604                 pbd->tcp_pseudo_csum =
3605                         bnx2x_csum_fix(skb_transport_header(skb),
3606                                        SKB_CS(skb), fix);
3607
3608                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3609                    pbd->tcp_pseudo_csum);
3610         }
3611
3612         return hlen;
3613 }
3614
3615 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3616                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3617                                       struct eth_tx_parse_2nd_bd *pbd2,
3618                                       u16 *global_data,
3619                                       u32 xmit_type)
3620 {
3621         u16 hlen_w = 0;
3622         u8 outerip_off, outerip_len = 0;
3623
3624         /* from outer IP to transport */
3625         hlen_w = (skb_inner_transport_header(skb) -
3626                   skb_network_header(skb)) >> 1;
3627
3628         /* transport len */
3629         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3630
3631         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3632
3633         /* outer IP header info */
3634         if (xmit_type & XMIT_CSUM_V4) {
3635                 struct iphdr *iph = ip_hdr(skb);
3636                 u32 csum = (__force u32)(~iph->check) -
3637                            (__force u32)iph->tot_len -
3638                            (__force u32)iph->frag_off;
3639
3640                 pbd2->fw_ip_csum_wo_len_flags_frag =
3641                         bswab16(csum_fold((__force __wsum)csum));
3642         } else {
3643                 pbd2->fw_ip_hdr_to_payload_w =
3644                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3645         }
3646
3647         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3648
3649         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3650
3651         if (xmit_type & XMIT_GSO_V4) {
3652                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3653
3654                 pbd_e2->data.tunnel_data.pseudo_csum =
3655                         bswab16(~csum_tcpudp_magic(
3656                                         inner_ip_hdr(skb)->saddr,
3657                                         inner_ip_hdr(skb)->daddr,
3658                                         0, IPPROTO_TCP, 0));
3659
3660                 outerip_len = ip_hdr(skb)->ihl << 1;
3661         } else {
3662                 pbd_e2->data.tunnel_data.pseudo_csum =
3663                         bswab16(~csum_ipv6_magic(
3664                                         &inner_ipv6_hdr(skb)->saddr,
3665                                         &inner_ipv6_hdr(skb)->daddr,
3666                                         0, IPPROTO_TCP, 0));
3667         }
3668
3669         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3670
3671         *global_data |=
3672                 outerip_off |
3673                 (!!(xmit_type & XMIT_CSUM_V6) <<
3674                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3675                 (outerip_len <<
3676                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3677                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3678                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3679
3680         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3681                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3682                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3683         }
3684 }
3685
3686 /* called with netif_tx_lock
3687  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3688  * netif_wake_queue()
3689  */
3690 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3691 {
3692         struct bnx2x *bp = netdev_priv(dev);
3693
3694         struct netdev_queue *txq;
3695         struct bnx2x_fp_txdata *txdata;
3696         struct sw_tx_bd *tx_buf;
3697         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3698         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3699         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3700         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3701         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3702         u32 pbd_e2_parsing_data = 0;
3703         u16 pkt_prod, bd_prod;
3704         int nbd, txq_index;
3705         dma_addr_t mapping;
3706         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3707         int i;
3708         u8 hlen = 0;
3709         __le16 pkt_size = 0;
3710         struct ethhdr *eth;
3711         u8 mac_type = UNICAST_ADDRESS;
3712
3713 #ifdef BNX2X_STOP_ON_ERROR
3714         if (unlikely(bp->panic))
3715                 return NETDEV_TX_BUSY;
3716 #endif
3717
3718         txq_index = skb_get_queue_mapping(skb);
3719         txq = netdev_get_tx_queue(dev, txq_index);
3720
3721         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3722
3723         txdata = &bp->bnx2x_txq[txq_index];
3724
3725         /* enable this debug print to view the transmission queue being used
3726         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3727            txq_index, fp_index, txdata_index); */
3728
3729         /* enable this debug print to view the transmission details
3730         DP(NETIF_MSG_TX_QUEUED,
3731            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3732            txdata->cid, fp_index, txdata_index, txdata, fp); */
3733
3734         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3735                         skb_shinfo(skb)->nr_frags +
3736                         BDS_PER_TX_PKT +
3737                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3738                 /* Handle special storage cases separately */
3739                 if (txdata->tx_ring_size == 0) {
3740                         struct bnx2x_eth_q_stats *q_stats =
3741                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3742                         q_stats->driver_filtered_tx_pkt++;
3743                         dev_kfree_skb(skb);
3744                         return NETDEV_TX_OK;
3745                 }
3746                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3747                 netif_tx_stop_queue(txq);
3748                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3749
3750                 return NETDEV_TX_BUSY;
3751         }
3752
3753         DP(NETIF_MSG_TX_QUEUED,
3754            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3755            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3756            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3757            skb->len);
3758
3759         eth = (struct ethhdr *)skb->data;
3760
3761         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3762         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3763                 if (is_broadcast_ether_addr(eth->h_dest))
3764                         mac_type = BROADCAST_ADDRESS;
3765                 else
3766                         mac_type = MULTICAST_ADDRESS;
3767         }
3768
3769 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3770         /* First, check if we need to linearize the skb (due to FW
3771            restrictions). No need to check fragmentation if page size > 8K
3772            (there will be no violation to FW restrictions) */
3773         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3774                 /* Statistics of linearization */
3775                 bp->lin_cnt++;
3776                 if (skb_linearize(skb) != 0) {
3777                         DP(NETIF_MSG_TX_QUEUED,
3778                            "SKB linearization failed - silently dropping this SKB\n");
3779                         dev_kfree_skb_any(skb);
3780                         return NETDEV_TX_OK;
3781                 }
3782         }
3783 #endif
3784         /* Map skb linear data for DMA */
3785         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3786                                  skb_headlen(skb), DMA_TO_DEVICE);
3787         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3788                 DP(NETIF_MSG_TX_QUEUED,
3789                    "SKB mapping failed - silently dropping this SKB\n");
3790                 dev_kfree_skb_any(skb);
3791                 return NETDEV_TX_OK;
3792         }
3793         /*
3794         Please read carefully. First we use one BD which we mark as start,
3795         then we have a parsing info BD (used for TSO or xsum),
3796         and only then we have the rest of the TSO BDs.
3797         (don't forget to mark the last one as last,
3798         and to unmap only AFTER you write to the BD ...)
3799         And above all, all pdb sizes are in words - NOT DWORDS!
3800         */
3801
3802         /* get current pkt produced now - advance it just before sending packet
3803          * since mapping of pages may fail and cause packet to be dropped
3804          */
3805         pkt_prod = txdata->tx_pkt_prod;
3806         bd_prod = TX_BD(txdata->tx_bd_prod);
3807
3808         /* get a tx_buf and first BD
3809          * tx_start_bd may be changed during SPLIT,
3810          * but first_bd will always stay first
3811          */
3812         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3813         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3814         first_bd = tx_start_bd;
3815
3816         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3817
3818         /* header nbd: indirectly zero other flags! */
3819         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3820
3821         /* remember the first BD of the packet */
3822         tx_buf->first_bd = txdata->tx_bd_prod;
3823         tx_buf->skb = skb;
3824         tx_buf->flags = 0;
3825
3826         DP(NETIF_MSG_TX_QUEUED,
3827            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3828            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3829
3830         if (vlan_tx_tag_present(skb)) {
3831                 tx_start_bd->vlan_or_ethertype =
3832                     cpu_to_le16(vlan_tx_tag_get(skb));
3833                 tx_start_bd->bd_flags.as_bitfield |=
3834                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3835         } else {
3836                 /* when transmitting in a vf, start bd must hold the ethertype
3837                  * for fw to enforce it
3838                  */
3839                 if (IS_VF(bp))
3840                         tx_start_bd->vlan_or_ethertype =
3841                                 cpu_to_le16(ntohs(eth->h_proto));
3842                 else
3843                         /* used by FW for packet accounting */
3844                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3845         }
3846
3847         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3848
3849         /* turn on parsing and get a BD */
3850         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3851
3852         if (xmit_type & XMIT_CSUM)
3853                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3854
3855         if (!CHIP_IS_E1x(bp)) {
3856                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3857                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3858
3859                 if (xmit_type & XMIT_CSUM_ENC) {
3860                         u16 global_data = 0;
3861
3862                         /* Set PBD in enc checksum offload case */
3863                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3864                                                       &pbd_e2_parsing_data,
3865                                                       xmit_type);
3866
3867                         /* turn on 2nd parsing and get a BD */
3868                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3869
3870                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3871
3872                         memset(pbd2, 0, sizeof(*pbd2));
3873
3874                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3875                                 (skb_inner_network_header(skb) -
3876                                  skb->data) >> 1;
3877
3878                         if (xmit_type & XMIT_GSO_ENC)
3879                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3880                                                           &global_data,
3881                                                           xmit_type);
3882
3883                         pbd2->global_data = cpu_to_le16(global_data);
3884
3885                         /* add addition parse BD indication to start BD */
3886                         SET_FLAG(tx_start_bd->general_data,
3887                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3888                         /* set encapsulation flag in start BD */
3889                         SET_FLAG(tx_start_bd->general_data,
3890                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3891                         nbd++;
3892                 } else if (xmit_type & XMIT_CSUM) {
3893                         /* Set PBD in checksum offload case w/o encapsulation */
3894                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3895                                                      &pbd_e2_parsing_data,
3896                                                      xmit_type);
3897                 }
3898
3899                 /* Add the macs to the parsing BD if this is a vf or if
3900                  * Tx Switching is enabled.
3901                  */
3902                 if (IS_VF(bp)) {
3903                         /* override GRE parameters in BD */
3904                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3905                                               &pbd_e2->data.mac_addr.src_mid,
3906                                               &pbd_e2->data.mac_addr.src_lo,
3907                                               eth->h_source);
3908
3909                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3910                                               &pbd_e2->data.mac_addr.dst_mid,
3911                                               &pbd_e2->data.mac_addr.dst_lo,
3912                                               eth->h_dest);
3913                 } else if (bp->flags & TX_SWITCHING) {
3914                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3915                                               &pbd_e2->data.mac_addr.dst_mid,
3916                                               &pbd_e2->data.mac_addr.dst_lo,
3917                                               eth->h_dest);
3918                 }
3919
3920                 SET_FLAG(pbd_e2_parsing_data,
3921                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3922         } else {
3923                 u16 global_data = 0;
3924                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3925                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3926                 /* Set PBD in checksum offload case */
3927                 if (xmit_type & XMIT_CSUM)
3928                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3929
3930                 SET_FLAG(global_data,
3931                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3932                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3933         }
3934
3935         /* Setup the data pointer of the first BD of the packet */
3936         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3937         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3938         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3939         pkt_size = tx_start_bd->nbytes;
3940
3941         DP(NETIF_MSG_TX_QUEUED,
3942            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3943            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3944            le16_to_cpu(tx_start_bd->nbytes),
3945            tx_start_bd->bd_flags.as_bitfield,
3946            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3947
3948         if (xmit_type & XMIT_GSO) {
3949
3950                 DP(NETIF_MSG_TX_QUEUED,
3951                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3952                    skb->len, hlen, skb_headlen(skb),
3953                    skb_shinfo(skb)->gso_size);
3954
3955                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3956
3957                 if (unlikely(skb_headlen(skb) > hlen)) {
3958                         nbd++;
3959                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3960                                                  &tx_start_bd, hlen,
3961                                                  bd_prod);
3962                 }
3963                 if (!CHIP_IS_E1x(bp))
3964                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3965                                              xmit_type);
3966                 else
3967                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3968         }
3969
3970         /* Set the PBD's parsing_data field if not zero
3971          * (for the chips newer than 57711).
3972          */
3973         if (pbd_e2_parsing_data)
3974                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3975
3976         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3977
3978         /* Handle fragmented skb */
3979         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3980                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3981
3982                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3983                                            skb_frag_size(frag), DMA_TO_DEVICE);
3984                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3985                         unsigned int pkts_compl = 0, bytes_compl = 0;
3986
3987                         DP(NETIF_MSG_TX_QUEUED,
3988                            "Unable to map page - dropping packet...\n");
3989
3990                         /* we need unmap all buffers already mapped
3991                          * for this SKB;
3992                          * first_bd->nbd need to be properly updated
3993                          * before call to bnx2x_free_tx_pkt
3994                          */
3995                         first_bd->nbd = cpu_to_le16(nbd);
3996                         bnx2x_free_tx_pkt(bp, txdata,
3997                                           TX_BD(txdata->tx_pkt_prod),
3998                                           &pkts_compl, &bytes_compl);
3999                         return NETDEV_TX_OK;
4000                 }
4001
4002                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4003                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4004                 if (total_pkt_bd == NULL)
4005                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4006
4007                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4008                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4009                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4010                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
4011                 nbd++;
4012
4013                 DP(NETIF_MSG_TX_QUEUED,
4014                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4015                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4016                    le16_to_cpu(tx_data_bd->nbytes));
4017         }
4018
4019         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4020
4021         /* update with actual num BDs */
4022         first_bd->nbd = cpu_to_le16(nbd);
4023
4024         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4025
4026         /* now send a tx doorbell, counting the next BD
4027          * if the packet contains or ends with it
4028          */
4029         if (TX_BD_POFF(bd_prod) < nbd)
4030                 nbd++;
4031
4032         /* total_pkt_bytes should be set on the first data BD if
4033          * it's not an LSO packet and there is more than one
4034          * data BD. In this case pkt_size is limited by an MTU value.
4035          * However we prefer to set it for an LSO packet (while we don't
4036          * have to) in order to save some CPU cycles in a none-LSO
4037          * case, when we much more care about them.
4038          */
4039         if (total_pkt_bd != NULL)
4040                 total_pkt_bd->total_pkt_bytes = pkt_size;
4041
4042         if (pbd_e1x)
4043                 DP(NETIF_MSG_TX_QUEUED,
4044                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4045                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4046                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4047                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4048                     le16_to_cpu(pbd_e1x->total_hlen_w));
4049         if (pbd_e2)
4050                 DP(NETIF_MSG_TX_QUEUED,
4051                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4052                    pbd_e2,
4053                    pbd_e2->data.mac_addr.dst_hi,
4054                    pbd_e2->data.mac_addr.dst_mid,
4055                    pbd_e2->data.mac_addr.dst_lo,
4056                    pbd_e2->data.mac_addr.src_hi,
4057                    pbd_e2->data.mac_addr.src_mid,
4058                    pbd_e2->data.mac_addr.src_lo,
4059                    pbd_e2->parsing_data);
4060         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4061
4062         netdev_tx_sent_queue(txq, skb->len);
4063
4064         skb_tx_timestamp(skb);
4065
4066         txdata->tx_pkt_prod++;
4067         /*
4068          * Make sure that the BD data is updated before updating the producer
4069          * since FW might read the BD right after the producer is updated.
4070          * This is only applicable for weak-ordered memory model archs such
4071          * as IA-64. The following barrier is also mandatory since FW will
4072          * assumes packets must have BDs.
4073          */
4074         wmb();
4075
4076         txdata->tx_db.data.prod += nbd;
4077         barrier();
4078
4079         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4080
4081         mmiowb();
4082
4083         txdata->tx_bd_prod += nbd;
4084
4085         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4086                 netif_tx_stop_queue(txq);
4087
4088                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4089                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4090                  * fp->bd_tx_cons */
4091                 smp_mb();
4092
4093                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4094                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4095                         netif_tx_wake_queue(txq);
4096         }
4097         txdata->tx_pkt++;
4098
4099         return NETDEV_TX_OK;
4100 }
4101
4102 /**
4103  * bnx2x_setup_tc - routine to configure net_device for multi tc
4104  *
4105  * @netdev: net device to configure
4106  * @tc: number of traffic classes to enable
4107  *
4108  * callback connected to the ndo_setup_tc function pointer
4109  */
4110 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4111 {
4112         int cos, prio, count, offset;
4113         struct bnx2x *bp = netdev_priv(dev);
4114
4115         /* setup tc must be called under rtnl lock */
4116         ASSERT_RTNL();
4117
4118         /* no traffic classes requested. Aborting */
4119         if (!num_tc) {
4120                 netdev_reset_tc(dev);
4121                 return 0;
4122         }
4123
4124         /* requested to support too many traffic classes */
4125         if (num_tc > bp->max_cos) {
4126                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4127                           num_tc, bp->max_cos);
4128                 return -EINVAL;
4129         }
4130
4131         /* declare amount of supported traffic classes */
4132         if (netdev_set_num_tc(dev, num_tc)) {
4133                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4134                 return -EINVAL;
4135         }
4136
4137         /* configure priority to traffic class mapping */
4138         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4139                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4140                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4141                    "mapping priority %d to tc %d\n",
4142                    prio, bp->prio_to_cos[prio]);
4143         }
4144
4145         /* Use this configuration to differentiate tc0 from other COSes
4146            This can be used for ets or pfc, and save the effort of setting
4147            up a multio class queue disc or negotiating DCBX with a switch
4148         netdev_set_prio_tc_map(dev, 0, 0);
4149         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4150         for (prio = 1; prio < 16; prio++) {
4151                 netdev_set_prio_tc_map(dev, prio, 1);
4152                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4153         } */
4154
4155         /* configure traffic class to transmission queue mapping */
4156         for (cos = 0; cos < bp->max_cos; cos++) {
4157                 count = BNX2X_NUM_ETH_QUEUES(bp);
4158                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4159                 netdev_set_tc_queue(dev, cos, count, offset);
4160                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4161                    "mapping tc %d to offset %d count %d\n",
4162                    cos, offset, count);
4163         }
4164
4165         return 0;
4166 }
4167
4168 /* called with rtnl_lock */
4169 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4170 {
4171         struct sockaddr *addr = p;
4172         struct bnx2x *bp = netdev_priv(dev);
4173         int rc = 0;
4174
4175         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4176                 BNX2X_ERR("Requested MAC address is not valid\n");
4177                 return -EINVAL;
4178         }
4179
4180         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4181             !is_zero_ether_addr(addr->sa_data)) {
4182                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4183                 return -EINVAL;
4184         }
4185
4186         if (netif_running(dev))  {
4187                 rc = bnx2x_set_eth_mac(bp, false);
4188                 if (rc)
4189                         return rc;
4190         }
4191
4192         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4193
4194         if (netif_running(dev))
4195                 rc = bnx2x_set_eth_mac(bp, true);
4196
4197         return rc;
4198 }
4199
4200 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4201 {
4202         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4203         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4204         u8 cos;
4205
4206         /* Common */
4207
4208         if (IS_FCOE_IDX(fp_index)) {
4209                 memset(sb, 0, sizeof(union host_hc_status_block));
4210                 fp->status_blk_mapping = 0;
4211         } else {
4212                 /* status blocks */
4213                 if (!CHIP_IS_E1x(bp))
4214                         BNX2X_PCI_FREE(sb->e2_sb,
4215                                        bnx2x_fp(bp, fp_index,
4216                                                 status_blk_mapping),
4217                                        sizeof(struct host_hc_status_block_e2));
4218                 else
4219                         BNX2X_PCI_FREE(sb->e1x_sb,
4220                                        bnx2x_fp(bp, fp_index,
4221                                                 status_blk_mapping),
4222                                        sizeof(struct host_hc_status_block_e1x));
4223         }
4224
4225         /* Rx */
4226         if (!skip_rx_queue(bp, fp_index)) {
4227                 bnx2x_free_rx_bds(fp);
4228
4229                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4230                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4231                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4232                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4233                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4234
4235                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4236                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4237                                sizeof(struct eth_fast_path_rx_cqe) *
4238                                NUM_RCQ_BD);
4239
4240                 /* SGE ring */
4241                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4242                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4243                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4244                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4245         }
4246
4247         /* Tx */
4248         if (!skip_tx_queue(bp, fp_index)) {
4249                 /* fastpath tx rings: tx_buf tx_desc */
4250                 for_each_cos_in_tx_queue(fp, cos) {
4251                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4252
4253                         DP(NETIF_MSG_IFDOWN,
4254                            "freeing tx memory of fp %d cos %d cid %d\n",
4255                            fp_index, cos, txdata->cid);
4256
4257                         BNX2X_FREE(txdata->tx_buf_ring);
4258                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4259                                 txdata->tx_desc_mapping,
4260                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4261                 }
4262         }
4263         /* end of fastpath */
4264 }
4265
4266 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4267 {
4268         int i;
4269         for_each_cnic_queue(bp, i)
4270                 bnx2x_free_fp_mem_at(bp, i);
4271 }
4272
4273 void bnx2x_free_fp_mem(struct bnx2x *bp)
4274 {
4275         int i;
4276         for_each_eth_queue(bp, i)
4277                 bnx2x_free_fp_mem_at(bp, i);
4278 }
4279
4280 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4281 {
4282         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4283         if (!CHIP_IS_E1x(bp)) {
4284                 bnx2x_fp(bp, index, sb_index_values) =
4285                         (__le16 *)status_blk.e2_sb->sb.index_values;
4286                 bnx2x_fp(bp, index, sb_running_index) =
4287                         (__le16 *)status_blk.e2_sb->sb.running_index;
4288         } else {
4289                 bnx2x_fp(bp, index, sb_index_values) =
4290                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4291                 bnx2x_fp(bp, index, sb_running_index) =
4292                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4293         }
4294 }
4295
4296 /* Returns the number of actually allocated BDs */
4297 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4298                               int rx_ring_size)
4299 {
4300         struct bnx2x *bp = fp->bp;
4301         u16 ring_prod, cqe_ring_prod;
4302         int i, failure_cnt = 0;
4303
4304         fp->rx_comp_cons = 0;
4305         cqe_ring_prod = ring_prod = 0;
4306
4307         /* This routine is called only during fo init so
4308          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4309          */
4310         for (i = 0; i < rx_ring_size; i++) {
4311                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4312                         failure_cnt++;
4313                         continue;
4314                 }
4315                 ring_prod = NEXT_RX_IDX(ring_prod);
4316                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4317                 WARN_ON(ring_prod <= (i - failure_cnt));
4318         }
4319
4320         if (failure_cnt)
4321                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4322                           i - failure_cnt, fp->index);
4323
4324         fp->rx_bd_prod = ring_prod;
4325         /* Limit the CQE producer by the CQE ring size */
4326         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4327                                cqe_ring_prod);
4328         fp->rx_pkt = fp->rx_calls = 0;
4329
4330         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4331
4332         return i - failure_cnt;
4333 }
4334
4335 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4336 {
4337         int i;
4338
4339         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4340                 struct eth_rx_cqe_next_page *nextpg;
4341
4342                 nextpg = (struct eth_rx_cqe_next_page *)
4343                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4344                 nextpg->addr_hi =
4345                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4346                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4347                 nextpg->addr_lo =
4348                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4349                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4350         }
4351 }
4352
4353 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4354 {
4355         union host_hc_status_block *sb;
4356         struct bnx2x_fastpath *fp = &bp->fp[index];
4357         int ring_size = 0;
4358         u8 cos;
4359         int rx_ring_size = 0;
4360
4361         if (!bp->rx_ring_size &&
4362             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4363                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4364                 bp->rx_ring_size = rx_ring_size;
4365         } else if (!bp->rx_ring_size) {
4366                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4367
4368                 if (CHIP_IS_E3(bp)) {
4369                         u32 cfg = SHMEM_RD(bp,
4370                                            dev_info.port_hw_config[BP_PORT(bp)].
4371                                            default_cfg);
4372
4373                         /* Decrease ring size for 1G functions */
4374                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4375                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4376                                 rx_ring_size /= 10;
4377                 }
4378
4379                 /* allocate at least number of buffers required by FW */
4380                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4381                                      MIN_RX_SIZE_TPA, rx_ring_size);
4382
4383                 bp->rx_ring_size = rx_ring_size;
4384         } else /* if rx_ring_size specified - use it */
4385                 rx_ring_size = bp->rx_ring_size;
4386
4387         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4388
4389         /* Common */
4390         sb = &bnx2x_fp(bp, index, status_blk);
4391
4392         if (!IS_FCOE_IDX(index)) {
4393                 /* status blocks */
4394                 if (!CHIP_IS_E1x(bp)) {
4395                         sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4396                                                     sizeof(struct host_hc_status_block_e2));
4397                         if (!sb->e2_sb)
4398                                 goto alloc_mem_err;
4399                 } else {
4400                         sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4401                                                      sizeof(struct host_hc_status_block_e1x));
4402                         if (!sb->e1x_sb)
4403                                 goto alloc_mem_err;
4404                 }
4405         }
4406
4407         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4408          * set shortcuts for it.
4409          */
4410         if (!IS_FCOE_IDX(index))
4411                 set_sb_shortcuts(bp, index);
4412
4413         /* Tx */
4414         if (!skip_tx_queue(bp, index)) {
4415                 /* fastpath tx rings: tx_buf tx_desc */
4416                 for_each_cos_in_tx_queue(fp, cos) {
4417                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4418
4419                         DP(NETIF_MSG_IFUP,
4420                            "allocating tx memory of fp %d cos %d\n",
4421                            index, cos);
4422
4423                         txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4424                                                       sizeof(struct sw_tx_bd),
4425                                                       GFP_KERNEL);
4426                         if (!txdata->tx_buf_ring)
4427                                 goto alloc_mem_err;
4428                         txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4429                                                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4430                         if (!txdata->tx_desc_ring)
4431                                 goto alloc_mem_err;
4432                 }
4433         }
4434
4435         /* Rx */
4436         if (!skip_rx_queue(bp, index)) {
4437                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4438                 bnx2x_fp(bp, index, rx_buf_ring) =
4439                         kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4440                 if (!bnx2x_fp(bp, index, rx_buf_ring))
4441                         goto alloc_mem_err;
4442                 bnx2x_fp(bp, index, rx_desc_ring) =
4443                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4444                                         sizeof(struct eth_rx_bd) * NUM_RX_BD);
4445                 if (!bnx2x_fp(bp, index, rx_desc_ring))
4446                         goto alloc_mem_err;
4447
4448                 /* Seed all CQEs by 1s */
4449                 bnx2x_fp(bp, index, rx_comp_ring) =
4450                         BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4451                                          sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4452                 if (!bnx2x_fp(bp, index, rx_comp_ring))
4453                         goto alloc_mem_err;
4454
4455                 /* SGE ring */
4456                 bnx2x_fp(bp, index, rx_page_ring) =
4457                         kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4458                                 GFP_KERNEL);
4459                 if (!bnx2x_fp(bp, index, rx_page_ring))
4460                         goto alloc_mem_err;
4461                 bnx2x_fp(bp, index, rx_sge_ring) =
4462                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4463                                         BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4464                 if (!bnx2x_fp(bp, index, rx_sge_ring))
4465                         goto alloc_mem_err;
4466                 /* RX BD ring */
4467                 bnx2x_set_next_page_rx_bd(fp);
4468
4469                 /* CQ ring */
4470                 bnx2x_set_next_page_rx_cq(fp);
4471
4472                 /* BDs */
4473                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4474                 if (ring_size < rx_ring_size)
4475                         goto alloc_mem_err;
4476         }
4477
4478         return 0;
4479
4480 /* handles low memory cases */
4481 alloc_mem_err:
4482         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4483                                                 index, ring_size);
4484         /* FW will drop all packets if queue is not big enough,
4485          * In these cases we disable the queue
4486          * Min size is different for OOO, TPA and non-TPA queues
4487          */
4488         if (ring_size < (fp->disable_tpa ?
4489                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4490                         /* release memory allocated for this queue */
4491                         bnx2x_free_fp_mem_at(bp, index);
4492                         return -ENOMEM;
4493         }
4494         return 0;
4495 }
4496
4497 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4498 {
4499         if (!NO_FCOE(bp))
4500                 /* FCoE */
4501                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4502                         /* we will fail load process instead of mark
4503                          * NO_FCOE_FLAG
4504                          */
4505                         return -ENOMEM;
4506
4507         return 0;
4508 }
4509
4510 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4511 {
4512         int i;
4513
4514         /* 1. Allocate FP for leading - fatal if error
4515          * 2. Allocate RSS - fix number of queues if error
4516          */
4517
4518         /* leading */
4519         if (bnx2x_alloc_fp_mem_at(bp, 0))
4520                 return -ENOMEM;
4521
4522         /* RSS */
4523         for_each_nondefault_eth_queue(bp, i)
4524                 if (bnx2x_alloc_fp_mem_at(bp, i))
4525                         break;
4526
4527         /* handle memory failures */
4528         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4529                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4530
4531                 WARN_ON(delta < 0);
4532                 bnx2x_shrink_eth_fp(bp, delta);
4533                 if (CNIC_SUPPORT(bp))
4534                         /* move non eth FPs next to last eth FP
4535                          * must be done in that order
4536                          * FCOE_IDX < FWD_IDX < OOO_IDX
4537                          */
4538
4539                         /* move FCoE fp even NO_FCOE_FLAG is on */
4540                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4541                 bp->num_ethernet_queues -= delta;
4542                 bp->num_queues = bp->num_ethernet_queues +
4543                                  bp->num_cnic_queues;
4544                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4545                           bp->num_queues + delta, bp->num_queues);
4546         }
4547
4548         return 0;
4549 }
4550
4551 void bnx2x_free_mem_bp(struct bnx2x *bp)
4552 {
4553         int i;
4554
4555         for (i = 0; i < bp->fp_array_size; i++)
4556                 kfree(bp->fp[i].tpa_info);
4557         kfree(bp->fp);
4558         kfree(bp->sp_objs);
4559         kfree(bp->fp_stats);
4560         kfree(bp->bnx2x_txq);
4561         kfree(bp->msix_table);
4562         kfree(bp->ilt);
4563 }
4564
4565 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4566 {
4567         struct bnx2x_fastpath *fp;
4568         struct msix_entry *tbl;
4569         struct bnx2x_ilt *ilt;
4570         int msix_table_size = 0;
4571         int fp_array_size, txq_array_size;
4572         int i;
4573
4574         /*
4575          * The biggest MSI-X table we might need is as a maximum number of fast
4576          * path IGU SBs plus default SB (for PF only).
4577          */
4578         msix_table_size = bp->igu_sb_cnt;
4579         if (IS_PF(bp))
4580                 msix_table_size++;
4581         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4582
4583         /* fp array: RSS plus CNIC related L2 queues */
4584         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4585         bp->fp_array_size = fp_array_size;
4586         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4587
4588         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4589         if (!fp)
4590                 goto alloc_err;
4591         for (i = 0; i < bp->fp_array_size; i++) {
4592                 fp[i].tpa_info =
4593                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4594                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4595                 if (!(fp[i].tpa_info))
4596                         goto alloc_err;
4597         }
4598
4599         bp->fp = fp;
4600
4601         /* allocate sp objs */
4602         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4603                               GFP_KERNEL);
4604         if (!bp->sp_objs)
4605                 goto alloc_err;
4606
4607         /* allocate fp_stats */
4608         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4609                                GFP_KERNEL);
4610         if (!bp->fp_stats)
4611                 goto alloc_err;
4612
4613         /* Allocate memory for the transmission queues array */
4614         txq_array_size =
4615                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4616         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4617
4618         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4619                                 GFP_KERNEL);
4620         if (!bp->bnx2x_txq)
4621                 goto alloc_err;
4622
4623         /* msix table */
4624         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4625         if (!tbl)
4626                 goto alloc_err;
4627         bp->msix_table = tbl;
4628
4629         /* ilt */
4630         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4631         if (!ilt)
4632                 goto alloc_err;
4633         bp->ilt = ilt;
4634
4635         return 0;
4636 alloc_err:
4637         bnx2x_free_mem_bp(bp);
4638         return -ENOMEM;
4639 }
4640
4641 int bnx2x_reload_if_running(struct net_device *dev)
4642 {
4643         struct bnx2x *bp = netdev_priv(dev);
4644
4645         if (unlikely(!netif_running(dev)))
4646                 return 0;
4647
4648         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4649         return bnx2x_nic_load(bp, LOAD_NORMAL);
4650 }
4651
4652 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4653 {
4654         u32 sel_phy_idx = 0;
4655         if (bp->link_params.num_phys <= 1)
4656                 return INT_PHY;
4657
4658         if (bp->link_vars.link_up) {
4659                 sel_phy_idx = EXT_PHY1;
4660                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4661                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4662                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4663                         sel_phy_idx = EXT_PHY2;
4664         } else {
4665
4666                 switch (bnx2x_phy_selection(&bp->link_params)) {
4667                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4668                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4669                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4670                        sel_phy_idx = EXT_PHY1;
4671                        break;
4672                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4673                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4674                        sel_phy_idx = EXT_PHY2;
4675                        break;
4676                 }
4677         }
4678
4679         return sel_phy_idx;
4680 }
4681 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4682 {
4683         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4684         /*
4685          * The selected activated PHY is always after swapping (in case PHY
4686          * swapping is enabled). So when swapping is enabled, we need to reverse
4687          * the configuration
4688          */
4689
4690         if (bp->link_params.multi_phy_config &
4691             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4692                 if (sel_phy_idx == EXT_PHY1)
4693                         sel_phy_idx = EXT_PHY2;
4694                 else if (sel_phy_idx == EXT_PHY2)
4695                         sel_phy_idx = EXT_PHY1;
4696         }
4697         return LINK_CONFIG_IDX(sel_phy_idx);
4698 }
4699
4700 #ifdef NETDEV_FCOE_WWNN
4701 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4702 {
4703         struct bnx2x *bp = netdev_priv(dev);
4704         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4705
4706         switch (type) {
4707         case NETDEV_FCOE_WWNN:
4708                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4709                                 cp->fcoe_wwn_node_name_lo);
4710                 break;
4711         case NETDEV_FCOE_WWPN:
4712                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4713                                 cp->fcoe_wwn_port_name_lo);
4714                 break;
4715         default:
4716                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4717                 return -EINVAL;
4718         }
4719
4720         return 0;
4721 }
4722 #endif
4723
4724 /* called with rtnl_lock */
4725 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4726 {
4727         struct bnx2x *bp = netdev_priv(dev);
4728
4729         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4730                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4731                 return -EAGAIN;
4732         }
4733
4734         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4735             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4736                 BNX2X_ERR("Can't support requested MTU size\n");
4737                 return -EINVAL;
4738         }
4739
4740         /* This does not race with packet allocation
4741          * because the actual alloc size is
4742          * only updated as part of load
4743          */
4744         dev->mtu = new_mtu;
4745
4746         return bnx2x_reload_if_running(dev);
4747 }
4748
4749 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4750                                      netdev_features_t features)
4751 {
4752         struct bnx2x *bp = netdev_priv(dev);
4753
4754         /* TPA requires Rx CSUM offloading */
4755         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4756                 features &= ~NETIF_F_LRO;
4757                 features &= ~NETIF_F_GRO;
4758         }
4759
4760         return features;
4761 }
4762
4763 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4764 {
4765         struct bnx2x *bp = netdev_priv(dev);
4766         u32 flags = bp->flags;
4767         u32 changes;
4768         bool bnx2x_reload = false;
4769
4770         if (features & NETIF_F_LRO)
4771                 flags |= TPA_ENABLE_FLAG;
4772         else
4773                 flags &= ~TPA_ENABLE_FLAG;
4774
4775         if (features & NETIF_F_GRO)
4776                 flags |= GRO_ENABLE_FLAG;
4777         else
4778                 flags &= ~GRO_ENABLE_FLAG;
4779
4780         if (features & NETIF_F_LOOPBACK) {
4781                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4782                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4783                         bnx2x_reload = true;
4784                 }
4785         } else {
4786                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4787                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4788                         bnx2x_reload = true;
4789                 }
4790         }
4791
4792         changes = flags ^ bp->flags;
4793
4794         /* if GRO is changed while LRO is enabled, don't force a reload */
4795         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4796                 changes &= ~GRO_ENABLE_FLAG;
4797
4798         if (changes)
4799                 bnx2x_reload = true;
4800
4801         bp->flags = flags;
4802
4803         if (bnx2x_reload) {
4804                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4805                         return bnx2x_reload_if_running(dev);
4806                 /* else: bnx2x_nic_load() will be called at end of recovery */
4807         }
4808
4809         return 0;
4810 }
4811
4812 void bnx2x_tx_timeout(struct net_device *dev)
4813 {
4814         struct bnx2x *bp = netdev_priv(dev);
4815
4816 #ifdef BNX2X_STOP_ON_ERROR
4817         if (!bp->panic)
4818                 bnx2x_panic();
4819 #endif
4820
4821         /* This allows the netif to be shutdown gracefully before resetting */
4822         bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4823 }
4824
4825 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4826 {
4827         struct net_device *dev = pci_get_drvdata(pdev);
4828         struct bnx2x *bp;
4829
4830         if (!dev) {
4831                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4832                 return -ENODEV;
4833         }
4834         bp = netdev_priv(dev);
4835
4836         rtnl_lock();
4837
4838         pci_save_state(pdev);
4839
4840         if (!netif_running(dev)) {
4841                 rtnl_unlock();
4842                 return 0;
4843         }
4844
4845         netif_device_detach(dev);
4846
4847         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4848
4849         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4850
4851         rtnl_unlock();
4852
4853         return 0;
4854 }
4855
4856 int bnx2x_resume(struct pci_dev *pdev)
4857 {
4858         struct net_device *dev = pci_get_drvdata(pdev);
4859         struct bnx2x *bp;
4860         int rc;
4861
4862         if (!dev) {
4863                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4864                 return -ENODEV;
4865         }
4866         bp = netdev_priv(dev);
4867
4868         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4869                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4870                 return -EAGAIN;
4871         }
4872
4873         rtnl_lock();
4874
4875         pci_restore_state(pdev);
4876
4877         if (!netif_running(dev)) {
4878                 rtnl_unlock();
4879                 return 0;
4880         }
4881
4882         bnx2x_set_power_state(bp, PCI_D0);
4883         netif_device_attach(dev);
4884
4885         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4886
4887         rtnl_unlock();
4888
4889         return rc;
4890 }
4891
4892 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4893                               u32 cid)
4894 {
4895         if (!cxt) {
4896                 BNX2X_ERR("bad context pointer %p\n", cxt);
4897                 return;
4898         }
4899
4900         /* ustorm cxt validation */
4901         cxt->ustorm_ag_context.cdu_usage =
4902                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4903                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4904         /* xcontext validation */
4905         cxt->xstorm_ag_context.cdu_reserved =
4906                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4907                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4908 }
4909
4910 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4911                                     u8 fw_sb_id, u8 sb_index,
4912                                     u8 ticks)
4913 {
4914         u32 addr = BAR_CSTRORM_INTMEM +
4915                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4916         REG_WR8(bp, addr, ticks);
4917         DP(NETIF_MSG_IFUP,
4918            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4919            port, fw_sb_id, sb_index, ticks);
4920 }
4921
4922 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4923                                     u16 fw_sb_id, u8 sb_index,
4924                                     u8 disable)
4925 {
4926         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4927         u32 addr = BAR_CSTRORM_INTMEM +
4928                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4929         u8 flags = REG_RD8(bp, addr);
4930         /* clear and set */
4931         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4932         flags |= enable_flag;
4933         REG_WR8(bp, addr, flags);
4934         DP(NETIF_MSG_IFUP,
4935            "port %x fw_sb_id %d sb_index %d disable %d\n",
4936            port, fw_sb_id, sb_index, disable);
4937 }
4938
4939 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4940                                     u8 sb_index, u8 disable, u16 usec)
4941 {
4942         int port = BP_PORT(bp);
4943         u8 ticks = usec / BNX2X_BTR;
4944
4945         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4946
4947         disable = disable ? 1 : (usec ? 0 : 1);
4948         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4949 }
4950
4951 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
4952                             u32 verbose)
4953 {
4954         smp_mb__before_atomic();
4955         set_bit(flag, &bp->sp_rtnl_state);
4956         smp_mb__after_atomic();
4957         DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
4958            flag);
4959         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4960 }
4961 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);