drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/busy_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
  34 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
  35 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
  36 static int bnx2x_poll(struct napi_struct *napi, int budget);
  37
  38 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
  39 {
  40         int i;
  41
  42         /* Add NAPI objects */
  43         for_each_rx_queue_cnic(bp, i) {
  44                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  45                                bnx2x_poll, NAPI_POLL_WEIGHT);
  46                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  47         }
  48 }
  49
  50 static void bnx2x_add_all_napi(struct bnx2x *bp)
  51 {
  52         int i;
  53
  54         /* Add NAPI objects */
  55         for_each_eth_queue(bp, i) {
  56                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  57                                bnx2x_poll, NAPI_POLL_WEIGHT);
  58                 napi_hash_add(&bnx2x_fp(bp, i, napi));
  59         }
  60 }
  61
  62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
  63 {
  64         int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
  65
  66         /* Reduce memory usage in kdump environment by using only one queue */
  67         if (reset_devices)
  68                 nq = 1;
  69
  70         nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
  71         return nq;
  72 }
  73
  74 /**
  75  * bnx2x_move_fp - move content of the fastpath structure.
  76  *
  77  * @bp:         driver handle
  78  * @from:       source FP index
  79  * @to:         destination FP index
  80  *
  81  * Makes sure the contents of the bp->fp[to].napi is kept
  82  * intact. This is done by first copying the napi struct from
  83  * the target to the source, and then mem copying the entire
  84  * source onto the target. Update txdata pointers and related
  85  * content.
  86  */
  87 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  88 {
  89         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  90         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  91         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  92         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  93         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  94         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  95         int old_max_eth_txqs, new_max_eth_txqs;
  96         int old_txdata_index = 0, new_txdata_index = 0;
  97         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  98
  99         /* Copy the NAPI object as it has been already initialized */
 100         from_fp->napi = to_fp->napi;
 101
 102         /* Move bnx2x_fastpath contents */
 103         memcpy(to_fp, from_fp, sizeof(*to_fp));
 104         to_fp->index = to;
 105
 106         /* Retain the tpa_info of the original `to' version as we don't want
 107          * 2 FPs to contain the same tpa_info pointer.
 108          */
 109         to_fp->tpa_info = old_tpa_info;
 110
 111         /* move sp_objs contents as well, as their indices match fp ones */
 112         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
 113
 114         /* move fp_stats contents as well, as their indices match fp ones */
 115         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
 116
 117         /* Update txdata pointers in fp and move txdata content accordingly:
 118          * Each fp consumes 'max_cos' txdata structures, so the index should be
 119          * decremented by max_cos x delta.
 120          */
 121
 122         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
 123         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
 124                                 (bp)->max_cos;
 125         if (from == FCOE_IDX(bp)) {
 126                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 127                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 128         }
 129
 130         memcpy(&bp->bnx2x_txq[new_txdata_index],
 131                &bp->bnx2x_txq[old_txdata_index],
 132                sizeof(struct bnx2x_fp_txdata));
 133         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
 134 }
 135
 136 /**
 137  * bnx2x_fill_fw_str - Fill buffer with FW version string.
 138  *
 139  * @bp:        driver handle
 140  * @buf:       character buffer to fill with the fw name
 141  * @buf_len:   length of the above buffer
 142  *
 143  */
 144 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 145 {
 146         if (IS_PF(bp)) {
 147                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 148
 149                 phy_fw_ver[0] = '\0';
 150                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 151                                              phy_fw_ver, PHY_FW_VER_LEN);
 152                 strlcpy(buf, bp->fw_ver, buf_len);
 153                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 154                          "bc %d.%d.%d%s%s",
 155                          (bp->common.bc_ver & 0xff0000) >> 16,
 156                          (bp->common.bc_ver & 0xff00) >> 8,
 157                          (bp->common.bc_ver & 0xff),
 158                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 159         } else {
 160                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 161         }
 162 }
 163
 164 /**
 165  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 166  *
 167  * @bp: driver handle
 168  * @delta:      number of eth queues which were not allocated
 169  */
 170 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 171 {
 172         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 173
 174         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 175          * backward along the array could cause memory to be overridden
 176          */
 177         for (cos = 1; cos < bp->max_cos; cos++) {
 178                 for (i = 0; i < old_eth_num - delta; i++) {
 179                         struct bnx2x_fastpath *fp = &bp->fp[i];
 180                         int new_idx = cos * (old_eth_num - delta) + i;
 181
 182                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 183                                sizeof(struct bnx2x_fp_txdata));
 184                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 185                 }
 186         }
 187 }
 188
 189 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 190
 191 /* free skb in the packet ring at pos idx
 192  * return idx of last bd freed
 193  */
 194 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 195                              u16 idx, unsigned int *pkts_compl,
 196                              unsigned int *bytes_compl)
 197 {
 198         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 199         struct eth_tx_start_bd *tx_start_bd;
 200         struct eth_tx_bd *tx_data_bd;
 201         struct sk_buff *skb = tx_buf->skb;
 202         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 203         int nbd;
 204         u16 split_bd_len = 0;
 205
 206         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 207         prefetch(&skb->end);
 208
 209         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 210            txdata->txq_index, idx, tx_buf, skb);
 211
 212         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 213
 214         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 215 #ifdef BNX2X_STOP_ON_ERROR
 216         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 217                 BNX2X_ERR("BAD nbd!\n");
 218                 bnx2x_panic();
 219         }
 220 #endif
 221         new_cons = nbd + tx_buf->first_bd;
 222
 223         /* Get the next bd */
 224         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 225
 226         /* Skip a parse bd... */
 227         --nbd;
 228         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 229
 230         if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
 231                 /* Skip second parse bd... */
 232                 --nbd;
 233                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 234         }
 235
 236         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 237         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 238                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 239                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 240                 --nbd;
 241                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 242         }
 243
 244         /* unmap first bd */
 245         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 246                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 247                          DMA_TO_DEVICE);
 248
 249         /* now free frags */
 250         while (nbd > 0) {
 251
 252                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 253                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 254                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 255                 if (--nbd)
 256                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 257         }
 258
 259         /* release skb */
 260         WARN_ON(!skb);
 261         if (likely(skb)) {
 262                 (*pkts_compl)++;
 263                 (*bytes_compl) += skb->len;
 264         }
 265
 266         dev_kfree_skb_any(skb);
 267         tx_buf->first_bd = 0;
 268         tx_buf->skb = NULL;
 269
 270         return new_cons;
 271 }
 272
 273 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 274 {
 275         struct netdev_queue *txq;
 276         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 277         unsigned int pkts_compl = 0, bytes_compl = 0;
 278
 279 #ifdef BNX2X_STOP_ON_ERROR
 280         if (unlikely(bp->panic))
 281                 return -1;
 282 #endif
 283
 284         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 285         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 286         sw_cons = txdata->tx_pkt_cons;
 287
 288         while (sw_cons != hw_cons) {
 289                 u16 pkt_cons;
 290
 291                 pkt_cons = TX_BD(sw_cons);
 292
 293                 DP(NETIF_MSG_TX_DONE,
 294                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 295                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 296
 297                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 298                                             &pkts_compl, &bytes_compl);
 299
 300                 sw_cons++;
 301         }
 302
 303         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 304
 305         txdata->tx_pkt_cons = sw_cons;
 306         txdata->tx_bd_cons = bd_cons;
 307
 308         /* Need to make the tx_bd_cons update visible to start_xmit()
 309          * before checking for netif_tx_queue_stopped().  Without the
 310          * memory barrier, there is a small possibility that
 311          * start_xmit() will miss it and cause the queue to be stopped
 312          * forever.
 313          * On the other hand we need an rmb() here to ensure the proper
 314          * ordering of bit testing in the following
 315          * netif_tx_queue_stopped(txq) call.
 316          */
 317         smp_mb();
 318
 319         if (unlikely(netif_tx_queue_stopped(txq))) {
 320                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 321                  * while it's empty. This could have happen if rx_action() gets
 322                  * suspended in bnx2x_tx_int() after the condition before
 323                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 324                  *
 325                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 326                  * sends some packets consuming the whole queue again->
 327                  * stops the queue
 328                  */
 329
 330                 __netif_tx_lock(txq, smp_processor_id());
 331
 332                 if ((netif_tx_queue_stopped(txq)) &&
 333                     (bp->state == BNX2X_STATE_OPEN) &&
 334                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 335                         netif_tx_wake_queue(txq);
 336
 337                 __netif_tx_unlock(txq);
 338         }
 339         return 0;
 340 }
 341
 342 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 343                                              u16 idx)
 344 {
 345         u16 last_max = fp->last_max_sge;
 346
 347         if (SUB_S16(idx, last_max) > 0)
 348                 fp->last_max_sge = idx;
 349 }
 350
 351 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 352                                          u16 sge_len,
 353                                          struct eth_end_agg_rx_cqe *cqe)
 354 {
 355         struct bnx2x *bp = fp->bp;
 356         u16 last_max, last_elem, first_elem;
 357         u16 delta = 0;
 358         u16 i;
 359
 360         if (!sge_len)
 361                 return;
 362
 363         /* First mark all used pages */
 364         for (i = 0; i < sge_len; i++)
 365                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 366                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 367
 368         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 369            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 370
 371         /* Here we assume that the last SGE index is the biggest */
 372         prefetch((void *)(fp->sge_mask));
 373         bnx2x_update_last_max_sge(fp,
 374                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 375
 376         last_max = RX_SGE(fp->last_max_sge);
 377         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 378         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 379
 380         /* If ring is not full */
 381         if (last_elem + 1 != first_elem)
 382                 last_elem++;
 383
 384         /* Now update the prod */
 385         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 386                 if (likely(fp->sge_mask[i]))
 387                         break;
 388
 389                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 390                 delta += BIT_VEC64_ELEM_SZ;
 391         }
 392
 393         if (delta > 0) {
 394                 fp->rx_sge_prod += delta;
 395                 /* clear page-end entries */
 396                 bnx2x_clear_sge_mask_next_elems(fp);
 397         }
 398
 399         DP(NETIF_MSG_RX_STATUS,
 400            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 401            fp->last_max_sge, fp->rx_sge_prod);
 402 }
 403
 404 /* Get Toeplitz hash value in the skb using the value from the
 405  * CQE (calculated by HW).
 406  */
 407 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 408                             const struct eth_fast_path_rx_cqe *cqe,
 409                             enum pkt_hash_types *rxhash_type)
 410 {
 411         /* Get Toeplitz hash from CQE */
 412         if ((bp->dev->features & NETIF_F_RXHASH) &&
 413             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 414                 enum eth_rss_hash_type htype;
 415
 416                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 417                 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
 418                                 (htype == TCP_IPV6_HASH_TYPE)) ?
 419                                PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
 420
 421                 return le32_to_cpu(cqe->rss_hash_result);
 422         }
 423         *rxhash_type = PKT_HASH_TYPE_NONE;
 424         return 0;
 425 }
 426
 427 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 428                             u16 cons, u16 prod,
 429                             struct eth_fast_path_rx_cqe *cqe)
 430 {
 431         struct bnx2x *bp = fp->bp;
 432         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 433         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 434         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 435         dma_addr_t mapping;
 436         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 437         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 438
 439         /* print error if current state != stop */
 440         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 441                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 442
 443         /* Try to map an empty data buffer from the aggregation info  */
 444         mapping = dma_map_single(&bp->pdev->dev,
 445                                  first_buf->data + NET_SKB_PAD,
 446                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 447         /*
 448          *  ...if it fails - move the skb from the consumer to the producer
 449          *  and set the current aggregation state as ERROR to drop it
 450          *  when TPA_STOP arrives.
 451          */
 452
 453         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 454                 /* Move the BD from the consumer to the producer */
 455                 bnx2x_reuse_rx_data(fp, cons, prod);
 456                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 457                 return;
 458         }
 459
 460         /* move empty data from pool to prod */
 461         prod_rx_buf->data = first_buf->data;
 462         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 463         /* point prod_bd to new data */
 464         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 465         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 466
 467         /* move partial skb from cons to pool (don't unmap yet) */
 468         *first_buf = *cons_rx_buf;
 469
 470         /* mark bin state as START */
 471         tpa_info->parsing_flags =
 472                 le16_to_cpu(cqe->pars_flags.flags);
 473         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 474         tpa_info->tpa_state = BNX2X_TPA_START;
 475         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 476         tpa_info->placement_offset = cqe->placement_offset;
 477         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
 478         if (fp->mode == TPA_MODE_GRO) {
 479                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 480                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 481                 tpa_info->gro_size = gro_size;
 482         }
 483
 484 #ifdef BNX2X_STOP_ON_ERROR
 485         fp->tpa_queue_used |= (1 << queue);
 486         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 487            fp->tpa_queue_used);
 488 #endif
 489 }
 490
 491 /* Timestamp option length allowed for TPA aggregation:
 492  *
 493  *              nop nop kind length echo val
 494  */
 495 #define TPA_TSTAMP_OPT_LEN      12
 496 /**
 497  * bnx2x_set_gro_params - compute GRO values
 498  *
 499  * @skb:                packet skb
 500  * @parsing_flags:      parsing flags from the START CQE
 501  * @len_on_bd:          total length of the first packet for the
 502  *                      aggregation.
 503  * @pkt_len:            length of all segments
 504  *
 505  * Approximate value of the MSS for this aggregation calculated using
 506  * the first packet of it.
 507  * Compute number of aggregated segments, and gso_type.
 508  */
 509 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 510                                  u16 len_on_bd, unsigned int pkt_len,
 511                                  u16 num_of_coalesced_segs)
 512 {
 513         /* TPA aggregation won't have either IP options or TCP options
 514          * other than timestamp or IPv6 extension headers.
 515          */
 516         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 517
 518         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 519             PRS_FLAG_OVERETH_IPV6) {
 520                 hdrs_len += sizeof(struct ipv6hdr);
 521                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 522         } else {
 523                 hdrs_len += sizeof(struct iphdr);
 524                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 525         }
 526
 527         /* Check if there was a TCP timestamp, if there is it's will
 528          * always be 12 bytes length: nop nop kind length echo val.
 529          *
 530          * Otherwise FW would close the aggregation.
 531          */
 532         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 533                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 534
 535         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 536
 537         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 538          * to skb_shinfo(skb)->gso_segs
 539          */
 540         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 541 }
 542
 543 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 544                               u16 index, gfp_t gfp_mask)
 545 {
 546         struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 547         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 548         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 549         dma_addr_t mapping;
 550
 551         if (unlikely(page == NULL)) {
 552                 BNX2X_ERR("Can't alloc sge\n");
 553                 return -ENOMEM;
 554         }
 555
 556         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 557                                SGE_PAGES, DMA_FROM_DEVICE);
 558         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 559                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 560                 BNX2X_ERR("Can't map sge\n");
 561                 return -ENOMEM;
 562         }
 563
 564         sw_buf->page = page;
 565         dma_unmap_addr_set(sw_buf, mapping, mapping);
 566
 567         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 568         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 569
 570         return 0;
 571 }
 572
 573 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 574                                struct bnx2x_agg_info *tpa_info,
 575                                u16 pages,
 576                                struct sk_buff *skb,
 577                                struct eth_end_agg_rx_cqe *cqe,
 578                                u16 cqe_idx)
 579 {
 580         struct sw_rx_page *rx_pg, old_rx_pg;
 581         u32 i, frag_len, frag_size;
 582         int err, j, frag_id = 0;
 583         u16 len_on_bd = tpa_info->len_on_bd;
 584         u16 full_page = 0, gro_size = 0;
 585
 586         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 587
 588         if (fp->mode == TPA_MODE_GRO) {
 589                 gro_size = tpa_info->gro_size;
 590                 full_page = tpa_info->full_page;
 591         }
 592
 593         /* This is needed in order to enable forwarding support */
 594         if (frag_size)
 595                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 596                                      le16_to_cpu(cqe->pkt_len),
 597                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 598
 599 #ifdef BNX2X_STOP_ON_ERROR
 600         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 601                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 602                           pages, cqe_idx);
 603                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 604                 bnx2x_panic();
 605                 return -EINVAL;
 606         }
 607 #endif
 608
 609         /* Run through the SGL and compose the fragmented skb */
 610         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 611                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 612
 613                 /* FW gives the indices of the SGE as if the ring is an array
 614                    (meaning that "next" element will consume 2 indices) */
 615                 if (fp->mode == TPA_MODE_GRO)
 616                         frag_len = min_t(u32, frag_size, (u32)full_page);
 617                 else /* LRO */
 618                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 619
 620                 rx_pg = &fp->rx_page_ring[sge_idx];
 621                 old_rx_pg = *rx_pg;
 622
 623                 /* If we fail to allocate a substitute page, we simply stop
 624                    where we are and drop the whole packet */
 625                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 626                 if (unlikely(err)) {
 627                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 628                         return err;
 629                 }
 630
 631                 /* Unmap the page as we're going to pass it to the stack */
 632                 dma_unmap_page(&bp->pdev->dev,
 633                                dma_unmap_addr(&old_rx_pg, mapping),
 634                                SGE_PAGES, DMA_FROM_DEVICE);
 635                 /* Add one frag and update the appropriate fields in the skb */
 636                 if (fp->mode == TPA_MODE_LRO)
 637                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 638                 else { /* GRO */
 639                         int rem;
 640                         int offset = 0;
 641                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 642                                 int len = rem > gro_size ? gro_size : rem;
 643                                 skb_fill_page_desc(skb, frag_id++,
 644                                                    old_rx_pg.page, offset, len);
 645                                 if (offset)
 646                                         get_page(old_rx_pg.page);
 647                                 offset += len;
 648                         }
 649                 }
 650
 651                 skb->data_len += frag_len;
 652                 skb->truesize += SGE_PAGES;
 653                 skb->len += frag_len;
 654
 655                 frag_size -= frag_len;
 656         }
 657
 658         return 0;
 659 }
 660
 661 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 662 {
 663         if (fp->rx_frag_size)
 664                 put_page(virt_to_head_page(data));
 665         else
 666                 kfree(data);
 667 }
 668
 669 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 670 {
 671         if (fp->rx_frag_size) {
 672                 /* GFP_KERNEL allocations are used only during initialization */
 673                 if (unlikely(gfp_mask & __GFP_WAIT))
 674                         return (void *)__get_free_page(gfp_mask);
 675
 676                 return netdev_alloc_frag(fp->rx_frag_size);
 677         }
 678
 679         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 680 }
 681
 682 #ifdef CONFIG_INET
 683 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 684 {
 685         const struct iphdr *iph = ip_hdr(skb);
 686         struct tcphdr *th;
 687
 688         skb_set_transport_header(skb, sizeof(struct iphdr));
 689         th = tcp_hdr(skb);
 690
 691         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 692                                   iph->saddr, iph->daddr, 0);
 693 }
 694
 695 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 696 {
 697         struct ipv6hdr *iph = ipv6_hdr(skb);
 698         struct tcphdr *th;
 699
 700         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 701         th = tcp_hdr(skb);
 702
 703         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 704                                   &iph->saddr, &iph->daddr, 0);
 705 }
 706
 707 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 708                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 709 {
 710         skb_set_network_header(skb, 0);
 711         gro_func(bp, skb);
 712         tcp_gro_complete(skb);
 713 }
 714 #endif
 715
 716 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 717                                struct sk_buff *skb)
 718 {
 719 #ifdef CONFIG_INET
 720         if (skb_shinfo(skb)->gso_size) {
 721                 switch (be16_to_cpu(skb->protocol)) {
 722                 case ETH_P_IP:
 723                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 724                         break;
 725                 case ETH_P_IPV6:
 726                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 727                         break;
 728                 default:
 729                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 730                                   be16_to_cpu(skb->protocol));
 731                 }
 732         }
 733 #endif
 734         skb_record_rx_queue(skb, fp->rx_queue);
 735         napi_gro_receive(&fp->napi, skb);
 736 }
 737
 738 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 739                            struct bnx2x_agg_info *tpa_info,
 740                            u16 pages,
 741                            struct eth_end_agg_rx_cqe *cqe,
 742                            u16 cqe_idx)
 743 {
 744         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 745         u8 pad = tpa_info->placement_offset;
 746         u16 len = tpa_info->len_on_bd;
 747         struct sk_buff *skb = NULL;
 748         u8 *new_data, *data = rx_buf->data;
 749         u8 old_tpa_state = tpa_info->tpa_state;
 750
 751         tpa_info->tpa_state = BNX2X_TPA_STOP;
 752
 753         /* If we there was an error during the handling of the TPA_START -
 754          * drop this aggregation.
 755          */
 756         if (old_tpa_state == BNX2X_TPA_ERROR)
 757                 goto drop;
 758
 759         /* Try to allocate the new data */
 760         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 761         /* Unmap skb in the pool anyway, as we are going to change
 762            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 763            fails. */
 764         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 765                          fp->rx_buf_size, DMA_FROM_DEVICE);
 766         if (likely(new_data))
 767                 skb = build_skb(data, fp->rx_frag_size);
 768
 769         if (likely(skb)) {
 770 #ifdef BNX2X_STOP_ON_ERROR
 771                 if (pad + len > fp->rx_buf_size) {
 772                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 773                                   pad, len, fp->rx_buf_size);
 774                         bnx2x_panic();
 775                         return;
 776                 }
 777 #endif
 778
 779                 skb_reserve(skb, pad + NET_SKB_PAD);
 780                 skb_put(skb, len);
 781                 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
 782
 783                 skb->protocol = eth_type_trans(skb, bp->dev);
 784                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 785
 786                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 787                                          skb, cqe, cqe_idx)) {
 788                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 789                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 790                         bnx2x_gro_receive(bp, fp, skb);
 791                 } else {
 792                         DP(NETIF_MSG_RX_STATUS,
 793                            "Failed to allocate new pages - dropping packet!\n");
 794                         dev_kfree_skb_any(skb);
 795                 }
 796
 797                 /* put new data in bin */
 798                 rx_buf->data = new_data;
 799
 800                 return;
 801         }
 802         if (new_data)
 803                 bnx2x_frag_free(fp, new_data);
 804 drop:
 805         /* drop the packet and keep the buffer in the bin */
 806         DP(NETIF_MSG_RX_STATUS,
 807            "Failed to allocate or map a new skb - dropping packet!\n");
 808         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 809 }
 810
 811 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 812                                u16 index, gfp_t gfp_mask)
 813 {
 814         u8 *data;
 815         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 816         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 817         dma_addr_t mapping;
 818
 819         data = bnx2x_frag_alloc(fp, gfp_mask);
 820         if (unlikely(data == NULL))
 821                 return -ENOMEM;
 822
 823         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 824                                  fp->rx_buf_size,
 825                                  DMA_FROM_DEVICE);
 826         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 827                 bnx2x_frag_free(fp, data);
 828                 BNX2X_ERR("Can't map rx data\n");
 829                 return -ENOMEM;
 830         }
 831
 832         rx_buf->data = data;
 833         dma_unmap_addr_set(rx_buf, mapping, mapping);
 834
 835         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 836         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 837
 838         return 0;
 839 }
 840
 841 static
 842 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 843                                  struct bnx2x_fastpath *fp,
 844                                  struct bnx2x_eth_q_stats *qstats)
 845 {
 846         /* Do nothing if no L4 csum validation was done.
 847          * We do not check whether IP csum was validated. For IPv4 we assume
 848          * that if the card got as far as validating the L4 csum, it also
 849          * validated the IP csum. IPv6 has no IP csum.
 850          */
 851         if (cqe->fast_path_cqe.status_flags &
 852             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 853                 return;
 854
 855         /* If L4 validation was done, check if an error was found. */
 856
 857         if (cqe->fast_path_cqe.type_error_flags &
 858             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 859              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 860                 qstats->hw_csum_err++;
 861         else
 862                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 863 }
 864
 865 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 866 {
 867         struct bnx2x *bp = fp->bp;
 868         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 869         u16 sw_comp_cons, sw_comp_prod;
 870         int rx_pkt = 0;
 871         union eth_rx_cqe *cqe;
 872         struct eth_fast_path_rx_cqe *cqe_fp;
 873
 874 #ifdef BNX2X_STOP_ON_ERROR
 875         if (unlikely(bp->panic))
 876                 return 0;
 877 #endif
 878         if (budget <= 0)
 879                 return rx_pkt;
 880
 881         bd_cons = fp->rx_bd_cons;
 882         bd_prod = fp->rx_bd_prod;
 883         bd_prod_fw = bd_prod;
 884         sw_comp_cons = fp->rx_comp_cons;
 885         sw_comp_prod = fp->rx_comp_prod;
 886
 887         comp_ring_cons = RCQ_BD(sw_comp_cons);
 888         cqe = &fp->rx_comp_ring[comp_ring_cons];
 889         cqe_fp = &cqe->fast_path_cqe;
 890
 891         DP(NETIF_MSG_RX_STATUS,
 892            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 893
 894         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 895                 struct sw_rx_bd *rx_buf = NULL;
 896                 struct sk_buff *skb;
 897                 u8 cqe_fp_flags;
 898                 enum eth_rx_cqe_type cqe_fp_type;
 899                 u16 len, pad, queue;
 900                 u8 *data;
 901                 u32 rxhash;
 902                 enum pkt_hash_types rxhash_type;
 903
 904 #ifdef BNX2X_STOP_ON_ERROR
 905                 if (unlikely(bp->panic))
 906                         return 0;
 907 #endif
 908
 909                 bd_prod = RX_BD(bd_prod);
 910                 bd_cons = RX_BD(bd_cons);
 911
 912                 /* A rmb() is required to ensure that the CQE is not read
 913                  * before it is written by the adapter DMA.  PCI ordering
 914                  * rules will make sure the other fields are written before
 915                  * the marker at the end of struct eth_fast_path_rx_cqe
 916                  * but without rmb() a weakly ordered processor can process
 917                  * stale data.  Without the barrier TPA state-machine might
 918                  * enter inconsistent state and kernel stack might be
 919                  * provided with incorrect packet description - these lead
 920                  * to various kernel crashed.
 921                  */
 922                 rmb();
 923
 924                 cqe_fp_flags = cqe_fp->type_error_flags;
 925                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 926
 927                 DP(NETIF_MSG_RX_STATUS,
 928                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 929                    CQE_TYPE(cqe_fp_flags),
 930                    cqe_fp_flags, cqe_fp->status_flags,
 931                    le32_to_cpu(cqe_fp->rss_hash_result),
 932                    le16_to_cpu(cqe_fp->vlan_tag),
 933                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 934
 935                 /* is this a slowpath msg? */
 936                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 937                         bnx2x_sp_event(fp, cqe);
 938                         goto next_cqe;
 939                 }
 940
 941                 rx_buf = &fp->rx_buf_ring[bd_cons];
 942                 data = rx_buf->data;
 943
 944                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 945                         struct bnx2x_agg_info *tpa_info;
 946                         u16 frag_size, pages;
 947 #ifdef BNX2X_STOP_ON_ERROR
 948                         /* sanity check */
 949                         if (fp->disable_tpa &&
 950                             (CQE_TYPE_START(cqe_fp_type) ||
 951                              CQE_TYPE_STOP(cqe_fp_type)))
 952                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 953                                           CQE_TYPE(cqe_fp_type));
 954 #endif
 955
 956                         if (CQE_TYPE_START(cqe_fp_type)) {
 957                                 u16 queue = cqe_fp->queue_index;
 958                                 DP(NETIF_MSG_RX_STATUS,
 959                                    "calling tpa_start on queue %d\n",
 960                                    queue);
 961
 962                                 bnx2x_tpa_start(fp, queue,
 963                                                 bd_cons, bd_prod,
 964                                                 cqe_fp);
 965
 966                                 goto next_rx;
 967                         }
 968                         queue = cqe->end_agg_cqe.queue_index;
 969                         tpa_info = &fp->tpa_info[queue];
 970                         DP(NETIF_MSG_RX_STATUS,
 971                            "calling tpa_stop on queue %d\n",
 972                            queue);
 973
 974                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 975                                     tpa_info->len_on_bd;
 976
 977                         if (fp->mode == TPA_MODE_GRO)
 978                                 pages = (frag_size + tpa_info->full_page - 1) /
 979                                          tpa_info->full_page;
 980                         else
 981                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 982                                         SGE_PAGE_SHIFT;
 983
 984                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 985                                        &cqe->end_agg_cqe, comp_ring_cons);
 986 #ifdef BNX2X_STOP_ON_ERROR
 987                         if (bp->panic)
 988                                 return 0;
 989 #endif
 990
 991                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 992                         goto next_cqe;
 993                 }
 994                 /* non TPA */
 995                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 996                 pad = cqe_fp->placement_offset;
 997                 dma_sync_single_for_cpu(&bp->pdev->dev,
 998                                         dma_unmap_addr(rx_buf, mapping),
 999                                         pad + RX_COPY_THRESH,
1000                                         DMA_FROM_DEVICE);
1001                 pad += NET_SKB_PAD;
1002                 prefetch(data + pad); /* speedup eth_type_trans() */
1003                 /* is this an error packet? */
1004                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1005                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1006                            "ERROR  flags %x  rx packet %u\n",
1007                            cqe_fp_flags, sw_comp_cons);
1008                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1009                         goto reuse_rx;
1010                 }
1011
1012                 /* Since we don't have a jumbo ring
1013                  * copy small packets if mtu > 1500
1014                  */
1015                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1016                     (len <= RX_COPY_THRESH)) {
1017                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
1018                         if (skb == NULL) {
1019                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1020                                    "ERROR  packet dropped because of alloc failure\n");
1021                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1022                                 goto reuse_rx;
1023                         }
1024                         memcpy(skb->data, data + pad, len);
1025                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1026                 } else {
1027                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1028                                                        GFP_ATOMIC) == 0)) {
1029                                 dma_unmap_single(&bp->pdev->dev,
1030                                                  dma_unmap_addr(rx_buf, mapping),
1031                                                  fp->rx_buf_size,
1032                                                  DMA_FROM_DEVICE);
1033                                 skb = build_skb(data, fp->rx_frag_size);
1034                                 if (unlikely(!skb)) {
1035                                         bnx2x_frag_free(fp, data);
1036                                         bnx2x_fp_qstats(bp, fp)->
1037                                                         rx_skb_alloc_failed++;
1038                                         goto next_rx;
1039                                 }
1040                                 skb_reserve(skb, pad);
1041                         } else {
1042                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1043                                    "ERROR  packet dropped because of alloc failure\n");
1044                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1045 reuse_rx:
1046                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1047                                 goto next_rx;
1048                         }
1049                 }
1050
1051                 skb_put(skb, len);
1052                 skb->protocol = eth_type_trans(skb, bp->dev);
1053
1054                 /* Set Toeplitz hash for a none-LRO skb */
1055                 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1056                 skb_set_hash(skb, rxhash, rxhash_type);
1057
1058                 skb_checksum_none_assert(skb);
1059
1060                 if (bp->dev->features & NETIF_F_RXCSUM)
1061                         bnx2x_csum_validate(skb, cqe, fp,
1062                                             bnx2x_fp_qstats(bp, fp));
1063
1064                 skb_record_rx_queue(skb, fp->rx_queue);
1065
1066                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1067                     PARSING_FLAGS_VLAN)
1068                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1069                                                le16_to_cpu(cqe_fp->vlan_tag));
1070
1071                 skb_mark_napi_id(skb, &fp->napi);
1072
1073                 if (bnx2x_fp_ll_polling(fp))
1074                         netif_receive_skb(skb);
1075                 else
1076                         napi_gro_receive(&fp->napi, skb);
1077 next_rx:
1078                 rx_buf->data = NULL;
1079
1080                 bd_cons = NEXT_RX_IDX(bd_cons);
1081                 bd_prod = NEXT_RX_IDX(bd_prod);
1082                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1083                 rx_pkt++;
1084 next_cqe:
1085                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1086                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1087
1088                 /* mark CQE as free */
1089                 BNX2X_SEED_CQE(cqe_fp);
1090
1091                 if (rx_pkt == budget)
1092                         break;
1093
1094                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1095                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1096                 cqe_fp = &cqe->fast_path_cqe;
1097         } /* while */
1098
1099         fp->rx_bd_cons = bd_cons;
1100         fp->rx_bd_prod = bd_prod_fw;
1101         fp->rx_comp_cons = sw_comp_cons;
1102         fp->rx_comp_prod = sw_comp_prod;
1103
1104         /* Update producers */
1105         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1106                              fp->rx_sge_prod);
1107
1108         fp->rx_pkt += rx_pkt;
1109         fp->rx_calls++;
1110
1111         return rx_pkt;
1112 }
1113
1114 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1115 {
1116         struct bnx2x_fastpath *fp = fp_cookie;
1117         struct bnx2x *bp = fp->bp;
1118         u8 cos;
1119
1120         DP(NETIF_MSG_INTR,
1121            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1122            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1123
1124         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1125
1126 #ifdef BNX2X_STOP_ON_ERROR
1127         if (unlikely(bp->panic))
1128                 return IRQ_HANDLED;
1129 #endif
1130
1131         /* Handle Rx and Tx according to MSI-X vector */
1132         for_each_cos_in_tx_queue(fp, cos)
1133                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1134
1135         prefetch(&fp->sb_running_index[SM_RX_ID]);
1136         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1137
1138         return IRQ_HANDLED;
1139 }
1140
1141 /* HW Lock for shared dual port PHYs */
1142 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1143 {
1144         mutex_lock(&bp->port.phy_mutex);
1145
1146         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1147 }
1148
1149 void bnx2x_release_phy_lock(struct bnx2x *bp)
1150 {
1151         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1152
1153         mutex_unlock(&bp->port.phy_mutex);
1154 }
1155
1156 /* calculates MF speed according to current linespeed and MF configuration */
1157 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1158 {
1159         u16 line_speed = bp->link_vars.line_speed;
1160         if (IS_MF(bp)) {
1161                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1162                                                    bp->mf_config[BP_VN(bp)]);
1163
1164                 /* Calculate the current MAX line speed limit for the MF
1165                  * devices
1166                  */
1167                 if (IS_MF_SI(bp))
1168                         line_speed = (line_speed * maxCfg) / 100;
1169                 else { /* SD mode */
1170                         u16 vn_max_rate = maxCfg * 100;
1171
1172                         if (vn_max_rate < line_speed)
1173                                 line_speed = vn_max_rate;
1174                 }
1175         }
1176
1177         return line_speed;
1178 }
1179
1180 /**
1181  * bnx2x_fill_report_data - fill link report data to report
1182  *
1183  * @bp:         driver handle
1184  * @data:       link state to update
1185  *
1186  * It uses a none-atomic bit operations because is called under the mutex.
1187  */
1188 static void bnx2x_fill_report_data(struct bnx2x *bp,
1189                                    struct bnx2x_link_report_data *data)
1190 {
1191         memset(data, 0, sizeof(*data));
1192
1193         if (IS_PF(bp)) {
1194                 /* Fill the report data: effective line speed */
1195                 data->line_speed = bnx2x_get_mf_speed(bp);
1196
1197                 /* Link is down */
1198                 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1199                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1200                                   &data->link_report_flags);
1201
1202                 if (!BNX2X_NUM_ETH_QUEUES(bp))
1203                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1204                                   &data->link_report_flags);
1205
1206                 /* Full DUPLEX */
1207                 if (bp->link_vars.duplex == DUPLEX_FULL)
1208                         __set_bit(BNX2X_LINK_REPORT_FD,
1209                                   &data->link_report_flags);
1210
1211                 /* Rx Flow Control is ON */
1212                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1213                         __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1214                                   &data->link_report_flags);
1215
1216                 /* Tx Flow Control is ON */
1217                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1218                         __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1219                                   &data->link_report_flags);
1220         } else { /* VF */
1221                 *data = bp->vf_link_vars;
1222         }
1223 }
1224
1225 /**
1226  * bnx2x_link_report - report link status to OS.
1227  *
1228  * @bp:         driver handle
1229  *
1230  * Calls the __bnx2x_link_report() under the same locking scheme
1231  * as a link/PHY state managing code to ensure a consistent link
1232  * reporting.
1233  */
1234
1235 void bnx2x_link_report(struct bnx2x *bp)
1236 {
1237         bnx2x_acquire_phy_lock(bp);
1238         __bnx2x_link_report(bp);
1239         bnx2x_release_phy_lock(bp);
1240 }
1241
1242 /**
1243  * __bnx2x_link_report - report link status to OS.
1244  *
1245  * @bp:         driver handle
1246  *
1247  * None atomic implementation.
1248  * Should be called under the phy_lock.
1249  */
1250 void __bnx2x_link_report(struct bnx2x *bp)
1251 {
1252         struct bnx2x_link_report_data cur_data;
1253
1254         /* reread mf_cfg */
1255         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1256                 bnx2x_read_mf_cfg(bp);
1257
1258         /* Read the current link report info */
1259         bnx2x_fill_report_data(bp, &cur_data);
1260
1261         /* Don't report link down or exactly the same link status twice */
1262         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1263             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1264                       &bp->last_reported_link.link_report_flags) &&
1265              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1266                       &cur_data.link_report_flags)))
1267                 return;
1268
1269         bp->link_cnt++;
1270
1271         /* We are going to report a new link parameters now -
1272          * remember the current data for the next time.
1273          */
1274         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1275
1276         /* propagate status to VFs */
1277         if (IS_PF(bp))
1278                 bnx2x_iov_link_update(bp);
1279
1280         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1281                      &cur_data.link_report_flags)) {
1282                 netif_carrier_off(bp->dev);
1283                 netdev_err(bp->dev, "NIC Link is Down\n");
1284                 return;
1285         } else {
1286                 const char *duplex;
1287                 const char *flow;
1288
1289                 netif_carrier_on(bp->dev);
1290
1291                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1292                                        &cur_data.link_report_flags))
1293                         duplex = "full";
1294                 else
1295                         duplex = "half";
1296
1297                 /* Handle the FC at the end so that only these flags would be
1298                  * possibly set. This way we may easily check if there is no FC
1299                  * enabled.
1300                  */
1301                 if (cur_data.link_report_flags) {
1302                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1303                                      &cur_data.link_report_flags)) {
1304                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1305                                      &cur_data.link_report_flags))
1306                                         flow = "ON - receive & transmit";
1307                                 else
1308                                         flow = "ON - receive";
1309                         } else {
1310                                 flow = "ON - transmit";
1311                         }
1312                 } else {
1313                         flow = "none";
1314                 }
1315                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1316                             cur_data.line_speed, duplex, flow);
1317         }
1318 }
1319
1320 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1321 {
1322         int i;
1323
1324         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1325                 struct eth_rx_sge *sge;
1326
1327                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1328                 sge->addr_hi =
1329                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1330                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1331
1332                 sge->addr_lo =
1333                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1334                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1335         }
1336 }
1337
1338 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1339                                 struct bnx2x_fastpath *fp, int last)
1340 {
1341         int i;
1342
1343         for (i = 0; i < last; i++) {
1344                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1345                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1346                 u8 *data = first_buf->data;
1347
1348                 if (data == NULL) {
1349                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1350                         continue;
1351                 }
1352                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1353                         dma_unmap_single(&bp->pdev->dev,
1354                                          dma_unmap_addr(first_buf, mapping),
1355                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1356                 bnx2x_frag_free(fp, data);
1357                 first_buf->data = NULL;
1358         }
1359 }
1360
1361 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1362 {
1363         int j;
1364
1365         for_each_rx_queue_cnic(bp, j) {
1366                 struct bnx2x_fastpath *fp = &bp->fp[j];
1367
1368                 fp->rx_bd_cons = 0;
1369
1370                 /* Activate BD ring */
1371                 /* Warning!
1372                  * this will generate an interrupt (to the TSTORM)
1373                  * must only be done after chip is initialized
1374                  */
1375                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1376                                      fp->rx_sge_prod);
1377         }
1378 }
1379
1380 void bnx2x_init_rx_rings(struct bnx2x *bp)
1381 {
1382         int func = BP_FUNC(bp);
1383         u16 ring_prod;
1384         int i, j;
1385
1386         /* Allocate TPA resources */
1387         for_each_eth_queue(bp, j) {
1388                 struct bnx2x_fastpath *fp = &bp->fp[j];
1389
1390                 DP(NETIF_MSG_IFUP,
1391                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1392
1393                 if (!fp->disable_tpa) {
1394                         /* Fill the per-aggregation pool */
1395                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1396                                 struct bnx2x_agg_info *tpa_info =
1397                                         &fp->tpa_info[i];
1398                                 struct sw_rx_bd *first_buf =
1399                                         &tpa_info->first_buf;
1400
1401                                 first_buf->data =
1402                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1403                                 if (!first_buf->data) {
1404                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1405                                                   j);
1406                                         bnx2x_free_tpa_pool(bp, fp, i);
1407                                         fp->disable_tpa = 1;
1408                                         break;
1409                                 }
1410                                 dma_unmap_addr_set(first_buf, mapping, 0);
1411                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1412                         }
1413
1414                         /* "next page" elements initialization */
1415                         bnx2x_set_next_page_sgl(fp);
1416
1417                         /* set SGEs bit mask */
1418                         bnx2x_init_sge_ring_bit_mask(fp);
1419
1420                         /* Allocate SGEs and initialize the ring elements */
1421                         for (i = 0, ring_prod = 0;
1422                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1423
1424                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1425                                                        GFP_KERNEL) < 0) {
1426                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1427                                                   i);
1428                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1429                                                   j);
1430                                         /* Cleanup already allocated elements */
1431                                         bnx2x_free_rx_sge_range(bp, fp,
1432                                                                 ring_prod);
1433                                         bnx2x_free_tpa_pool(bp, fp,
1434                                                             MAX_AGG_QS(bp));
1435                                         fp->disable_tpa = 1;
1436                                         ring_prod = 0;
1437                                         break;
1438                                 }
1439                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1440                         }
1441
1442                         fp->rx_sge_prod = ring_prod;
1443                 }
1444         }
1445
1446         for_each_eth_queue(bp, j) {
1447                 struct bnx2x_fastpath *fp = &bp->fp[j];
1448
1449                 fp->rx_bd_cons = 0;
1450
1451                 /* Activate BD ring */
1452                 /* Warning!
1453                  * this will generate an interrupt (to the TSTORM)
1454                  * must only be done after chip is initialized
1455                  */
1456                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1457                                      fp->rx_sge_prod);
1458
1459                 if (j != 0)
1460                         continue;
1461
1462                 if (CHIP_IS_E1(bp)) {
1463                         REG_WR(bp, BAR_USTRORM_INTMEM +
1464                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1465                                U64_LO(fp->rx_comp_mapping));
1466                         REG_WR(bp, BAR_USTRORM_INTMEM +
1467                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1468                                U64_HI(fp->rx_comp_mapping));
1469                 }
1470         }
1471 }
1472
1473 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1474 {
1475         u8 cos;
1476         struct bnx2x *bp = fp->bp;
1477
1478         for_each_cos_in_tx_queue(fp, cos) {
1479                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1480                 unsigned pkts_compl = 0, bytes_compl = 0;
1481
1482                 u16 sw_prod = txdata->tx_pkt_prod;
1483                 u16 sw_cons = txdata->tx_pkt_cons;
1484
1485                 while (sw_cons != sw_prod) {
1486                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1487                                           &pkts_compl, &bytes_compl);
1488                         sw_cons++;
1489                 }
1490
1491                 netdev_tx_reset_queue(
1492                         netdev_get_tx_queue(bp->dev,
1493                                             txdata->txq_index));
1494         }
1495 }
1496
1497 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1498 {
1499         int i;
1500
1501         for_each_tx_queue_cnic(bp, i) {
1502                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1503         }
1504 }
1505
1506 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1507 {
1508         int i;
1509
1510         for_each_eth_queue(bp, i) {
1511                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1512         }
1513 }
1514
1515 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1516 {
1517         struct bnx2x *bp = fp->bp;
1518         int i;
1519
1520         /* ring wasn't allocated */
1521         if (fp->rx_buf_ring == NULL)
1522                 return;
1523
1524         for (i = 0; i < NUM_RX_BD; i++) {
1525                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1526                 u8 *data = rx_buf->data;
1527
1528                 if (data == NULL)
1529                         continue;
1530                 dma_unmap_single(&bp->pdev->dev,
1531                                  dma_unmap_addr(rx_buf, mapping),
1532                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1533
1534                 rx_buf->data = NULL;
1535                 bnx2x_frag_free(fp, data);
1536         }
1537 }
1538
1539 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1540 {
1541         int j;
1542
1543         for_each_rx_queue_cnic(bp, j) {
1544                 bnx2x_free_rx_bds(&bp->fp[j]);
1545         }
1546 }
1547
1548 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1549 {
1550         int j;
1551
1552         for_each_eth_queue(bp, j) {
1553                 struct bnx2x_fastpath *fp = &bp->fp[j];
1554
1555                 bnx2x_free_rx_bds(fp);
1556
1557                 if (!fp->disable_tpa)
1558                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1559         }
1560 }
1561
1562 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1563 {
1564         bnx2x_free_tx_skbs_cnic(bp);
1565         bnx2x_free_rx_skbs_cnic(bp);
1566 }
1567
1568 void bnx2x_free_skbs(struct bnx2x *bp)
1569 {
1570         bnx2x_free_tx_skbs(bp);
1571         bnx2x_free_rx_skbs(bp);
1572 }
1573
1574 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1575 {
1576         /* load old values */
1577         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1578
1579         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1580                 /* leave all but MAX value */
1581                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1582
1583                 /* set new MAX value */
1584                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1585                                 & FUNC_MF_CFG_MAX_BW_MASK;
1586
1587                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1588         }
1589 }
1590
1591 /**
1592  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1593  *
1594  * @bp:         driver handle
1595  * @nvecs:      number of vectors to be released
1596  */
1597 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1598 {
1599         int i, offset = 0;
1600
1601         if (nvecs == offset)
1602                 return;
1603
1604         /* VFs don't have a default SB */
1605         if (IS_PF(bp)) {
1606                 free_irq(bp->msix_table[offset].vector, bp->dev);
1607                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1608                    bp->msix_table[offset].vector);
1609                 offset++;
1610         }
1611
1612         if (CNIC_SUPPORT(bp)) {
1613                 if (nvecs == offset)
1614                         return;
1615                 offset++;
1616         }
1617
1618         for_each_eth_queue(bp, i) {
1619                 if (nvecs == offset)
1620                         return;
1621                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1622                    i, bp->msix_table[offset].vector);
1623
1624                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1625         }
1626 }
1627
1628 void bnx2x_free_irq(struct bnx2x *bp)
1629 {
1630         if (bp->flags & USING_MSIX_FLAG &&
1631             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1632                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1633
1634                 /* vfs don't have a default status block */
1635                 if (IS_PF(bp))
1636                         nvecs++;
1637
1638                 bnx2x_free_msix_irqs(bp, nvecs);
1639         } else {
1640                 free_irq(bp->dev->irq, bp->dev);
1641         }
1642 }
1643
1644 int bnx2x_enable_msix(struct bnx2x *bp)
1645 {
1646         int msix_vec = 0, i, rc;
1647
1648         /* VFs don't have a default status block */
1649         if (IS_PF(bp)) {
1650                 bp->msix_table[msix_vec].entry = msix_vec;
1651                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1652                                bp->msix_table[0].entry);
1653                 msix_vec++;
1654         }
1655
1656         /* Cnic requires an msix vector for itself */
1657         if (CNIC_SUPPORT(bp)) {
1658                 bp->msix_table[msix_vec].entry = msix_vec;
1659                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1660                                msix_vec, bp->msix_table[msix_vec].entry);
1661                 msix_vec++;
1662         }
1663
1664         /* We need separate vectors for ETH queues only (not FCoE) */
1665         for_each_eth_queue(bp, i) {
1666                 bp->msix_table[msix_vec].entry = msix_vec;
1667                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1668                                msix_vec, msix_vec, i);
1669                 msix_vec++;
1670         }
1671
1672         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1673            msix_vec);
1674
1675         rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1676                                    BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1677         /*
1678          * reconfigure number of tx/rx queues according to available
1679          * MSI-X vectors
1680          */
1681         if (rc == -ENOSPC) {
1682                 /* Get by with single vector */
1683                 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1684                 if (rc < 0) {
1685                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1686                                        rc);
1687                         goto no_msix;
1688                 }
1689
1690                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1691                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1692
1693                 BNX2X_DEV_INFO("set number of queues to 1\n");
1694                 bp->num_ethernet_queues = 1;
1695                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1696         } else if (rc < 0) {
1697                 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1698                 goto no_msix;
1699         } else if (rc < msix_vec) {
1700                 /* how less vectors we will have? */
1701                 int diff = msix_vec - rc;
1702
1703                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1704
1705                 /*
1706                  * decrease number of queues by number of unallocated entries
1707                  */
1708                 bp->num_ethernet_queues -= diff;
1709                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1710
1711                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1712                                bp->num_queues);
1713         }
1714
1715         bp->flags |= USING_MSIX_FLAG;
1716
1717         return 0;
1718
1719 no_msix:
1720         /* fall to INTx if not enough memory */
1721         if (rc == -ENOMEM)
1722                 bp->flags |= DISABLE_MSI_FLAG;
1723
1724         return rc;
1725 }
1726
1727 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1728 {
1729         int i, rc, offset = 0;
1730
1731         /* no default status block for vf */
1732         if (IS_PF(bp)) {
1733                 rc = request_irq(bp->msix_table[offset++].vector,
1734                                  bnx2x_msix_sp_int, 0,
1735                                  bp->dev->name, bp->dev);
1736                 if (rc) {
1737                         BNX2X_ERR("request sp irq failed\n");
1738                         return -EBUSY;
1739                 }
1740         }
1741
1742         if (CNIC_SUPPORT(bp))
1743                 offset++;
1744
1745         for_each_eth_queue(bp, i) {
1746                 struct bnx2x_fastpath *fp = &bp->fp[i];
1747                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1748                          bp->dev->name, i);
1749
1750                 rc = request_irq(bp->msix_table[offset].vector,
1751                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1752                 if (rc) {
1753                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1754                               bp->msix_table[offset].vector, rc);
1755                         bnx2x_free_msix_irqs(bp, offset);
1756                         return -EBUSY;
1757                 }
1758
1759                 offset++;
1760         }
1761
1762         i = BNX2X_NUM_ETH_QUEUES(bp);
1763         if (IS_PF(bp)) {
1764                 offset = 1 + CNIC_SUPPORT(bp);
1765                 netdev_info(bp->dev,
1766                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1767                             bp->msix_table[0].vector,
1768                             0, bp->msix_table[offset].vector,
1769                             i - 1, bp->msix_table[offset + i - 1].vector);
1770         } else {
1771                 offset = CNIC_SUPPORT(bp);
1772                 netdev_info(bp->dev,
1773                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1774                             0, bp->msix_table[offset].vector,
1775                             i - 1, bp->msix_table[offset + i - 1].vector);
1776         }
1777         return 0;
1778 }
1779
1780 int bnx2x_enable_msi(struct bnx2x *bp)
1781 {
1782         int rc;
1783
1784         rc = pci_enable_msi(bp->pdev);
1785         if (rc) {
1786                 BNX2X_DEV_INFO("MSI is not attainable\n");
1787                 return -1;
1788         }
1789         bp->flags |= USING_MSI_FLAG;
1790
1791         return 0;
1792 }
1793
1794 static int bnx2x_req_irq(struct bnx2x *bp)
1795 {
1796         unsigned long flags;
1797         unsigned int irq;
1798
1799         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1800                 flags = 0;
1801         else
1802                 flags = IRQF_SHARED;
1803
1804         if (bp->flags & USING_MSIX_FLAG)
1805                 irq = bp->msix_table[0].vector;
1806         else
1807                 irq = bp->pdev->irq;
1808
1809         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1810 }
1811
1812 static int bnx2x_setup_irqs(struct bnx2x *bp)
1813 {
1814         int rc = 0;
1815         if (bp->flags & USING_MSIX_FLAG &&
1816             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1817                 rc = bnx2x_req_msix_irqs(bp);
1818                 if (rc)
1819                         return rc;
1820         } else {
1821                 rc = bnx2x_req_irq(bp);
1822                 if (rc) {
1823                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1824                         return rc;
1825                 }
1826                 if (bp->flags & USING_MSI_FLAG) {
1827                         bp->dev->irq = bp->pdev->irq;
1828                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1829                                     bp->dev->irq);
1830                 }
1831                 if (bp->flags & USING_MSIX_FLAG) {
1832                         bp->dev->irq = bp->msix_table[0].vector;
1833                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1834                                     bp->dev->irq);
1835                 }
1836         }
1837
1838         return 0;
1839 }
1840
1841 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1842 {
1843         int i;
1844
1845         for_each_rx_queue_cnic(bp, i) {
1846                 bnx2x_fp_init_lock(&bp->fp[i]);
1847                 napi_enable(&bnx2x_fp(bp, i, napi));
1848         }
1849 }
1850
1851 static void bnx2x_napi_enable(struct bnx2x *bp)
1852 {
1853         int i;
1854
1855         for_each_eth_queue(bp, i) {
1856                 bnx2x_fp_init_lock(&bp->fp[i]);
1857                 napi_enable(&bnx2x_fp(bp, i, napi));
1858         }
1859 }
1860
1861 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1862 {
1863         int i;
1864
1865         for_each_rx_queue_cnic(bp, i) {
1866                 napi_disable(&bnx2x_fp(bp, i, napi));
1867                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1868                         usleep_range(1000, 2000);
1869         }
1870 }
1871
1872 static void bnx2x_napi_disable(struct bnx2x *bp)
1873 {
1874         int i;
1875
1876         for_each_eth_queue(bp, i) {
1877                 napi_disable(&bnx2x_fp(bp, i, napi));
1878                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1879                         usleep_range(1000, 2000);
1880         }
1881 }
1882
1883 void bnx2x_netif_start(struct bnx2x *bp)
1884 {
1885         if (netif_running(bp->dev)) {
1886                 bnx2x_napi_enable(bp);
1887                 if (CNIC_LOADED(bp))
1888                         bnx2x_napi_enable_cnic(bp);
1889                 bnx2x_int_enable(bp);
1890                 if (bp->state == BNX2X_STATE_OPEN)
1891                         netif_tx_wake_all_queues(bp->dev);
1892         }
1893 }
1894
1895 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1896 {
1897         bnx2x_int_disable_sync(bp, disable_hw);
1898         bnx2x_napi_disable(bp);
1899         if (CNIC_LOADED(bp))
1900                 bnx2x_napi_disable_cnic(bp);
1901 }
1902
1903 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1904                        void *accel_priv, select_queue_fallback_t fallback)
1905 {
1906         struct bnx2x *bp = netdev_priv(dev);
1907
1908         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1909                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1910                 u16 ether_type = ntohs(hdr->h_proto);
1911
1912                 /* Skip VLAN tag if present */
1913                 if (ether_type == ETH_P_8021Q) {
1914                         struct vlan_ethhdr *vhdr =
1915                                 (struct vlan_ethhdr *)skb->data;
1916
1917                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1918                 }
1919
1920                 /* If ethertype is FCoE or FIP - use FCoE ring */
1921                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1922                         return bnx2x_fcoe_tx(bp, txq_index);
1923         }
1924
1925         /* select a non-FCoE queue */
1926         return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1927 }
1928
1929 void bnx2x_set_num_queues(struct bnx2x *bp)
1930 {
1931         /* RSS queues */
1932         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1933
1934         /* override in STORAGE SD modes */
1935         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1936                 bp->num_ethernet_queues = 1;
1937
1938         /* Add special queues */
1939         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1940         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1941
1942         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1943 }
1944
1945 /**
1946  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1947  *
1948  * @bp:         Driver handle
1949  *
1950  * We currently support for at most 16 Tx queues for each CoS thus we will
1951  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1952  * bp->max_cos.
1953  *
1954  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1955  * index after all ETH L2 indices.
1956  *
1957  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1958  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1959  * 16..31,...) with indices that are not coupled with any real Tx queue.
1960  *
1961  * The proper configuration of skb->queue_mapping is handled by
1962  * bnx2x_select_queue() and __skb_tx_hash().
1963  *
1964  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1965  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1966  */
1967 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1968 {
1969         int rc, tx, rx;
1970
1971         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1972         rx = BNX2X_NUM_ETH_QUEUES(bp);
1973
1974 /* account for fcoe queue */
1975         if (include_cnic && !NO_FCOE(bp)) {
1976                 rx++;
1977                 tx++;
1978         }
1979
1980         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1981         if (rc) {
1982                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1983                 return rc;
1984         }
1985         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1986         if (rc) {
1987                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1988                 return rc;
1989         }
1990
1991         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1992                           tx, rx);
1993
1994         return rc;
1995 }
1996
1997 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1998 {
1999         int i;
2000
2001         for_each_queue(bp, i) {
2002                 struct bnx2x_fastpath *fp = &bp->fp[i];
2003                 u32 mtu;
2004
2005                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2006                 if (IS_FCOE_IDX(i))
2007                         /*
2008                          * Although there are no IP frames expected to arrive to
2009                          * this ring we still want to add an
2010                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2011                          * overrun attack.
2012                          */
2013                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2014                 else
2015                         mtu = bp->dev->mtu;
2016                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2017                                   IP_HEADER_ALIGNMENT_PADDING +
2018                                   ETH_OVREHEAD +
2019                                   mtu +
2020                                   BNX2X_FW_RX_ALIGN_END;
2021                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2022                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2023                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2024                 else
2025                         fp->rx_frag_size = 0;
2026         }
2027 }
2028
2029 static int bnx2x_init_rss(struct bnx2x *bp)
2030 {
2031         int i;
2032         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2033
2034         /* Prepare the initial contents for the indirection table if RSS is
2035          * enabled
2036          */
2037         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2038                 bp->rss_conf_obj.ind_table[i] =
2039                         bp->fp->cl_id +
2040                         ethtool_rxfh_indir_default(i, num_eth_queues);
2041
2042         /*
2043          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2044          * per-port, so if explicit configuration is needed , do it only
2045          * for a PMF.
2046          *
2047          * For 57712 and newer on the other hand it's a per-function
2048          * configuration.
2049          */
2050         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2051 }
2052
2053 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2054               bool config_hash, bool enable)
2055 {
2056         struct bnx2x_config_rss_params params = {NULL};
2057
2058         /* Although RSS is meaningless when there is a single HW queue we
2059          * still need it enabled in order to have HW Rx hash generated.
2060          *
2061          * if (!is_eth_multi(bp))
2062          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2063          */
2064
2065         params.rss_obj = rss_obj;
2066
2067         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2068
2069         if (enable) {
2070                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2071
2072                 /* RSS configuration */
2073                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2074                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2075                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2076                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2077                 if (rss_obj->udp_rss_v4)
2078                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2079                 if (rss_obj->udp_rss_v6)
2080                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2081         } else {
2082                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2083         }
2084
2085         /* Hash bits */
2086         params.rss_result_mask = MULTI_MASK;
2087
2088         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2089
2090         if (config_hash) {
2091                 /* RSS keys */
2092                 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2093                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2094         }
2095
2096         if (IS_PF(bp))
2097                 return bnx2x_config_rss(bp, &params);
2098         else
2099                 return bnx2x_vfpf_config_rss(bp, &params);
2100 }
2101
2102 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2103 {
2104         struct bnx2x_func_state_params func_params = {NULL};
2105
2106         /* Prepare parameters for function state transitions */
2107         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2108
2109         func_params.f_obj = &bp->func_obj;
2110         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2111
2112         func_params.params.hw_init.load_phase = load_code;
2113
2114         return bnx2x_func_state_change(bp, &func_params);
2115 }
2116
2117 /*
2118  * Cleans the object that have internal lists without sending
2119  * ramrods. Should be run when interrupts are disabled.
2120  */
2121 void bnx2x_squeeze_objects(struct bnx2x *bp)
2122 {
2123         int rc;
2124         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2125         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2126         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2127
2128         /***************** Cleanup MACs' object first *************************/
2129
2130         /* Wait for completion of requested */
2131         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2132         /* Perform a dry cleanup */
2133         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2134
2135         /* Clean ETH primary MAC */
2136         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2137         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2138                                  &ramrod_flags);
2139         if (rc != 0)
2140                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2141
2142         /* Cleanup UC list */
2143         vlan_mac_flags = 0;
2144         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2145         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2146                                  &ramrod_flags);
2147         if (rc != 0)
2148                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2149
2150         /***************** Now clean mcast object *****************************/
2151         rparam.mcast_obj = &bp->mcast_obj;
2152         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2153
2154         /* Add a DEL command... - Since we're doing a driver cleanup only,
2155          * we take a lock surrounding both the initial send and the CONTs,
2156          * as we don't want a true completion to disrupt us in the middle.
2157          */
2158         netif_addr_lock_bh(bp->dev);
2159         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2160         if (rc < 0)
2161                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2162                           rc);
2163
2164         /* ...and wait until all pending commands are cleared */
2165         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2166         while (rc != 0) {
2167                 if (rc < 0) {
2168                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2169                                   rc);
2170                         netif_addr_unlock_bh(bp->dev);
2171                         return;
2172                 }
2173
2174                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2175         }
2176         netif_addr_unlock_bh(bp->dev);
2177 }
2178
2179 #ifndef BNX2X_STOP_ON_ERROR
2180 #define LOAD_ERROR_EXIT(bp, label) \
2181         do { \
2182                 (bp)->state = BNX2X_STATE_ERROR; \
2183                 goto label; \
2184         } while (0)
2185
2186 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2187         do { \
2188                 bp->cnic_loaded = false; \
2189                 goto label; \
2190         } while (0)
2191 #else /*BNX2X_STOP_ON_ERROR*/
2192 #define LOAD_ERROR_EXIT(bp, label) \
2193         do { \
2194                 (bp)->state = BNX2X_STATE_ERROR; \
2195                 (bp)->panic = 1; \
2196                 return -EBUSY; \
2197         } while (0)
2198 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2199         do { \
2200                 bp->cnic_loaded = false; \
2201                 (bp)->panic = 1; \
2202                 return -EBUSY; \
2203         } while (0)
2204 #endif /*BNX2X_STOP_ON_ERROR*/
2205
2206 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2207 {
2208         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2209                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2210         return;
2211 }
2212
2213 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2214 {
2215         int num_groups, vf_headroom = 0;
2216         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2217
2218         /* number of queues for statistics is number of eth queues + FCoE */
2219         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2220
2221         /* Total number of FW statistics requests =
2222          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2223          * and fcoe l2 queue) stats + num of queues (which includes another 1
2224          * for fcoe l2 queue if applicable)
2225          */
2226         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2227
2228         /* vf stats appear in the request list, but their data is allocated by
2229          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2230          * it is used to determine where to place the vf stats queries in the
2231          * request struct
2232          */
2233         if (IS_SRIOV(bp))
2234                 vf_headroom = bnx2x_vf_headroom(bp);
2235
2236         /* Request is built from stats_query_header and an array of
2237          * stats_query_cmd_group each of which contains
2238          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2239          * configured in the stats_query_header.
2240          */
2241         num_groups =
2242                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2243                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2244                  1 : 0));
2245
2246         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2247            bp->fw_stats_num, vf_headroom, num_groups);
2248         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2249                 num_groups * sizeof(struct stats_query_cmd_group);
2250
2251         /* Data for statistics requests + stats_counter
2252          * stats_counter holds per-STORM counters that are incremented
2253          * when STORM has finished with the current request.
2254          * memory for FCoE offloaded statistics are counted anyway,
2255          * even if they will not be sent.
2256          * VF stats are not accounted for here as the data of VF stats is stored
2257          * in memory allocated by the VF, not here.
2258          */
2259         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2260                 sizeof(struct per_pf_stats) +
2261                 sizeof(struct fcoe_statistics_params) +
2262                 sizeof(struct per_queue_stats) * num_queue_stats +
2263                 sizeof(struct stats_counter);
2264
2265         bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2266                                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2267         if (!bp->fw_stats)
2268                 goto alloc_mem_err;
2269
2270         /* Set shortcuts */
2271         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2272         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2273         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2274                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2275         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2276                 bp->fw_stats_req_sz;
2277
2278         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2279            U64_HI(bp->fw_stats_req_mapping),
2280            U64_LO(bp->fw_stats_req_mapping));
2281         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2282            U64_HI(bp->fw_stats_data_mapping),
2283            U64_LO(bp->fw_stats_data_mapping));
2284         return 0;
2285
2286 alloc_mem_err:
2287         bnx2x_free_fw_stats_mem(bp);
2288         BNX2X_ERR("Can't allocate FW stats memory\n");
2289         return -ENOMEM;
2290 }
2291
2292 /* send load request to mcp and analyze response */
2293 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2294 {
2295         u32 param;
2296
2297         /* init fw_seq */
2298         bp->fw_seq =
2299                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2300                  DRV_MSG_SEQ_NUMBER_MASK);
2301         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2302
2303         /* Get current FW pulse sequence */
2304         bp->fw_drv_pulse_wr_seq =
2305                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2306                  DRV_PULSE_SEQ_MASK);
2307         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2308
2309         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2310
2311         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2312                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2313
2314         /* load request */
2315         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2316
2317         /* if mcp fails to respond we must abort */
2318         if (!(*load_code)) {
2319                 BNX2X_ERR("MCP response failure, aborting\n");
2320                 return -EBUSY;
2321         }
2322
2323         /* If mcp refused (e.g. other port is in diagnostic mode) we
2324          * must abort
2325          */
2326         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2327                 BNX2X_ERR("MCP refused load request, aborting\n");
2328                 return -EBUSY;
2329         }
2330         return 0;
2331 }
2332
2333 /* check whether another PF has already loaded FW to chip. In
2334  * virtualized environments a pf from another VM may have already
2335  * initialized the device including loading FW
2336  */
2337 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2338 {
2339         /* is another pf loaded on this engine? */
2340         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2341             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2342                 /* build my FW version dword */
2343                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2344                         (BCM_5710_FW_MINOR_VERSION << 8) +
2345                         (BCM_5710_FW_REVISION_VERSION << 16) +
2346                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2347
2348                 /* read loaded FW from chip */
2349                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2350
2351                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2352                    loaded_fw, my_fw);
2353
2354                 /* abort nic load if version mismatch */
2355                 if (my_fw != loaded_fw) {
2356                         if (print_err)
2357                                 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2358                                           loaded_fw, my_fw);
2359                         else
2360                                 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2361                                                loaded_fw, my_fw);
2362                         return -EBUSY;
2363                 }
2364         }
2365         return 0;
2366 }
2367
2368 /* returns the "mcp load_code" according to global load_count array */
2369 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2370 {
2371         int path = BP_PATH(bp);
2372
2373         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2374            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2375            bnx2x_load_count[path][2]);
2376         bnx2x_load_count[path][0]++;
2377         bnx2x_load_count[path][1 + port]++;
2378         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2379            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2380            bnx2x_load_count[path][2]);
2381         if (bnx2x_load_count[path][0] == 1)
2382                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2383         else if (bnx2x_load_count[path][1 + port] == 1)
2384                 return FW_MSG_CODE_DRV_LOAD_PORT;
2385         else
2386                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2387 }
2388
2389 /* mark PMF if applicable */
2390 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2391 {
2392         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2393             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2394             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2395                 bp->port.pmf = 1;
2396                 /* We need the barrier to ensure the ordering between the
2397                  * writing to bp->port.pmf here and reading it from the
2398                  * bnx2x_periodic_task().
2399                  */
2400                 smp_mb();
2401         } else {
2402                 bp->port.pmf = 0;
2403         }
2404
2405         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2406 }
2407
2408 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2409 {
2410         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2411              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2412             (bp->common.shmem2_base)) {
2413                 if (SHMEM2_HAS(bp, dcc_support))
2414                         SHMEM2_WR(bp, dcc_support,
2415                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2416                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2417                 if (SHMEM2_HAS(bp, afex_driver_support))
2418                         SHMEM2_WR(bp, afex_driver_support,
2419                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2420         }
2421
2422         /* Set AFEX default VLAN tag to an invalid value */
2423         bp->afex_def_vlan_tag = -1;
2424 }
2425
2426 /**
2427  * bnx2x_bz_fp - zero content of the fastpath structure.
2428  *
2429  * @bp:         driver handle
2430  * @index:      fastpath index to be zeroed
2431  *
2432  * Makes sure the contents of the bp->fp[index].napi is kept
2433  * intact.
2434  */
2435 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2436 {
2437         struct bnx2x_fastpath *fp = &bp->fp[index];
2438         int cos;
2439         struct napi_struct orig_napi = fp->napi;
2440         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2441
2442         /* bzero bnx2x_fastpath contents */
2443         if (fp->tpa_info)
2444                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2445                        sizeof(struct bnx2x_agg_info));
2446         memset(fp, 0, sizeof(*fp));
2447
2448         /* Restore the NAPI object as it has been already initialized */
2449         fp->napi = orig_napi;
2450         fp->tpa_info = orig_tpa_info;
2451         fp->bp = bp;
2452         fp->index = index;
2453         if (IS_ETH_FP(fp))
2454                 fp->max_cos = bp->max_cos;
2455         else
2456                 /* Special queues support only one CoS */
2457                 fp->max_cos = 1;
2458
2459         /* Init txdata pointers */
2460         if (IS_FCOE_FP(fp))
2461                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2462         if (IS_ETH_FP(fp))
2463                 for_each_cos_in_tx_queue(fp, cos)
2464                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2465                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2466
2467         /* set the tpa flag for each queue. The tpa flag determines the queue
2468          * minimal size so it must be set prior to queue memory allocation
2469          */
2470         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2471                                   (bp->flags & GRO_ENABLE_FLAG &&
2472                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2473         if (bp->flags & TPA_ENABLE_FLAG)
2474                 fp->mode = TPA_MODE_LRO;
2475         else if (bp->flags & GRO_ENABLE_FLAG)
2476                 fp->mode = TPA_MODE_GRO;
2477
2478         /* We don't want TPA on an FCoE L2 ring */
2479         if (IS_FCOE_FP(fp))
2480                 fp->disable_tpa = 1;
2481 }
2482
2483 int bnx2x_load_cnic(struct bnx2x *bp)
2484 {
2485         int i, rc, port = BP_PORT(bp);
2486
2487         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2488
2489         mutex_init(&bp->cnic_mutex);
2490
2491         if (IS_PF(bp)) {
2492                 rc = bnx2x_alloc_mem_cnic(bp);
2493                 if (rc) {
2494                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2495                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2496                 }
2497         }
2498
2499         rc = bnx2x_alloc_fp_mem_cnic(bp);
2500         if (rc) {
2501                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2502                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2503         }
2504
2505         /* Update the number of queues with the cnic queues */
2506         rc = bnx2x_set_real_num_queues(bp, 1);
2507         if (rc) {
2508                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2509                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2510         }
2511
2512         /* Add all CNIC NAPI objects */
2513         bnx2x_add_all_napi_cnic(bp);
2514         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2515         bnx2x_napi_enable_cnic(bp);
2516
2517         rc = bnx2x_init_hw_func_cnic(bp);
2518         if (rc)
2519                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2520
2521         bnx2x_nic_init_cnic(bp);
2522
2523         if (IS_PF(bp)) {
2524                 /* Enable Timer scan */
2525                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2526
2527                 /* setup cnic queues */
2528                 for_each_cnic_queue(bp, i) {
2529                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2530                         if (rc) {
2531                                 BNX2X_ERR("Queue setup failed\n");
2532                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2533                         }
2534                 }
2535         }
2536
2537         /* Initialize Rx filter. */
2538         bnx2x_set_rx_mode_inner(bp);
2539
2540         /* re-read iscsi info */
2541         bnx2x_get_iscsi_info(bp);
2542         bnx2x_setup_cnic_irq_info(bp);
2543         bnx2x_setup_cnic_info(bp);
2544         bp->cnic_loaded = true;
2545         if (bp->state == BNX2X_STATE_OPEN)
2546                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2547
2548         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2549
2550         return 0;
2551
2552 #ifndef BNX2X_STOP_ON_ERROR
2553 load_error_cnic2:
2554         /* Disable Timer scan */
2555         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2556
2557 load_error_cnic1:
2558         bnx2x_napi_disable_cnic(bp);
2559         /* Update the number of queues without the cnic queues */
2560         if (bnx2x_set_real_num_queues(bp, 0))
2561                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2562 load_error_cnic0:
2563         BNX2X_ERR("CNIC-related load failed\n");
2564         bnx2x_free_fp_mem_cnic(bp);
2565         bnx2x_free_mem_cnic(bp);
2566         return rc;
2567 #endif /* ! BNX2X_STOP_ON_ERROR */
2568 }
2569
2570 /* must be called with rtnl_lock */
2571 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2572 {
2573         int port = BP_PORT(bp);
2574         int i, rc = 0, load_code = 0;
2575
2576         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2577         DP(NETIF_MSG_IFUP,
2578            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2579
2580 #ifdef BNX2X_STOP_ON_ERROR
2581         if (unlikely(bp->panic)) {
2582                 BNX2X_ERR("Can't load NIC when there is panic\n");
2583                 return -EPERM;
2584         }
2585 #endif
2586
2587         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2588
2589         /* zero the structure w/o any lock, before SP handler is initialized */
2590         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2591         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2592                 &bp->last_reported_link.link_report_flags);
2593
2594         if (IS_PF(bp))
2595                 /* must be called before memory allocation and HW init */
2596                 bnx2x_ilt_set_info(bp);
2597
2598         /*
2599          * Zero fastpath structures preserving invariants like napi, which are
2600          * allocated only once, fp index, max_cos, bp pointer.
2601          * Also set fp->disable_tpa and txdata_ptr.
2602          */
2603         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2604         for_each_queue(bp, i)
2605                 bnx2x_bz_fp(bp, i);
2606         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2607                                   bp->num_cnic_queues) *
2608                                   sizeof(struct bnx2x_fp_txdata));
2609
2610         bp->fcoe_init = false;
2611
2612         /* Set the receive queues buffer size */
2613         bnx2x_set_rx_buf_size(bp);
2614
2615         if (IS_PF(bp)) {
2616                 rc = bnx2x_alloc_mem(bp);
2617                 if (rc) {
2618                         BNX2X_ERR("Unable to allocate bp memory\n");
2619                         return rc;
2620                 }
2621         }
2622
2623         /* need to be done after alloc mem, since it's self adjusting to amount
2624          * of memory available for RSS queues
2625          */
2626         rc = bnx2x_alloc_fp_mem(bp);
2627         if (rc) {
2628                 BNX2X_ERR("Unable to allocate memory for fps\n");
2629                 LOAD_ERROR_EXIT(bp, load_error0);
2630         }
2631
2632         /* Allocated memory for FW statistics  */
2633         if (bnx2x_alloc_fw_stats_mem(bp))
2634                 LOAD_ERROR_EXIT(bp, load_error0);
2635
2636         /* request pf to initialize status blocks */
2637         if (IS_VF(bp)) {
2638                 rc = bnx2x_vfpf_init(bp);
2639                 if (rc)
2640                         LOAD_ERROR_EXIT(bp, load_error0);
2641         }
2642
2643         /* As long as bnx2x_alloc_mem() may possibly update
2644          * bp->num_queues, bnx2x_set_real_num_queues() should always
2645          * come after it. At this stage cnic queues are not counted.
2646          */
2647         rc = bnx2x_set_real_num_queues(bp, 0);
2648         if (rc) {
2649                 BNX2X_ERR("Unable to set real_num_queues\n");
2650                 LOAD_ERROR_EXIT(bp, load_error0);
2651         }
2652
2653         /* configure multi cos mappings in kernel.
2654          * this configuration may be overridden by a multi class queue
2655          * discipline or by a dcbx negotiation result.
2656          */
2657         bnx2x_setup_tc(bp->dev, bp->max_cos);
2658
2659         /* Add all NAPI objects */
2660         bnx2x_add_all_napi(bp);
2661         DP(NETIF_MSG_IFUP, "napi added\n");
2662         bnx2x_napi_enable(bp);
2663
2664         if (IS_PF(bp)) {
2665                 /* set pf load just before approaching the MCP */
2666                 bnx2x_set_pf_load(bp);
2667
2668                 /* if mcp exists send load request and analyze response */
2669                 if (!BP_NOMCP(bp)) {
2670                         /* attempt to load pf */
2671                         rc = bnx2x_nic_load_request(bp, &load_code);
2672                         if (rc)
2673                                 LOAD_ERROR_EXIT(bp, load_error1);
2674
2675                         /* what did mcp say? */
2676                         rc = bnx2x_compare_fw_ver(bp, load_code, true);
2677                         if (rc) {
2678                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2679                                 LOAD_ERROR_EXIT(bp, load_error2);
2680                         }
2681                 } else {
2682                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2683                 }
2684
2685                 /* mark pmf if applicable */
2686                 bnx2x_nic_load_pmf(bp, load_code);
2687
2688                 /* Init Function state controlling object */
2689                 bnx2x__init_func_obj(bp);
2690
2691                 /* Initialize HW */
2692                 rc = bnx2x_init_hw(bp, load_code);
2693                 if (rc) {
2694                         BNX2X_ERR("HW init failed, aborting\n");
2695                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2696                         LOAD_ERROR_EXIT(bp, load_error2);
2697                 }
2698         }
2699
2700         bnx2x_pre_irq_nic_init(bp);
2701
2702         /* Connect to IRQs */
2703         rc = bnx2x_setup_irqs(bp);
2704         if (rc) {
2705                 BNX2X_ERR("setup irqs failed\n");
2706                 if (IS_PF(bp))
2707                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2708                 LOAD_ERROR_EXIT(bp, load_error2);
2709         }
2710
2711         /* Init per-function objects */
2712         if (IS_PF(bp)) {
2713                 /* Setup NIC internals and enable interrupts */
2714                 bnx2x_post_irq_nic_init(bp, load_code);
2715
2716                 bnx2x_init_bp_objs(bp);
2717                 bnx2x_iov_nic_init(bp);
2718
2719                 /* Set AFEX default VLAN tag to an invalid value */
2720                 bp->afex_def_vlan_tag = -1;
2721                 bnx2x_nic_load_afex_dcc(bp, load_code);
2722                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2723                 rc = bnx2x_func_start(bp);
2724                 if (rc) {
2725                         BNX2X_ERR("Function start failed!\n");
2726                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2727
2728                         LOAD_ERROR_EXIT(bp, load_error3);
2729                 }
2730
2731                 /* Send LOAD_DONE command to MCP */
2732                 if (!BP_NOMCP(bp)) {
2733                         load_code = bnx2x_fw_command(bp,
2734                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2735                         if (!load_code) {
2736                                 BNX2X_ERR("MCP response failure, aborting\n");
2737                                 rc = -EBUSY;
2738                                 LOAD_ERROR_EXIT(bp, load_error3);
2739                         }
2740                 }
2741
2742                 /* initialize FW coalescing state machines in RAM */
2743                 bnx2x_update_coalesce(bp);
2744         }
2745
2746         /* setup the leading queue */
2747         rc = bnx2x_setup_leading(bp);
2748         if (rc) {
2749                 BNX2X_ERR("Setup leading failed!\n");
2750                 LOAD_ERROR_EXIT(bp, load_error3);
2751         }
2752
2753         /* set up the rest of the queues */
2754         for_each_nondefault_eth_queue(bp, i) {
2755                 if (IS_PF(bp))
2756                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2757                 else /* VF */
2758                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2759                 if (rc) {
2760                         BNX2X_ERR("Queue %d setup failed\n", i);
2761                         LOAD_ERROR_EXIT(bp, load_error3);
2762                 }
2763         }
2764
2765         /* setup rss */
2766         rc = bnx2x_init_rss(bp);
2767         if (rc) {
2768                 BNX2X_ERR("PF RSS init failed\n");
2769                 LOAD_ERROR_EXIT(bp, load_error3);
2770         }
2771
2772         /* Now when Clients are configured we are ready to work */
2773         bp->state = BNX2X_STATE_OPEN;
2774
2775         /* Configure a ucast MAC */
2776         if (IS_PF(bp))
2777                 rc = bnx2x_set_eth_mac(bp, true);
2778         else /* vf */
2779                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2780                                            true);
2781         if (rc) {
2782                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2783                 LOAD_ERROR_EXIT(bp, load_error3);
2784         }
2785
2786         if (IS_PF(bp) && bp->pending_max) {
2787                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2788                 bp->pending_max = 0;
2789         }
2790
2791         if (bp->port.pmf) {
2792                 rc = bnx2x_initial_phy_init(bp, load_mode);
2793                 if (rc)
2794                         LOAD_ERROR_EXIT(bp, load_error3);
2795         }
2796         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2797
2798         /* Start fast path */
2799
2800         /* Initialize Rx filter. */
2801         bnx2x_set_rx_mode_inner(bp);
2802
2803         /* Start the Tx */
2804         switch (load_mode) {
2805         case LOAD_NORMAL:
2806                 /* Tx queue should be only re-enabled */
2807                 netif_tx_wake_all_queues(bp->dev);
2808                 break;
2809
2810         case LOAD_OPEN:
2811                 netif_tx_start_all_queues(bp->dev);
2812                 smp_mb__after_atomic();
2813                 break;
2814
2815         case LOAD_DIAG:
2816         case LOAD_LOOPBACK_EXT:
2817                 bp->state = BNX2X_STATE_DIAG;
2818                 break;
2819
2820         default:
2821                 break;
2822         }
2823
2824         if (bp->port.pmf)
2825                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2826         else
2827                 bnx2x__link_status_update(bp);
2828
2829         /* start the timer */
2830         mod_timer(&bp->timer, jiffies + bp->current_interval);
2831
2832         if (CNIC_ENABLED(bp))
2833                 bnx2x_load_cnic(bp);
2834
2835         if (IS_PF(bp))
2836                 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2837
2838         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2839                 /* mark driver is loaded in shmem2 */
2840                 u32 val;
2841                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2842                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2843                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2844                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2845         }
2846
2847         /* Wait for all pending SP commands to complete */
2848         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2849                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2850                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2851                 return -EBUSY;
2852         }
2853
2854         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2855         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2856                 bnx2x_dcbx_init(bp, false);
2857
2858         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2859
2860         return 0;
2861
2862 #ifndef BNX2X_STOP_ON_ERROR
2863 load_error3:
2864         if (IS_PF(bp)) {
2865                 bnx2x_int_disable_sync(bp, 1);
2866
2867                 /* Clean queueable objects */
2868                 bnx2x_squeeze_objects(bp);
2869         }
2870
2871         /* Free SKBs, SGEs, TPA pool and driver internals */
2872         bnx2x_free_skbs(bp);
2873         for_each_rx_queue(bp, i)
2874                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2875
2876         /* Release IRQs */
2877         bnx2x_free_irq(bp);
2878 load_error2:
2879         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2880                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2881                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2882         }
2883
2884         bp->port.pmf = 0;
2885 load_error1:
2886         bnx2x_napi_disable(bp);
2887         bnx2x_del_all_napi(bp);
2888
2889         /* clear pf_load status, as it was already set */
2890         if (IS_PF(bp))
2891                 bnx2x_clear_pf_load(bp);
2892 load_error0:
2893         bnx2x_free_fw_stats_mem(bp);
2894         bnx2x_free_fp_mem(bp);
2895         bnx2x_free_mem(bp);
2896
2897         return rc;
2898 #endif /* ! BNX2X_STOP_ON_ERROR */
2899 }
2900
2901 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2902 {
2903         u8 rc = 0, cos, i;
2904
2905         /* Wait until tx fastpath tasks complete */
2906         for_each_tx_queue(bp, i) {
2907                 struct bnx2x_fastpath *fp = &bp->fp[i];
2908
2909                 for_each_cos_in_tx_queue(fp, cos)
2910                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2911                 if (rc)
2912                         return rc;
2913         }
2914         return 0;
2915 }
2916
2917 /* must be called with rtnl_lock */
2918 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2919 {
2920         int i;
2921         bool global = false;
2922
2923         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2924
2925         /* mark driver is unloaded in shmem2 */
2926         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2927                 u32 val;
2928                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2929                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2930                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2931         }
2932
2933         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2934             (bp->state == BNX2X_STATE_CLOSED ||
2935              bp->state == BNX2X_STATE_ERROR)) {
2936                 /* We can get here if the driver has been unloaded
2937                  * during parity error recovery and is either waiting for a
2938                  * leader to complete or for other functions to unload and
2939                  * then ifdown has been issued. In this case we want to
2940                  * unload and let other functions to complete a recovery
2941                  * process.
2942                  */
2943                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2944                 bp->is_leader = 0;
2945                 bnx2x_release_leader_lock(bp);
2946                 smp_mb();
2947
2948                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2949                 BNX2X_ERR("Can't unload in closed or error state\n");
2950                 return -EINVAL;
2951         }
2952
2953         /* Nothing to do during unload if previous bnx2x_nic_load()
2954          * have not completed successfully - all resources are released.
2955          *
2956          * we can get here only after unsuccessful ndo_* callback, during which
2957          * dev->IFF_UP flag is still on.
2958          */
2959         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2960                 return 0;
2961
2962         /* It's important to set the bp->state to the value different from
2963          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2964          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2965          */
2966         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2967         smp_mb();
2968
2969         /* indicate to VFs that the PF is going down */
2970         bnx2x_iov_channel_down(bp);
2971
2972         if (CNIC_LOADED(bp))
2973                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2974
2975         /* Stop Tx */
2976         bnx2x_tx_disable(bp);
2977         netdev_reset_tc(bp->dev);
2978
2979         bp->rx_mode = BNX2X_RX_MODE_NONE;
2980
2981         del_timer_sync(&bp->timer);
2982
2983         if (IS_PF(bp)) {
2984                 /* Set ALWAYS_ALIVE bit in shmem */
2985                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2986                 bnx2x_drv_pulse(bp);
2987                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2988                 bnx2x_save_statistics(bp);
2989         }
2990
2991         /* wait till consumers catch up with producers in all queues */
2992         bnx2x_drain_tx_queues(bp);
2993
2994         /* if VF indicate to PF this function is going down (PF will delete sp
2995          * elements and clear initializations
2996          */
2997         if (IS_VF(bp))
2998                 bnx2x_vfpf_close_vf(bp);
2999         else if (unload_mode != UNLOAD_RECOVERY)
3000                 /* if this is a normal/close unload need to clean up chip*/
3001                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3002         else {
3003                 /* Send the UNLOAD_REQUEST to the MCP */
3004                 bnx2x_send_unload_req(bp, unload_mode);
3005
3006                 /* Prevent transactions to host from the functions on the
3007                  * engine that doesn't reset global blocks in case of global
3008                  * attention once global blocks are reset and gates are opened
3009                  * (the engine which leader will perform the recovery
3010                  * last).
3011                  */
3012                 if (!CHIP_IS_E1x(bp))
3013                         bnx2x_pf_disable(bp);
3014
3015                 /* Disable HW interrupts, NAPI */
3016                 bnx2x_netif_stop(bp, 1);
3017                 /* Delete all NAPI objects */
3018                 bnx2x_del_all_napi(bp);
3019                 if (CNIC_LOADED(bp))
3020                         bnx2x_del_all_napi_cnic(bp);
3021                 /* Release IRQs */
3022                 bnx2x_free_irq(bp);
3023
3024                 /* Report UNLOAD_DONE to MCP */
3025                 bnx2x_send_unload_done(bp, false);
3026         }
3027
3028         /*
3029          * At this stage no more interrupts will arrive so we may safely clean
3030          * the queueable objects here in case they failed to get cleaned so far.
3031          */
3032         if (IS_PF(bp))
3033                 bnx2x_squeeze_objects(bp);
3034
3035         /* There should be no more pending SP commands at this stage */
3036         bp->sp_state = 0;
3037
3038         bp->port.pmf = 0;
3039
3040         /* clear pending work in rtnl task */
3041         bp->sp_rtnl_state = 0;
3042         smp_mb();
3043
3044         /* Free SKBs, SGEs, TPA pool and driver internals */
3045         bnx2x_free_skbs(bp);
3046         if (CNIC_LOADED(bp))
3047                 bnx2x_free_skbs_cnic(bp);
3048         for_each_rx_queue(bp, i)
3049                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3050
3051         bnx2x_free_fp_mem(bp);
3052         if (CNIC_LOADED(bp))
3053                 bnx2x_free_fp_mem_cnic(bp);
3054
3055         if (IS_PF(bp)) {
3056                 if (CNIC_LOADED(bp))
3057                         bnx2x_free_mem_cnic(bp);
3058         }
3059         bnx2x_free_mem(bp);
3060
3061         bp->state = BNX2X_STATE_CLOSED;
3062         bp->cnic_loaded = false;
3063
3064         /* Clear driver version indication in shmem */
3065         if (IS_PF(bp))
3066                 bnx2x_update_mng_version(bp);
3067
3068         /* Check if there are pending parity attentions. If there are - set
3069          * RECOVERY_IN_PROGRESS.
3070          */
3071         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3072                 bnx2x_set_reset_in_progress(bp);
3073
3074                 /* Set RESET_IS_GLOBAL if needed */
3075                 if (global)
3076                         bnx2x_set_reset_global(bp);
3077         }
3078
3079         /* The last driver must disable a "close the gate" if there is no
3080          * parity attention or "process kill" pending.
3081          */
3082         if (IS_PF(bp) &&
3083             !bnx2x_clear_pf_load(bp) &&
3084             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3085                 bnx2x_disable_close_the_gate(bp);
3086
3087         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3088
3089         return 0;
3090 }
3091
3092 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3093 {
3094         u16 pmcsr;
3095
3096         /* If there is no power capability, silently succeed */
3097         if (!bp->pdev->pm_cap) {
3098                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3099                 return 0;
3100         }
3101
3102         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3103
3104         switch (state) {
3105         case PCI_D0:
3106                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3107                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3108                                        PCI_PM_CTRL_PME_STATUS));
3109
3110                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3111                         /* delay required during transition out of D3hot */
3112                         msleep(20);
3113                 break;
3114
3115         case PCI_D3hot:
3116                 /* If there are other clients above don't
3117                    shut down the power */
3118                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3119                         return 0;
3120                 /* Don't shut down the power for emulation and FPGA */
3121                 if (CHIP_REV_IS_SLOW(bp))
3122                         return 0;
3123
3124                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3125                 pmcsr |= 3;
3126
3127                 if (bp->wol)
3128                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3129
3130                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3131                                       pmcsr);
3132
3133                 /* No more memory access after this point until
3134                 * device is brought back to D0.
3135                 */
3136                 break;
3137
3138         default:
3139                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3140                 return -EINVAL;
3141         }
3142         return 0;
3143 }
3144
3145 /*
3146  * net_device service functions
3147  */
3148 static int bnx2x_poll(struct napi_struct *napi, int budget)
3149 {
3150         int work_done = 0;
3151         u8 cos;
3152         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3153                                                  napi);
3154         struct bnx2x *bp = fp->bp;
3155
3156         while (1) {
3157 #ifdef BNX2X_STOP_ON_ERROR
3158                 if (unlikely(bp->panic)) {
3159                         napi_complete(napi);
3160                         return 0;
3161                 }
3162 #endif
3163                 if (!bnx2x_fp_lock_napi(fp))
3164                         return work_done;
3165
3166                 for_each_cos_in_tx_queue(fp, cos)
3167                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3168                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3169
3170                 if (bnx2x_has_rx_work(fp)) {
3171                         work_done += bnx2x_rx_int(fp, budget - work_done);
3172
3173                         /* must not complete if we consumed full budget */
3174                         if (work_done >= budget) {
3175                                 bnx2x_fp_unlock_napi(fp);
3176                                 break;
3177                         }
3178                 }
3179
3180                 /* Fall out from the NAPI loop if needed */
3181                 if (!bnx2x_fp_unlock_napi(fp) &&
3182                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3183
3184                         /* No need to update SB for FCoE L2 ring as long as
3185                          * it's connected to the default SB and the SB
3186                          * has been updated when NAPI was scheduled.
3187                          */
3188                         if (IS_FCOE_FP(fp)) {
3189                                 napi_complete(napi);
3190                                 break;
3191                         }
3192                         bnx2x_update_fpsb_idx(fp);
3193                         /* bnx2x_has_rx_work() reads the status block,
3194                          * thus we need to ensure that status block indices
3195                          * have been actually read (bnx2x_update_fpsb_idx)
3196                          * prior to this check (bnx2x_has_rx_work) so that
3197                          * we won't write the "newer" value of the status block
3198                          * to IGU (if there was a DMA right after
3199                          * bnx2x_has_rx_work and if there is no rmb, the memory
3200                          * reading (bnx2x_update_fpsb_idx) may be postponed
3201                          * to right before bnx2x_ack_sb). In this case there
3202                          * will never be another interrupt until there is
3203                          * another update of the status block, while there
3204                          * is still unhandled work.
3205                          */
3206                         rmb();
3207
3208                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3209                                 napi_complete(napi);
3210                                 /* Re-enable interrupts */
3211                                 DP(NETIF_MSG_RX_STATUS,
3212                                    "Update index to %d\n", fp->fp_hc_idx);
3213                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3214                                              le16_to_cpu(fp->fp_hc_idx),
3215                                              IGU_INT_ENABLE, 1);
3216                                 break;
3217                         }
3218                 }
3219         }
3220
3221         return work_done;
3222 }
3223
3224 #ifdef CONFIG_NET_RX_BUSY_POLL
3225 /* must be called with local_bh_disable()d */
3226 int bnx2x_low_latency_recv(struct napi_struct *napi)
3227 {
3228         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3229                                                  napi);
3230         struct bnx2x *bp = fp->bp;
3231         int found = 0;
3232
3233         if ((bp->state == BNX2X_STATE_CLOSED) ||
3234             (bp->state == BNX2X_STATE_ERROR) ||
3235             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3236                 return LL_FLUSH_FAILED;
3237
3238         if (!bnx2x_fp_lock_poll(fp))
3239                 return LL_FLUSH_BUSY;
3240
3241         if (bnx2x_has_rx_work(fp))
3242                 found = bnx2x_rx_int(fp, 4);
3243
3244         bnx2x_fp_unlock_poll(fp);
3245
3246         return found;
3247 }
3248 #endif
3249
3250 /* we split the first BD into headers and data BDs
3251  * to ease the pain of our fellow microcode engineers
3252  * we use one mapping for both BDs
3253  */
3254 static u16 bnx2x_tx_split(struct bnx2x *bp,
3255                           struct bnx2x_fp_txdata *txdata,
3256                           struct sw_tx_bd *tx_buf,
3257                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3258                           u16 bd_prod)
3259 {
3260         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3261         struct eth_tx_bd *d_tx_bd;
3262         dma_addr_t mapping;
3263         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3264
3265         /* first fix first BD */
3266         h_tx_bd->nbytes = cpu_to_le16(hlen);
3267
3268         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3269            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3270
3271         /* now get a new data BD
3272          * (after the pbd) and fill it */
3273         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3274         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3275
3276         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3277                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3278
3279         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3280         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3281         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3282
3283         /* this marks the BD as one that has no individual mapping */
3284         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3285
3286         DP(NETIF_MSG_TX_QUEUED,
3287            "TSO split data size is %d (%x:%x)\n",
3288            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3289
3290         /* update tx_bd */
3291         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3292
3293         return bd_prod;
3294 }
3295
3296 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3297 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3298 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3299 {
3300         __sum16 tsum = (__force __sum16) csum;
3301
3302         if (fix > 0)
3303                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3304                                   csum_partial(t_header - fix, fix, 0)));
3305
3306         else if (fix < 0)
3307                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3308                                   csum_partial(t_header, -fix, 0)));
3309
3310         return bswab16(tsum);
3311 }
3312
3313 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3314 {
3315         u32 rc;
3316         __u8 prot = 0;
3317         __be16 protocol;
3318
3319         if (skb->ip_summed != CHECKSUM_PARTIAL)
3320                 return XMIT_PLAIN;
3321
3322         protocol = vlan_get_protocol(skb);
3323         if (protocol == htons(ETH_P_IPV6)) {
3324                 rc = XMIT_CSUM_V6;
3325                 prot = ipv6_hdr(skb)->nexthdr;
3326         } else {
3327                 rc = XMIT_CSUM_V4;
3328                 prot = ip_hdr(skb)->protocol;
3329         }
3330
3331         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3332                 if (inner_ip_hdr(skb)->version == 6) {
3333                         rc |= XMIT_CSUM_ENC_V6;
3334                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3335                                 rc |= XMIT_CSUM_TCP;
3336                 } else {
3337                         rc |= XMIT_CSUM_ENC_V4;
3338                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3339                                 rc |= XMIT_CSUM_TCP;
3340                 }
3341         }
3342         if (prot == IPPROTO_TCP)
3343                 rc |= XMIT_CSUM_TCP;
3344
3345         if (skb_is_gso(skb)) {
3346                 if (skb_is_gso_v6(skb)) {
3347                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3348                         if (rc & XMIT_CSUM_ENC)
3349                                 rc |= XMIT_GSO_ENC_V6;
3350                 } else {
3351                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3352                         if (rc & XMIT_CSUM_ENC)
3353                                 rc |= XMIT_GSO_ENC_V4;
3354                 }
3355         }
3356
3357         return rc;
3358 }
3359
3360 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3361 /* check if packet requires linearization (packet is too fragmented)
3362    no need to check fragmentation if page size > 8K (there will be no
3363    violation to FW restrictions) */
3364 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3365                              u32 xmit_type)
3366 {
3367         int to_copy = 0;
3368         int hlen = 0;
3369         int first_bd_sz = 0;
3370
3371         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3372         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3373
3374                 if (xmit_type & XMIT_GSO) {
3375                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3376                         /* Check if LSO packet needs to be copied:
3377                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3378                         int wnd_size = MAX_FETCH_BD - 3;
3379                         /* Number of windows to check */
3380                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3381                         int wnd_idx = 0;
3382                         int frag_idx = 0;
3383                         u32 wnd_sum = 0;
3384
3385                         /* Headers length */
3386                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3387                                 tcp_hdrlen(skb);
3388
3389                         /* Amount of data (w/o headers) on linear part of SKB*/
3390                         first_bd_sz = skb_headlen(skb) - hlen;
3391
3392                         wnd_sum  = first_bd_sz;
3393
3394                         /* Calculate the first sum - it's special */
3395                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3396                                 wnd_sum +=
3397                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3398
3399                         /* If there was data on linear skb data - check it */
3400                         if (first_bd_sz > 0) {
3401                                 if (unlikely(wnd_sum < lso_mss)) {
3402                                         to_copy = 1;
3403                                         goto exit_lbl;
3404                                 }
3405
3406                                 wnd_sum -= first_bd_sz;
3407                         }
3408
3409                         /* Others are easier: run through the frag list and
3410                            check all windows */
3411                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3412                                 wnd_sum +=
3413                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3414
3415                                 if (unlikely(wnd_sum < lso_mss)) {
3416                                         to_copy = 1;
3417                                         break;
3418                                 }
3419                                 wnd_sum -=
3420                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3421                         }
3422                 } else {
3423                         /* in non-LSO too fragmented packet should always
3424                            be linearized */
3425                         to_copy = 1;
3426                 }
3427         }
3428
3429 exit_lbl:
3430         if (unlikely(to_copy))
3431                 DP(NETIF_MSG_TX_QUEUED,
3432                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3433                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3434                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3435
3436         return to_copy;
3437 }
3438 #endif
3439
3440 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3441                                  u32 xmit_type)
3442 {
3443         struct ipv6hdr *ipv6;
3444
3445         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3446                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3447                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3448
3449         if (xmit_type & XMIT_GSO_ENC_V6)
3450                 ipv6 = inner_ipv6_hdr(skb);
3451         else if (xmit_type & XMIT_GSO_V6)
3452                 ipv6 = ipv6_hdr(skb);
3453         else
3454                 ipv6 = NULL;
3455
3456         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3457                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3458 }
3459
3460 /**
3461  * bnx2x_set_pbd_gso - update PBD in GSO case.
3462  *
3463  * @skb:        packet skb
3464  * @pbd:        parse BD
3465  * @xmit_type:  xmit flags
3466  */
3467 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3468                               struct eth_tx_parse_bd_e1x *pbd,
3469                               struct eth_tx_start_bd *tx_start_bd,
3470                               u32 xmit_type)
3471 {
3472         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3473         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3474         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3475
3476         if (xmit_type & XMIT_GSO_V4) {
3477                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3478                 pbd->tcp_pseudo_csum =
3479                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3480                                                    ip_hdr(skb)->daddr,
3481                                                    0, IPPROTO_TCP, 0));
3482
3483                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3484                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3485         } else {
3486                 pbd->tcp_pseudo_csum =
3487                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3488                                                  &ipv6_hdr(skb)->daddr,
3489                                                  0, IPPROTO_TCP, 0));
3490         }
3491
3492         pbd->global_data |=
3493                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3494 }
3495
3496 /**
3497  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3498  *
3499  * @bp:                 driver handle
3500  * @skb:                packet skb
3501  * @parsing_data:       data to be updated
3502  * @xmit_type:          xmit flags
3503  *
3504  * 57712/578xx related, when skb has encapsulation
3505  */
3506 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3507                                  u32 *parsing_data, u32 xmit_type)
3508 {
3509         *parsing_data |=
3510                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3511                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3512                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3513
3514         if (xmit_type & XMIT_CSUM_TCP) {
3515                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3516                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3517                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3518
3519                 return skb_inner_transport_header(skb) +
3520                         inner_tcp_hdrlen(skb) - skb->data;
3521         }
3522
3523         /* We support checksum offload for TCP and UDP only.
3524          * No need to pass the UDP header length - it's a constant.
3525          */
3526         return skb_inner_transport_header(skb) +
3527                 sizeof(struct udphdr) - skb->data;
3528 }
3529
3530 /**
3531  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3532  *
3533  * @bp:                 driver handle
3534  * @skb:                packet skb
3535  * @parsing_data:       data to be updated
3536  * @xmit_type:          xmit flags
3537  *
3538  * 57712/578xx related
3539  */
3540 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3541                                 u32 *parsing_data, u32 xmit_type)
3542 {
3543         *parsing_data |=
3544                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3545                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3546                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3547
3548         if (xmit_type & XMIT_CSUM_TCP) {
3549                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3550                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3551                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3552
3553                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3554         }
3555         /* We support checksum offload for TCP and UDP only.
3556          * No need to pass the UDP header length - it's a constant.
3557          */
3558         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3559 }
3560
3561 /* set FW indication according to inner or outer protocols if tunneled */
3562 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3563                                struct eth_tx_start_bd *tx_start_bd,
3564                                u32 xmit_type)
3565 {
3566         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3567
3568         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3569                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3570
3571         if (!(xmit_type & XMIT_CSUM_TCP))
3572                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3573 }
3574
3575 /**
3576  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3577  *
3578  * @bp:         driver handle
3579  * @skb:        packet skb
3580  * @pbd:        parse BD to be updated
3581  * @xmit_type:  xmit flags
3582  */
3583 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3584                              struct eth_tx_parse_bd_e1x *pbd,
3585                              u32 xmit_type)
3586 {
3587         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3588
3589         /* for now NS flag is not used in Linux */
3590         pbd->global_data =
3591                 cpu_to_le16(hlen |
3592                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3593                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3594
3595         pbd->ip_hlen_w = (skb_transport_header(skb) -
3596                         skb_network_header(skb)) >> 1;
3597
3598         hlen += pbd->ip_hlen_w;
3599
3600         /* We support checksum offload for TCP and UDP only */
3601         if (xmit_type & XMIT_CSUM_TCP)
3602                 hlen += tcp_hdrlen(skb) / 2;
3603         else
3604                 hlen += sizeof(struct udphdr) / 2;
3605
3606         pbd->total_hlen_w = cpu_to_le16(hlen);
3607         hlen = hlen*2;
3608
3609         if (xmit_type & XMIT_CSUM_TCP) {
3610                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3611
3612         } else {
3613                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3614
3615                 DP(NETIF_MSG_TX_QUEUED,
3616                    "hlen %d  fix %d  csum before fix %x\n",
3617                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3618
3619                 /* HW bug: fixup the CSUM */
3620                 pbd->tcp_pseudo_csum =
3621                         bnx2x_csum_fix(skb_transport_header(skb),
3622                                        SKB_CS(skb), fix);
3623
3624                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3625                    pbd->tcp_pseudo_csum);
3626         }
3627
3628         return hlen;
3629 }
3630
3631 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3632                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3633                                       struct eth_tx_parse_2nd_bd *pbd2,
3634                                       u16 *global_data,
3635                                       u32 xmit_type)
3636 {
3637         u16 hlen_w = 0;
3638         u8 outerip_off, outerip_len = 0;
3639
3640         /* from outer IP to transport */
3641         hlen_w = (skb_inner_transport_header(skb) -
3642                   skb_network_header(skb)) >> 1;
3643
3644         /* transport len */
3645         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3646
3647         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3648
3649         /* outer IP header info */
3650         if (xmit_type & XMIT_CSUM_V4) {
3651                 struct iphdr *iph = ip_hdr(skb);
3652                 u32 csum = (__force u32)(~iph->check) -
3653                            (__force u32)iph->tot_len -
3654                            (__force u32)iph->frag_off;
3655
3656                 pbd2->fw_ip_csum_wo_len_flags_frag =
3657                         bswab16(csum_fold((__force __wsum)csum));
3658         } else {
3659                 pbd2->fw_ip_hdr_to_payload_w =
3660                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3661         }
3662
3663         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3664
3665         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3666
3667         if (xmit_type & XMIT_GSO_V4) {
3668                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3669
3670                 pbd_e2->data.tunnel_data.pseudo_csum =
3671                         bswab16(~csum_tcpudp_magic(
3672                                         inner_ip_hdr(skb)->saddr,
3673                                         inner_ip_hdr(skb)->daddr,
3674                                         0, IPPROTO_TCP, 0));
3675
3676                 outerip_len = ip_hdr(skb)->ihl << 1;
3677         } else {
3678                 pbd_e2->data.tunnel_data.pseudo_csum =
3679                         bswab16(~csum_ipv6_magic(
3680                                         &inner_ipv6_hdr(skb)->saddr,
3681                                         &inner_ipv6_hdr(skb)->daddr,
3682                                         0, IPPROTO_TCP, 0));
3683         }
3684
3685         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3686
3687         *global_data |=
3688                 outerip_off |
3689                 (!!(xmit_type & XMIT_CSUM_V6) <<
3690                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3691                 (outerip_len <<
3692                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3693                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3694                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3695
3696         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3697                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3698                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3699         }
3700 }
3701
3702 /* called with netif_tx_lock
3703  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3704  * netif_wake_queue()
3705  */
3706 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3707 {
3708         struct bnx2x *bp = netdev_priv(dev);
3709
3710         struct netdev_queue *txq;
3711         struct bnx2x_fp_txdata *txdata;
3712         struct sw_tx_bd *tx_buf;
3713         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3714         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3715         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3716         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3717         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3718         u32 pbd_e2_parsing_data = 0;
3719         u16 pkt_prod, bd_prod;
3720         int nbd, txq_index;
3721         dma_addr_t mapping;
3722         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3723         int i;
3724         u8 hlen = 0;
3725         __le16 pkt_size = 0;
3726         struct ethhdr *eth;
3727         u8 mac_type = UNICAST_ADDRESS;
3728
3729 #ifdef BNX2X_STOP_ON_ERROR
3730         if (unlikely(bp->panic))
3731                 return NETDEV_TX_BUSY;
3732 #endif
3733
3734         txq_index = skb_get_queue_mapping(skb);
3735         txq = netdev_get_tx_queue(dev, txq_index);
3736
3737         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3738
3739         txdata = &bp->bnx2x_txq[txq_index];
3740
3741         /* enable this debug print to view the transmission queue being used
3742         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3743            txq_index, fp_index, txdata_index); */
3744
3745         /* enable this debug print to view the transmission details
3746         DP(NETIF_MSG_TX_QUEUED,
3747            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3748            txdata->cid, fp_index, txdata_index, txdata, fp); */
3749
3750         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3751                         skb_shinfo(skb)->nr_frags +
3752                         BDS_PER_TX_PKT +
3753                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3754                 /* Handle special storage cases separately */
3755                 if (txdata->tx_ring_size == 0) {
3756                         struct bnx2x_eth_q_stats *q_stats =
3757                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3758                         q_stats->driver_filtered_tx_pkt++;
3759                         dev_kfree_skb(skb);
3760                         return NETDEV_TX_OK;
3761                 }
3762                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3763                 netif_tx_stop_queue(txq);
3764                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3765
3766                 return NETDEV_TX_BUSY;
3767         }
3768
3769         DP(NETIF_MSG_TX_QUEUED,
3770            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3771            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3772            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3773            skb->len);
3774
3775         eth = (struct ethhdr *)skb->data;
3776
3777         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3778         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3779                 if (is_broadcast_ether_addr(eth->h_dest))
3780                         mac_type = BROADCAST_ADDRESS;
3781                 else
3782                         mac_type = MULTICAST_ADDRESS;
3783         }
3784
3785 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3786         /* First, check if we need to linearize the skb (due to FW
3787            restrictions). No need to check fragmentation if page size > 8K
3788            (there will be no violation to FW restrictions) */
3789         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3790                 /* Statistics of linearization */
3791                 bp->lin_cnt++;
3792                 if (skb_linearize(skb) != 0) {
3793                         DP(NETIF_MSG_TX_QUEUED,
3794                            "SKB linearization failed - silently dropping this SKB\n");
3795                         dev_kfree_skb_any(skb);
3796                         return NETDEV_TX_OK;
3797                 }
3798         }
3799 #endif
3800         /* Map skb linear data for DMA */
3801         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3802                                  skb_headlen(skb), DMA_TO_DEVICE);
3803         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3804                 DP(NETIF_MSG_TX_QUEUED,
3805                    "SKB mapping failed - silently dropping this SKB\n");
3806                 dev_kfree_skb_any(skb);
3807                 return NETDEV_TX_OK;
3808         }
3809         /*
3810         Please read carefully. First we use one BD which we mark as start,
3811         then we have a parsing info BD (used for TSO or xsum),
3812         and only then we have the rest of the TSO BDs.
3813         (don't forget to mark the last one as last,
3814         and to unmap only AFTER you write to the BD ...)
3815         And above all, all pdb sizes are in words - NOT DWORDS!
3816         */
3817
3818         /* get current pkt produced now - advance it just before sending packet
3819          * since mapping of pages may fail and cause packet to be dropped
3820          */
3821         pkt_prod = txdata->tx_pkt_prod;
3822         bd_prod = TX_BD(txdata->tx_bd_prod);
3823
3824         /* get a tx_buf and first BD
3825          * tx_start_bd may be changed during SPLIT,
3826          * but first_bd will always stay first
3827          */
3828         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3829         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3830         first_bd = tx_start_bd;
3831
3832         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3833
3834         /* header nbd: indirectly zero other flags! */
3835         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3836
3837         /* remember the first BD of the packet */
3838         tx_buf->first_bd = txdata->tx_bd_prod;
3839         tx_buf->skb = skb;
3840         tx_buf->flags = 0;
3841
3842         DP(NETIF_MSG_TX_QUEUED,
3843            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3844            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3845
3846         if (vlan_tx_tag_present(skb)) {
3847                 tx_start_bd->vlan_or_ethertype =
3848                     cpu_to_le16(vlan_tx_tag_get(skb));
3849                 tx_start_bd->bd_flags.as_bitfield |=
3850                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3851         } else {
3852                 /* when transmitting in a vf, start bd must hold the ethertype
3853                  * for fw to enforce it
3854                  */
3855                 if (IS_VF(bp))
3856                         tx_start_bd->vlan_or_ethertype =
3857                                 cpu_to_le16(ntohs(eth->h_proto));
3858                 else
3859                         /* used by FW for packet accounting */
3860                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3861         }
3862
3863         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3864
3865         /* turn on parsing and get a BD */
3866         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3867
3868         if (xmit_type & XMIT_CSUM)
3869                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3870
3871         if (!CHIP_IS_E1x(bp)) {
3872                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3873                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3874
3875                 if (xmit_type & XMIT_CSUM_ENC) {
3876                         u16 global_data = 0;
3877
3878                         /* Set PBD in enc checksum offload case */
3879                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3880                                                       &pbd_e2_parsing_data,
3881                                                       xmit_type);
3882
3883                         /* turn on 2nd parsing and get a BD */
3884                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3885
3886                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3887
3888                         memset(pbd2, 0, sizeof(*pbd2));
3889
3890                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3891                                 (skb_inner_network_header(skb) -
3892                                  skb->data) >> 1;
3893
3894                         if (xmit_type & XMIT_GSO_ENC)
3895                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3896                                                           &global_data,
3897                                                           xmit_type);
3898
3899                         pbd2->global_data = cpu_to_le16(global_data);
3900
3901                         /* add addition parse BD indication to start BD */
3902                         SET_FLAG(tx_start_bd->general_data,
3903                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3904                         /* set encapsulation flag in start BD */
3905                         SET_FLAG(tx_start_bd->general_data,
3906                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3907
3908                         tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3909
3910                         nbd++;
3911                 } else if (xmit_type & XMIT_CSUM) {
3912                         /* Set PBD in checksum offload case w/o encapsulation */
3913                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3914                                                      &pbd_e2_parsing_data,
3915                                                      xmit_type);
3916                 }
3917
3918                 /* Add the macs to the parsing BD if this is a vf or if
3919                  * Tx Switching is enabled.
3920                  */
3921                 if (IS_VF(bp)) {
3922                         /* override GRE parameters in BD */
3923                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3924                                               &pbd_e2->data.mac_addr.src_mid,
3925                                               &pbd_e2->data.mac_addr.src_lo,
3926                                               eth->h_source);
3927
3928                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3929                                               &pbd_e2->data.mac_addr.dst_mid,
3930                                               &pbd_e2->data.mac_addr.dst_lo,
3931                                               eth->h_dest);
3932                 } else if (bp->flags & TX_SWITCHING) {
3933                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3934                                               &pbd_e2->data.mac_addr.dst_mid,
3935                                               &pbd_e2->data.mac_addr.dst_lo,
3936                                               eth->h_dest);
3937                 }
3938
3939                 SET_FLAG(pbd_e2_parsing_data,
3940                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3941         } else {
3942                 u16 global_data = 0;
3943                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3944                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3945                 /* Set PBD in checksum offload case */
3946                 if (xmit_type & XMIT_CSUM)
3947                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3948
3949                 SET_FLAG(global_data,
3950                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3951                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3952         }
3953
3954         /* Setup the data pointer of the first BD of the packet */
3955         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3956         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3957         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3958         pkt_size = tx_start_bd->nbytes;
3959
3960         DP(NETIF_MSG_TX_QUEUED,
3961            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3962            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3963            le16_to_cpu(tx_start_bd->nbytes),
3964            tx_start_bd->bd_flags.as_bitfield,
3965            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3966
3967         if (xmit_type & XMIT_GSO) {
3968
3969                 DP(NETIF_MSG_TX_QUEUED,
3970                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3971                    skb->len, hlen, skb_headlen(skb),
3972                    skb_shinfo(skb)->gso_size);
3973
3974                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3975
3976                 if (unlikely(skb_headlen(skb) > hlen)) {
3977                         nbd++;
3978                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3979                                                  &tx_start_bd, hlen,
3980                                                  bd_prod);
3981                 }
3982                 if (!CHIP_IS_E1x(bp))
3983                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3984                                              xmit_type);
3985                 else
3986                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3987         }
3988
3989         /* Set the PBD's parsing_data field if not zero
3990          * (for the chips newer than 57711).
3991          */
3992         if (pbd_e2_parsing_data)
3993                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3994
3995         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3996
3997         /* Handle fragmented skb */
3998         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3999                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4000
4001                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4002                                            skb_frag_size(frag), DMA_TO_DEVICE);
4003                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4004                         unsigned int pkts_compl = 0, bytes_compl = 0;
4005
4006                         DP(NETIF_MSG_TX_QUEUED,
4007                            "Unable to map page - dropping packet...\n");
4008
4009                         /* we need unmap all buffers already mapped
4010                          * for this SKB;
4011                          * first_bd->nbd need to be properly updated
4012                          * before call to bnx2x_free_tx_pkt
4013                          */
4014                         first_bd->nbd = cpu_to_le16(nbd);
4015                         bnx2x_free_tx_pkt(bp, txdata,
4016                                           TX_BD(txdata->tx_pkt_prod),
4017                                           &pkts_compl, &bytes_compl);
4018                         return NETDEV_TX_OK;
4019                 }
4020
4021                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4022                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4023                 if (total_pkt_bd == NULL)
4024                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4025
4026                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4027                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4028                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4029                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
4030                 nbd++;
4031
4032                 DP(NETIF_MSG_TX_QUEUED,
4033                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4034                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4035                    le16_to_cpu(tx_data_bd->nbytes));
4036         }
4037
4038         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4039
4040         /* update with actual num BDs */
4041         first_bd->nbd = cpu_to_le16(nbd);
4042
4043         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4044
4045         /* now send a tx doorbell, counting the next BD
4046          * if the packet contains or ends with it
4047          */
4048         if (TX_BD_POFF(bd_prod) < nbd)
4049                 nbd++;
4050
4051         /* total_pkt_bytes should be set on the first data BD if
4052          * it's not an LSO packet and there is more than one
4053          * data BD. In this case pkt_size is limited by an MTU value.
4054          * However we prefer to set it for an LSO packet (while we don't
4055          * have to) in order to save some CPU cycles in a none-LSO
4056          * case, when we much more care about them.
4057          */
4058         if (total_pkt_bd != NULL)
4059                 total_pkt_bd->total_pkt_bytes = pkt_size;
4060
4061         if (pbd_e1x)
4062                 DP(NETIF_MSG_TX_QUEUED,
4063                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4064                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4065                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4066                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4067                     le16_to_cpu(pbd_e1x->total_hlen_w));
4068         if (pbd_e2)
4069                 DP(NETIF_MSG_TX_QUEUED,
4070                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4071                    pbd_e2,
4072                    pbd_e2->data.mac_addr.dst_hi,
4073                    pbd_e2->data.mac_addr.dst_mid,
4074                    pbd_e2->data.mac_addr.dst_lo,
4075                    pbd_e2->data.mac_addr.src_hi,
4076                    pbd_e2->data.mac_addr.src_mid,
4077                    pbd_e2->data.mac_addr.src_lo,
4078                    pbd_e2->parsing_data);
4079         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4080
4081         netdev_tx_sent_queue(txq, skb->len);
4082
4083         skb_tx_timestamp(skb);
4084
4085         txdata->tx_pkt_prod++;
4086         /*
4087          * Make sure that the BD data is updated before updating the producer
4088          * since FW might read the BD right after the producer is updated.
4089          * This is only applicable for weak-ordered memory model archs such
4090          * as IA-64. The following barrier is also mandatory since FW will
4091          * assumes packets must have BDs.
4092          */
4093         wmb();
4094
4095         txdata->tx_db.data.prod += nbd;
4096         barrier();
4097
4098         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4099
4100         mmiowb();
4101
4102         txdata->tx_bd_prod += nbd;
4103
4104         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4105                 netif_tx_stop_queue(txq);
4106
4107                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4108                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4109                  * fp->bd_tx_cons */
4110                 smp_mb();
4111
4112                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4113                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4114                         netif_tx_wake_queue(txq);
4115         }
4116         txdata->tx_pkt++;
4117
4118         return NETDEV_TX_OK;
4119 }
4120
4121 /**
4122  * bnx2x_setup_tc - routine to configure net_device for multi tc
4123  *
4124  * @netdev: net device to configure
4125  * @tc: number of traffic classes to enable
4126  *
4127  * callback connected to the ndo_setup_tc function pointer
4128  */
4129 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4130 {
4131         int cos, prio, count, offset;
4132         struct bnx2x *bp = netdev_priv(dev);
4133
4134         /* setup tc must be called under rtnl lock */
4135         ASSERT_RTNL();
4136
4137         /* no traffic classes requested. Aborting */
4138         if (!num_tc) {
4139                 netdev_reset_tc(dev);
4140                 return 0;
4141         }
4142
4143         /* requested to support too many traffic classes */
4144         if (num_tc > bp->max_cos) {
4145                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4146                           num_tc, bp->max_cos);
4147                 return -EINVAL;
4148         }
4149
4150         /* declare amount of supported traffic classes */
4151         if (netdev_set_num_tc(dev, num_tc)) {
4152                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4153                 return -EINVAL;
4154         }
4155
4156         /* configure priority to traffic class mapping */
4157         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4158                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4159                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4160                    "mapping priority %d to tc %d\n",
4161                    prio, bp->prio_to_cos[prio]);
4162         }
4163
4164         /* Use this configuration to differentiate tc0 from other COSes
4165            This can be used for ets or pfc, and save the effort of setting
4166            up a multio class queue disc or negotiating DCBX with a switch
4167         netdev_set_prio_tc_map(dev, 0, 0);
4168         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4169         for (prio = 1; prio < 16; prio++) {
4170                 netdev_set_prio_tc_map(dev, prio, 1);
4171                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4172         } */
4173
4174         /* configure traffic class to transmission queue mapping */
4175         for (cos = 0; cos < bp->max_cos; cos++) {
4176                 count = BNX2X_NUM_ETH_QUEUES(bp);
4177                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4178                 netdev_set_tc_queue(dev, cos, count, offset);
4179                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4180                    "mapping tc %d to offset %d count %d\n",
4181                    cos, offset, count);
4182         }
4183
4184         return 0;
4185 }
4186
4187 /* called with rtnl_lock */
4188 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4189 {
4190         struct sockaddr *addr = p;
4191         struct bnx2x *bp = netdev_priv(dev);
4192         int rc = 0;
4193
4194         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4195                 BNX2X_ERR("Requested MAC address is not valid\n");
4196                 return -EINVAL;
4197         }
4198
4199         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4200             !is_zero_ether_addr(addr->sa_data)) {
4201                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4202                 return -EINVAL;
4203         }
4204
4205         if (netif_running(dev))  {
4206                 rc = bnx2x_set_eth_mac(bp, false);
4207                 if (rc)
4208                         return rc;
4209         }
4210
4211         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4212
4213         if (netif_running(dev))
4214                 rc = bnx2x_set_eth_mac(bp, true);
4215
4216         return rc;
4217 }
4218
4219 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4220 {
4221         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4222         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4223         u8 cos;
4224
4225         /* Common */
4226
4227         if (IS_FCOE_IDX(fp_index)) {
4228                 memset(sb, 0, sizeof(union host_hc_status_block));
4229                 fp->status_blk_mapping = 0;
4230         } else {
4231                 /* status blocks */
4232                 if (!CHIP_IS_E1x(bp))
4233                         BNX2X_PCI_FREE(sb->e2_sb,
4234                                        bnx2x_fp(bp, fp_index,
4235                                                 status_blk_mapping),
4236                                        sizeof(struct host_hc_status_block_e2));
4237                 else
4238                         BNX2X_PCI_FREE(sb->e1x_sb,
4239                                        bnx2x_fp(bp, fp_index,
4240                                                 status_blk_mapping),
4241                                        sizeof(struct host_hc_status_block_e1x));
4242         }
4243
4244         /* Rx */
4245         if (!skip_rx_queue(bp, fp_index)) {
4246                 bnx2x_free_rx_bds(fp);
4247
4248                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4249                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4250                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4251                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4252                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4253
4254                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4255                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4256                                sizeof(struct eth_fast_path_rx_cqe) *
4257                                NUM_RCQ_BD);
4258
4259                 /* SGE ring */
4260                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4261                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4262                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4263                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4264         }
4265
4266         /* Tx */
4267         if (!skip_tx_queue(bp, fp_index)) {
4268                 /* fastpath tx rings: tx_buf tx_desc */
4269                 for_each_cos_in_tx_queue(fp, cos) {
4270                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4271
4272                         DP(NETIF_MSG_IFDOWN,
4273                            "freeing tx memory of fp %d cos %d cid %d\n",
4274                            fp_index, cos, txdata->cid);
4275
4276                         BNX2X_FREE(txdata->tx_buf_ring);
4277                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4278                                 txdata->tx_desc_mapping,
4279                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4280                 }
4281         }
4282         /* end of fastpath */
4283 }
4284
4285 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4286 {
4287         int i;
4288         for_each_cnic_queue(bp, i)
4289                 bnx2x_free_fp_mem_at(bp, i);
4290 }
4291
4292 void bnx2x_free_fp_mem(struct bnx2x *bp)
4293 {
4294         int i;
4295         for_each_eth_queue(bp, i)
4296                 bnx2x_free_fp_mem_at(bp, i);
4297 }
4298
4299 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4300 {
4301         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4302         if (!CHIP_IS_E1x(bp)) {
4303                 bnx2x_fp(bp, index, sb_index_values) =
4304                         (__le16 *)status_blk.e2_sb->sb.index_values;
4305                 bnx2x_fp(bp, index, sb_running_index) =
4306                         (__le16 *)status_blk.e2_sb->sb.running_index;
4307         } else {
4308                 bnx2x_fp(bp, index, sb_index_values) =
4309                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4310                 bnx2x_fp(bp, index, sb_running_index) =
4311                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4312         }
4313 }
4314
4315 /* Returns the number of actually allocated BDs */
4316 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4317                               int rx_ring_size)
4318 {
4319         struct bnx2x *bp = fp->bp;
4320         u16 ring_prod, cqe_ring_prod;
4321         int i, failure_cnt = 0;
4322
4323         fp->rx_comp_cons = 0;
4324         cqe_ring_prod = ring_prod = 0;
4325
4326         /* This routine is called only during fo init so
4327          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4328          */
4329         for (i = 0; i < rx_ring_size; i++) {
4330                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4331                         failure_cnt++;
4332                         continue;
4333                 }
4334                 ring_prod = NEXT_RX_IDX(ring_prod);
4335                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4336                 WARN_ON(ring_prod <= (i - failure_cnt));
4337         }
4338
4339         if (failure_cnt)
4340                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4341                           i - failure_cnt, fp->index);
4342
4343         fp->rx_bd_prod = ring_prod;
4344         /* Limit the CQE producer by the CQE ring size */
4345         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4346                                cqe_ring_prod);
4347         fp->rx_pkt = fp->rx_calls = 0;
4348
4349         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4350
4351         return i - failure_cnt;
4352 }
4353
4354 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4355 {
4356         int i;
4357
4358         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4359                 struct eth_rx_cqe_next_page *nextpg;
4360
4361                 nextpg = (struct eth_rx_cqe_next_page *)
4362                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4363                 nextpg->addr_hi =
4364                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4365                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4366                 nextpg->addr_lo =
4367                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4368                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4369         }
4370 }
4371
4372 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4373 {
4374         union host_hc_status_block *sb;
4375         struct bnx2x_fastpath *fp = &bp->fp[index];
4376         int ring_size = 0;
4377         u8 cos;
4378         int rx_ring_size = 0;
4379
4380         if (!bp->rx_ring_size &&
4381             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4382                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4383                 bp->rx_ring_size = rx_ring_size;
4384         } else if (!bp->rx_ring_size) {
4385                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4386
4387                 if (CHIP_IS_E3(bp)) {
4388                         u32 cfg = SHMEM_RD(bp,
4389                                            dev_info.port_hw_config[BP_PORT(bp)].
4390                                            default_cfg);
4391
4392                         /* Decrease ring size for 1G functions */
4393                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4394                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4395                                 rx_ring_size /= 10;
4396                 }
4397
4398                 /* allocate at least number of buffers required by FW */
4399                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4400                                      MIN_RX_SIZE_TPA, rx_ring_size);
4401
4402                 bp->rx_ring_size = rx_ring_size;
4403         } else /* if rx_ring_size specified - use it */
4404                 rx_ring_size = bp->rx_ring_size;
4405
4406         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4407
4408         /* Common */
4409         sb = &bnx2x_fp(bp, index, status_blk);
4410
4411         if (!IS_FCOE_IDX(index)) {
4412                 /* status blocks */
4413                 if (!CHIP_IS_E1x(bp)) {
4414                         sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4415                                                     sizeof(struct host_hc_status_block_e2));
4416                         if (!sb->e2_sb)
4417                                 goto alloc_mem_err;
4418                 } else {
4419                         sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4420                                                      sizeof(struct host_hc_status_block_e1x));
4421                         if (!sb->e1x_sb)
4422                                 goto alloc_mem_err;
4423                 }
4424         }
4425
4426         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4427          * set shortcuts for it.
4428          */
4429         if (!IS_FCOE_IDX(index))
4430                 set_sb_shortcuts(bp, index);
4431
4432         /* Tx */
4433         if (!skip_tx_queue(bp, index)) {
4434                 /* fastpath tx rings: tx_buf tx_desc */
4435                 for_each_cos_in_tx_queue(fp, cos) {
4436                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4437
4438                         DP(NETIF_MSG_IFUP,
4439                            "allocating tx memory of fp %d cos %d\n",
4440                            index, cos);
4441
4442                         txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4443                                                       sizeof(struct sw_tx_bd),
4444                                                       GFP_KERNEL);
4445                         if (!txdata->tx_buf_ring)
4446                                 goto alloc_mem_err;
4447                         txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4448                                                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4449                         if (!txdata->tx_desc_ring)
4450                                 goto alloc_mem_err;
4451                 }
4452         }
4453
4454         /* Rx */
4455         if (!skip_rx_queue(bp, index)) {
4456                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4457                 bnx2x_fp(bp, index, rx_buf_ring) =
4458                         kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4459                 if (!bnx2x_fp(bp, index, rx_buf_ring))
4460                         goto alloc_mem_err;
4461                 bnx2x_fp(bp, index, rx_desc_ring) =
4462                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4463                                         sizeof(struct eth_rx_bd) * NUM_RX_BD);
4464                 if (!bnx2x_fp(bp, index, rx_desc_ring))
4465                         goto alloc_mem_err;
4466
4467                 /* Seed all CQEs by 1s */
4468                 bnx2x_fp(bp, index, rx_comp_ring) =
4469                         BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4470                                          sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4471                 if (!bnx2x_fp(bp, index, rx_comp_ring))
4472                         goto alloc_mem_err;
4473
4474                 /* SGE ring */
4475                 bnx2x_fp(bp, index, rx_page_ring) =
4476                         kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4477                                 GFP_KERNEL);
4478                 if (!bnx2x_fp(bp, index, rx_page_ring))
4479                         goto alloc_mem_err;
4480                 bnx2x_fp(bp, index, rx_sge_ring) =
4481                         BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4482                                         BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4483                 if (!bnx2x_fp(bp, index, rx_sge_ring))
4484                         goto alloc_mem_err;
4485                 /* RX BD ring */
4486                 bnx2x_set_next_page_rx_bd(fp);
4487
4488                 /* CQ ring */
4489                 bnx2x_set_next_page_rx_cq(fp);
4490
4491                 /* BDs */
4492                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4493                 if (ring_size < rx_ring_size)
4494                         goto alloc_mem_err;
4495         }
4496
4497         return 0;
4498
4499 /* handles low memory cases */
4500 alloc_mem_err:
4501         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4502                                                 index, ring_size);
4503         /* FW will drop all packets if queue is not big enough,
4504          * In these cases we disable the queue
4505          * Min size is different for OOO, TPA and non-TPA queues
4506          */
4507         if (ring_size < (fp->disable_tpa ?
4508                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4509                         /* release memory allocated for this queue */
4510                         bnx2x_free_fp_mem_at(bp, index);
4511                         return -ENOMEM;
4512         }
4513         return 0;
4514 }
4515
4516 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4517 {
4518         if (!NO_FCOE(bp))
4519                 /* FCoE */
4520                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4521                         /* we will fail load process instead of mark
4522                          * NO_FCOE_FLAG
4523                          */
4524                         return -ENOMEM;
4525
4526         return 0;
4527 }
4528
4529 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4530 {
4531         int i;
4532
4533         /* 1. Allocate FP for leading - fatal if error
4534          * 2. Allocate RSS - fix number of queues if error
4535          */
4536
4537         /* leading */
4538         if (bnx2x_alloc_fp_mem_at(bp, 0))
4539                 return -ENOMEM;
4540
4541         /* RSS */
4542         for_each_nondefault_eth_queue(bp, i)
4543                 if (bnx2x_alloc_fp_mem_at(bp, i))
4544                         break;
4545
4546         /* handle memory failures */
4547         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4548                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4549
4550                 WARN_ON(delta < 0);
4551                 bnx2x_shrink_eth_fp(bp, delta);
4552                 if (CNIC_SUPPORT(bp))
4553                         /* move non eth FPs next to last eth FP
4554                          * must be done in that order
4555                          * FCOE_IDX < FWD_IDX < OOO_IDX
4556                          */
4557
4558                         /* move FCoE fp even NO_FCOE_FLAG is on */
4559                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4560                 bp->num_ethernet_queues -= delta;
4561                 bp->num_queues = bp->num_ethernet_queues +
4562                                  bp->num_cnic_queues;
4563                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4564                           bp->num_queues + delta, bp->num_queues);
4565         }
4566
4567         return 0;
4568 }
4569
4570 void bnx2x_free_mem_bp(struct bnx2x *bp)
4571 {
4572         int i;
4573
4574         for (i = 0; i < bp->fp_array_size; i++)
4575                 kfree(bp->fp[i].tpa_info);
4576         kfree(bp->fp);
4577         kfree(bp->sp_objs);
4578         kfree(bp->fp_stats);
4579         kfree(bp->bnx2x_txq);
4580         kfree(bp->msix_table);
4581         kfree(bp->ilt);
4582 }
4583
4584 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4585 {
4586         struct bnx2x_fastpath *fp;
4587         struct msix_entry *tbl;
4588         struct bnx2x_ilt *ilt;
4589         int msix_table_size = 0;
4590         int fp_array_size, txq_array_size;
4591         int i;
4592
4593         /*
4594          * The biggest MSI-X table we might need is as a maximum number of fast
4595          * path IGU SBs plus default SB (for PF only).
4596          */
4597         msix_table_size = bp->igu_sb_cnt;
4598         if (IS_PF(bp))
4599                 msix_table_size++;
4600         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4601
4602         /* fp array: RSS plus CNIC related L2 queues */
4603         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4604         bp->fp_array_size = fp_array_size;
4605         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4606
4607         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4608         if (!fp)
4609                 goto alloc_err;
4610         for (i = 0; i < bp->fp_array_size; i++) {
4611                 fp[i].tpa_info =
4612                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4613                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4614                 if (!(fp[i].tpa_info))
4615                         goto alloc_err;
4616         }
4617
4618         bp->fp = fp;
4619
4620         /* allocate sp objs */
4621         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4622                               GFP_KERNEL);
4623         if (!bp->sp_objs)
4624                 goto alloc_err;
4625
4626         /* allocate fp_stats */
4627         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4628                                GFP_KERNEL);
4629         if (!bp->fp_stats)
4630                 goto alloc_err;
4631
4632         /* Allocate memory for the transmission queues array */
4633         txq_array_size =
4634                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4635         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4636
4637         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4638                                 GFP_KERNEL);
4639         if (!bp->bnx2x_txq)
4640                 goto alloc_err;
4641
4642         /* msix table */
4643         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4644         if (!tbl)
4645                 goto alloc_err;
4646         bp->msix_table = tbl;
4647
4648         /* ilt */
4649         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4650         if (!ilt)
4651                 goto alloc_err;
4652         bp->ilt = ilt;
4653
4654         return 0;
4655 alloc_err:
4656         bnx2x_free_mem_bp(bp);
4657         return -ENOMEM;
4658 }
4659
4660 int bnx2x_reload_if_running(struct net_device *dev)
4661 {
4662         struct bnx2x *bp = netdev_priv(dev);
4663
4664         if (unlikely(!netif_running(dev)))
4665                 return 0;
4666
4667         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4668         return bnx2x_nic_load(bp, LOAD_NORMAL);
4669 }
4670
4671 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4672 {
4673         u32 sel_phy_idx = 0;
4674         if (bp->link_params.num_phys <= 1)
4675                 return INT_PHY;
4676
4677         if (bp->link_vars.link_up) {
4678                 sel_phy_idx = EXT_PHY1;
4679                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4680                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4681                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4682                         sel_phy_idx = EXT_PHY2;
4683         } else {
4684
4685                 switch (bnx2x_phy_selection(&bp->link_params)) {
4686                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4687                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4688                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4689                        sel_phy_idx = EXT_PHY1;
4690                        break;
4691                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4692                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4693                        sel_phy_idx = EXT_PHY2;
4694                        break;
4695                 }
4696         }
4697
4698         return sel_phy_idx;
4699 }
4700 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4701 {
4702         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4703         /*
4704          * The selected activated PHY is always after swapping (in case PHY
4705          * swapping is enabled). So when swapping is enabled, we need to reverse
4706          * the configuration
4707          */
4708
4709         if (bp->link_params.multi_phy_config &
4710             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4711                 if (sel_phy_idx == EXT_PHY1)
4712                         sel_phy_idx = EXT_PHY2;
4713                 else if (sel_phy_idx == EXT_PHY2)
4714                         sel_phy_idx = EXT_PHY1;
4715         }
4716         return LINK_CONFIG_IDX(sel_phy_idx);
4717 }
4718
4719 #ifdef NETDEV_FCOE_WWNN
4720 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4721 {
4722         struct bnx2x *bp = netdev_priv(dev);
4723         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4724
4725         switch (type) {
4726         case NETDEV_FCOE_WWNN:
4727                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4728                                 cp->fcoe_wwn_node_name_lo);
4729                 break;
4730         case NETDEV_FCOE_WWPN:
4731                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4732                                 cp->fcoe_wwn_port_name_lo);
4733                 break;
4734         default:
4735                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4736                 return -EINVAL;
4737         }
4738
4739         return 0;
4740 }
4741 #endif
4742
4743 /* called with rtnl_lock */
4744 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4745 {
4746         struct bnx2x *bp = netdev_priv(dev);
4747
4748         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4749                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4750                 return -EAGAIN;
4751         }
4752
4753         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4754             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4755                 BNX2X_ERR("Can't support requested MTU size\n");
4756                 return -EINVAL;
4757         }
4758
4759         /* This does not race with packet allocation
4760          * because the actual alloc size is
4761          * only updated as part of load
4762          */
4763         dev->mtu = new_mtu;
4764
4765         return bnx2x_reload_if_running(dev);
4766 }
4767
4768 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4769                                      netdev_features_t features)
4770 {
4771         struct bnx2x *bp = netdev_priv(dev);
4772
4773         /* TPA requires Rx CSUM offloading */
4774         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4775                 features &= ~NETIF_F_LRO;
4776                 features &= ~NETIF_F_GRO;
4777         }
4778
4779         return features;
4780 }
4781
4782 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4783 {
4784         struct bnx2x *bp = netdev_priv(dev);
4785         u32 flags = bp->flags;
4786         u32 changes;
4787         bool bnx2x_reload = false;
4788
4789         if (features & NETIF_F_LRO)
4790                 flags |= TPA_ENABLE_FLAG;
4791         else
4792                 flags &= ~TPA_ENABLE_FLAG;
4793
4794         if (features & NETIF_F_GRO)
4795                 flags |= GRO_ENABLE_FLAG;
4796         else
4797                 flags &= ~GRO_ENABLE_FLAG;
4798
4799         if (features & NETIF_F_LOOPBACK) {
4800                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4801                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4802                         bnx2x_reload = true;
4803                 }
4804         } else {
4805                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4806                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4807                         bnx2x_reload = true;
4808                 }
4809         }
4810
4811         changes = flags ^ bp->flags;
4812
4813         /* if GRO is changed while LRO is enabled, don't force a reload */
4814         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4815                 changes &= ~GRO_ENABLE_FLAG;
4816
4817         if (changes)
4818                 bnx2x_reload = true;
4819
4820         bp->flags = flags;
4821
4822         if (bnx2x_reload) {
4823                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4824                         return bnx2x_reload_if_running(dev);
4825                 /* else: bnx2x_nic_load() will be called at end of recovery */
4826         }
4827
4828         return 0;
4829 }
4830
4831 void bnx2x_tx_timeout(struct net_device *dev)
4832 {
4833         struct bnx2x *bp = netdev_priv(dev);
4834
4835 #ifdef BNX2X_STOP_ON_ERROR
4836         if (!bp->panic)
4837                 bnx2x_panic();
4838 #endif
4839
4840         /* This allows the netif to be shutdown gracefully before resetting */
4841         bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4842 }
4843
4844 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4845 {
4846         struct net_device *dev = pci_get_drvdata(pdev);
4847         struct bnx2x *bp;
4848
4849         if (!dev) {
4850                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4851                 return -ENODEV;
4852         }
4853         bp = netdev_priv(dev);
4854
4855         rtnl_lock();
4856
4857         pci_save_state(pdev);
4858
4859         if (!netif_running(dev)) {
4860                 rtnl_unlock();
4861                 return 0;
4862         }
4863
4864         netif_device_detach(dev);
4865
4866         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4867
4868         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4869
4870         rtnl_unlock();
4871
4872         return 0;
4873 }
4874
4875 int bnx2x_resume(struct pci_dev *pdev)
4876 {
4877         struct net_device *dev = pci_get_drvdata(pdev);
4878         struct bnx2x *bp;
4879         int rc;
4880
4881         if (!dev) {
4882                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4883                 return -ENODEV;
4884         }
4885         bp = netdev_priv(dev);
4886
4887         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4888                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4889                 return -EAGAIN;
4890         }
4891
4892         rtnl_lock();
4893
4894         pci_restore_state(pdev);
4895
4896         if (!netif_running(dev)) {
4897                 rtnl_unlock();
4898                 return 0;
4899         }
4900
4901         bnx2x_set_power_state(bp, PCI_D0);
4902         netif_device_attach(dev);
4903
4904         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4905
4906         rtnl_unlock();
4907
4908         return rc;
4909 }
4910
4911 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4912                               u32 cid)
4913 {
4914         if (!cxt) {
4915                 BNX2X_ERR("bad context pointer %p\n", cxt);
4916                 return;
4917         }
4918
4919         /* ustorm cxt validation */
4920         cxt->ustorm_ag_context.cdu_usage =
4921                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4922                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4923         /* xcontext validation */
4924         cxt->xstorm_ag_context.cdu_reserved =
4925                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4926                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4927 }
4928
4929 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4930                                     u8 fw_sb_id, u8 sb_index,
4931                                     u8 ticks)
4932 {
4933         u32 addr = BAR_CSTRORM_INTMEM +
4934                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4935         REG_WR8(bp, addr, ticks);
4936         DP(NETIF_MSG_IFUP,
4937            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4938            port, fw_sb_id, sb_index, ticks);
4939 }
4940
4941 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4942                                     u16 fw_sb_id, u8 sb_index,
4943                                     u8 disable)
4944 {
4945         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4946         u32 addr = BAR_CSTRORM_INTMEM +
4947                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4948         u8 flags = REG_RD8(bp, addr);
4949         /* clear and set */
4950         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4951         flags |= enable_flag;
4952         REG_WR8(bp, addr, flags);
4953         DP(NETIF_MSG_IFUP,
4954            "port %x fw_sb_id %d sb_index %d disable %d\n",
4955            port, fw_sb_id, sb_index, disable);
4956 }
4957
4958 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4959                                     u8 sb_index, u8 disable, u16 usec)
4960 {
4961         int port = BP_PORT(bp);
4962         u8 ticks = usec / BNX2X_BTR;
4963
4964         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4965
4966         disable = disable ? 1 : (usec ? 0 : 1);
4967         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4968 }
4969
4970 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
4971                             u32 verbose)
4972 {
4973         smp_mb__before_atomic();
4974         set_bit(flag, &bp->sp_rtnl_state);
4975         smp_mb__after_atomic();
4976         DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
4977            flag);
4978         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4979 }
4980 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);