drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <net/addrconf.h>
  64 #include <net/bonding.h>
  65 #include <net/addrconf.h>
  66 #include <asm/uaccess.h>
  67
  68 #include "cxgb4.h"
  69 #include "t4_regs.h"
  70 #include "t4_values.h"
  71 #include "t4_msg.h"
  72 #include "t4fw_api.h"
  73 #include "t4fw_version.h"
  74 #include "cxgb4_dcb.h"
  75 #include "cxgb4_debugfs.h"
  76 #include "clip_tbl.h"
  77 #include "l2t.h"
  78
  79 #ifdef DRV_VERSION
  80 #undef DRV_VERSION
  81 #endif
  82 #define DRV_VERSION "2.0.0-ko"
  83 #define DRV_DESC "Chelsio T4/T5 Network Driver"
  84
  85 enum {
  86         MAX_TXQ_ENTRIES      = 16384,
  87         MAX_CTRL_TXQ_ENTRIES = 1024,
  88         MAX_RSPQ_ENTRIES     = 16384,
  89         MAX_RX_BUFFERS       = 16384,
  90         MIN_TXQ_ENTRIES      = 32,
  91         MIN_CTRL_TXQ_ENTRIES = 32,
  92         MIN_RSPQ_ENTRIES     = 128,
  93         MIN_FL_ENTRIES       = 16
  94 };
  95
  96 /* Host shadow copy of ingress filter entry.  This is in host native format
  97  * and doesn't match the ordering or bit order, etc. of the hardware of the
  98  * firmware command.  The use of bit-field structure elements is purely to
  99  * remind ourselves of the field size limitations and save memory in the case
 100  * where the filter table is large.
 101  */
 102 struct filter_entry {
 103         /* Administrative fields for filter.
 104          */
 105         u32 valid:1;            /* filter allocated and valid */
 106         u32 locked:1;           /* filter is administratively locked */
 107
 108         u32 pending:1;          /* filter action is pending firmware reply */
 109         u32 smtidx:8;           /* Source MAC Table index for smac */
 110         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 111
 112         /* The filter itself.  Most of this is a straight copy of information
 113          * provided by the extended ioctl().  Some fields are translated to
 114          * internal forms -- for instance the Ingress Queue ID passed in from
 115          * the ioctl() is translated into the Absolute Ingress Queue ID.
 116          */
 117         struct ch_filter_specification fs;
 118 };
 119
 120 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 121                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 122                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 123
 124 /* Macros needed to support the PCI Device ID Table ...
 125  */
 126 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
 127         static struct pci_device_id cxgb4_pci_tbl[] = {
 128 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
 129
 130 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
 131  * called for both.
 132  */
 133 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
 134
 135 #define CH_PCI_ID_TABLE_ENTRY(devid) \
 136                 {PCI_VDEVICE(CHELSIO, (devid)), 4}
 137
 138 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
 139                 { 0, } \
 140         }
 141
 142 #include "t4_pci_id_tbl.h"
 143
 144 #define FW4_FNAME "cxgb4/t4fw.bin"
 145 #define FW5_FNAME "cxgb4/t5fw.bin"
 146 #define FW4_CFNAME "cxgb4/t4-config.txt"
 147 #define FW5_CFNAME "cxgb4/t5-config.txt"
 148
 149 MODULE_DESCRIPTION(DRV_DESC);
 150 MODULE_AUTHOR("Chelsio Communications");
 151 MODULE_LICENSE("Dual BSD/GPL");
 152 MODULE_VERSION(DRV_VERSION);
 153 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 154 MODULE_FIRMWARE(FW4_FNAME);
 155 MODULE_FIRMWARE(FW5_FNAME);
 156
 157 /*
 158  * Normally we're willing to become the firmware's Master PF but will be happy
 159  * if another PF has already become the Master and initialized the adapter.
 160  * Setting "force_init" will cause this driver to forcibly establish itself as
 161  * the Master PF and initialize the adapter.
 162  */
 163 static uint force_init;
 164
 165 module_param(force_init, uint, 0644);
 166 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 167
 168 /*
 169  * Normally if the firmware we connect to has Configuration File support, we
 170  * use that and only fall back to the old Driver-based initialization if the
 171  * Configuration File fails for some reason.  If force_old_init is set, then
 172  * we'll always use the old Driver-based initialization sequence.
 173  */
 174 static uint force_old_init;
 175
 176 module_param(force_old_init, uint, 0644);
 177 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
 178                  " parameter");
 179
 180 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 181
 182 module_param(dflt_msg_enable, int, 0644);
 183 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 184
 185 /*
 186  * The driver uses the best interrupt scheme available on a platform in the
 187  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 188  * of these schemes the driver may consider as follows:
 189  *
 190  * msi = 2: choose from among all three options
 191  * msi = 1: only consider MSI and INTx interrupts
 192  * msi = 0: force INTx interrupts
 193  */
 194 static int msi = 2;
 195
 196 module_param(msi, int, 0644);
 197 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 198
 199 /*
 200  * Queue interrupt hold-off timer values.  Queues default to the first of these
 201  * upon creation.
 202  */
 203 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 204
 205 module_param_array(intr_holdoff, uint, NULL, 0644);
 206 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 207                  "0..4 in microseconds, deprecated parameter");
 208
 209 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 210
 211 module_param_array(intr_cnt, uint, NULL, 0644);
 212 MODULE_PARM_DESC(intr_cnt,
 213                  "thresholds 1..3 for queue interrupt packet counters, "
 214                  "deprecated parameter");
 215
 216 /*
 217  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 218  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 219  * boundaries.  This is a requirement for many architectures which will throw
 220  * a machine check fault if an attempt is made to access one of the 4-byte IP
 221  * header fields on a non-4-byte boundary.  And it's a major performance issue
 222  * even on some architectures which allow it like some implementations of the
 223  * x86 ISA.  However, some architectures don't mind this and for some very
 224  * edge-case performance sensitive applications (like forwarding large volumes
 225  * of small packets), setting this DMA offset to 0 will decrease the number of
 226  * PCI-E Bus transfers enough to measurably affect performance.
 227  */
 228 static int rx_dma_offset = 2;
 229
 230 static bool vf_acls;
 231
 232 #ifdef CONFIG_PCI_IOV
 233 module_param(vf_acls, bool, 0644);
 234 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
 235                  "deprecated parameter");
 236
 237 /* Configure the number of PCI-E Virtual Function which are to be instantiated
 238  * on SR-IOV Capable Physical Functions.
 239  */
 240 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
 241
 242 module_param_array(num_vf, uint, NULL, 0644);
 243 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 244 #endif
 245
 246 /* TX Queue select used to determine what algorithm to use for selecting TX
 247  * queue. Select between the kernel provided function (select_queue=0) or user
 248  * cxgb_select_queue function (select_queue=1)
 249  *
 250  * Default: select_queue=0
 251  */
 252 static int select_queue;
 253 module_param(select_queue, int, 0644);
 254 MODULE_PARM_DESC(select_queue,
 255                  "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
 256
 257 static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
 258
 259 module_param(tp_vlan_pri_map, uint, 0644);
 260 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
 261                  "deprecated parameter");
 262
 263 static struct dentry *cxgb4_debugfs_root;
 264
 265 static LIST_HEAD(adapter_list);
 266 static DEFINE_MUTEX(uld_mutex);
 267 /* Adapter list to be accessed from atomic context */
 268 static LIST_HEAD(adap_rcu_list);
 269 static DEFINE_SPINLOCK(adap_rcu_lock);
 270 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 271 static const char *uld_str[] = { "RDMA", "iSCSI" };
 272
 273 static void link_report(struct net_device *dev)
 274 {
 275         if (!netif_carrier_ok(dev))
 276                 netdev_info(dev, "link down\n");
 277         else {
 278                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 279
 280                 const char *s = "10Mbps";
 281                 const struct port_info *p = netdev_priv(dev);
 282
 283                 switch (p->link_cfg.speed) {
 284                 case 10000:
 285                         s = "10Gbps";
 286                         break;
 287                 case 1000:
 288                         s = "1000Mbps";
 289                         break;
 290                 case 100:
 291                         s = "100Mbps";
 292                         break;
 293                 case 40000:
 294                         s = "40Gbps";
 295                         break;
 296                 }
 297
 298                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 299                             fc[p->link_cfg.fc]);
 300         }
 301 }
 302
 303 #ifdef CONFIG_CHELSIO_T4_DCB
 304 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */
 305 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
 306 {
 307         struct port_info *pi = netdev_priv(dev);
 308         struct adapter *adap = pi->adapter;
 309         struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
 310         int i;
 311
 312         /* We use a simple mapping of Port TX Queue Index to DCB
 313          * Priority when we're enabling DCB.
 314          */
 315         for (i = 0; i < pi->nqsets; i++, txq++) {
 316                 u32 name, value;
 317                 int err;
 318
 319                 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
 320                         FW_PARAMS_PARAM_X_V(
 321                                 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
 322                         FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
 323                 value = enable ? i : 0xffffffff;
 324
 325                 /* Since we can be called while atomic (from "interrupt
 326                  * level") we need to issue the Set Parameters Commannd
 327                  * without sleeping (timeout < 0).
 328                  */
 329                 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
 330                                             &name, &value);
 331
 332                 if (err)
 333                         dev_err(adap->pdev_dev,
 334                                 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
 335                                 enable ? "set" : "unset", pi->port_id, i, -err);
 336                 else
 337                         txq->dcb_prio = value;
 338         }
 339 }
 340 #endif /* CONFIG_CHELSIO_T4_DCB */
 341
 342 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 343 {
 344         struct net_device *dev = adapter->port[port_id];
 345
 346         /* Skip changes from disabled ports. */
 347         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 348                 if (link_stat)
 349                         netif_carrier_on(dev);
 350                 else {
 351 #ifdef CONFIG_CHELSIO_T4_DCB
 352                         cxgb4_dcb_state_init(dev);
 353                         dcb_tx_queue_prio_enable(dev, false);
 354 #endif /* CONFIG_CHELSIO_T4_DCB */
 355                         netif_carrier_off(dev);
 356                 }
 357
 358                 link_report(dev);
 359         }
 360 }
 361
 362 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 363 {
 364         static const char *mod_str[] = {
 365                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 366         };
 367
 368         const struct net_device *dev = adap->port[port_id];
 369         const struct port_info *pi = netdev_priv(dev);
 370
 371         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 372                 netdev_info(dev, "port module unplugged\n");
 373         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 374                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 375 }
 376
 377 /*
 378  * Configure the exact and hash address filters to handle a port's multicast
 379  * and secondary unicast MAC addresses.
 380  */
 381 static int set_addr_filters(const struct net_device *dev, bool sleep)
 382 {
 383         u64 mhash = 0;
 384         u64 uhash = 0;
 385         bool free = true;
 386         u16 filt_idx[7];
 387         const u8 *addr[7];
 388         int ret, naddr = 0;
 389         const struct netdev_hw_addr *ha;
 390         int uc_cnt = netdev_uc_count(dev);
 391         int mc_cnt = netdev_mc_count(dev);
 392         const struct port_info *pi = netdev_priv(dev);
 393         unsigned int mb = pi->adapter->fn;
 394
 395         /* first do the secondary unicast addresses */
 396         netdev_for_each_uc_addr(ha, dev) {
 397                 addr[naddr++] = ha->addr;
 398                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 399                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 400                                         naddr, addr, filt_idx, &uhash, sleep);
 401                         if (ret < 0)
 402                                 return ret;
 403
 404                         free = false;
 405                         naddr = 0;
 406                 }
 407         }
 408
 409         /* next set up the multicast addresses */
 410         netdev_for_each_mc_addr(ha, dev) {
 411                 addr[naddr++] = ha->addr;
 412                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 413                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 414                                         naddr, addr, filt_idx, &mhash, sleep);
 415                         if (ret < 0)
 416                                 return ret;
 417
 418                         free = false;
 419                         naddr = 0;
 420                 }
 421         }
 422
 423         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 424                                 uhash | mhash, sleep);
 425 }
 426
 427 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 428 module_param(dbfifo_int_thresh, int, 0644);
 429 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 430
 431 /*
 432  * usecs to sleep while draining the dbfifo
 433  */
 434 static int dbfifo_drain_delay = 1000;
 435 module_param(dbfifo_drain_delay, int, 0644);
 436 MODULE_PARM_DESC(dbfifo_drain_delay,
 437                  "usecs to sleep while draining the dbfifo");
 438
 439 /*
 440  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 441  * If @mtu is -1 it is left unchanged.
 442  */
 443 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 444 {
 445         int ret;
 446         struct port_info *pi = netdev_priv(dev);
 447
 448         ret = set_addr_filters(dev, sleep_ok);
 449         if (ret == 0)
 450                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 451                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 452                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 453                                     sleep_ok);
 454         return ret;
 455 }
 456
 457 /**
 458  *      link_start - enable a port
 459  *      @dev: the port to enable
 460  *
 461  *      Performs the MAC and PHY actions needed to enable a port.
 462  */
 463 static int link_start(struct net_device *dev)
 464 {
 465         int ret;
 466         struct port_info *pi = netdev_priv(dev);
 467         unsigned int mb = pi->adapter->fn;
 468
 469         /*
 470          * We do not set address filters and promiscuity here, the stack does
 471          * that step explicitly.
 472          */
 473         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 474                             !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
 475         if (ret == 0) {
 476                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 477                                     pi->xact_addr_filt, dev->dev_addr, true,
 478                                     true);
 479                 if (ret >= 0) {
 480                         pi->xact_addr_filt = ret;
 481                         ret = 0;
 482                 }
 483         }
 484         if (ret == 0)
 485                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 486                                     &pi->link_cfg);
 487         if (ret == 0) {
 488                 local_bh_disable();
 489                 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
 490                                           true, CXGB4_DCB_ENABLED);
 491                 local_bh_enable();
 492         }
 493
 494         return ret;
 495 }
 496
 497 int cxgb4_dcb_enabled(const struct net_device *dev)
 498 {
 499 #ifdef CONFIG_CHELSIO_T4_DCB
 500         struct port_info *pi = netdev_priv(dev);
 501
 502         if (!pi->dcb.enabled)
 503                 return 0;
 504
 505         return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
 506                 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
 507 #else
 508         return 0;
 509 #endif
 510 }
 511 EXPORT_SYMBOL(cxgb4_dcb_enabled);
 512
 513 #ifdef CONFIG_CHELSIO_T4_DCB
 514 /* Handle a Data Center Bridging update message from the firmware. */
 515 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
 516 {
 517         int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
 518         struct net_device *dev = adap->port[port];
 519         int old_dcb_enabled = cxgb4_dcb_enabled(dev);
 520         int new_dcb_enabled;
 521
 522         cxgb4_dcb_handle_fw_update(adap, pcmd);
 523         new_dcb_enabled = cxgb4_dcb_enabled(dev);
 524
 525         /* If the DCB has become enabled or disabled on the port then we're
 526          * going to need to set up/tear down DCB Priority parameters for the
 527          * TX Queues associated with the port.
 528          */
 529         if (new_dcb_enabled != old_dcb_enabled)
 530                 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
 531 }
 532 #endif /* CONFIG_CHELSIO_T4_DCB */
 533
 534 /* Clear a filter and release any of its resources that we own.  This also
 535  * clears the filter's "pending" status.
 536  */
 537 static void clear_filter(struct adapter *adap, struct filter_entry *f)
 538 {
 539         /* If the new or old filter have loopback rewriteing rules then we'll
 540          * need to free any existing Layer Two Table (L2T) entries of the old
 541          * filter rule.  The firmware will handle freeing up any Source MAC
 542          * Table (SMT) entries used for rewriting Source MAC Addresses in
 543          * loopback rules.
 544          */
 545         if (f->l2t)
 546                 cxgb4_l2t_release(f->l2t);
 547
 548         /* The zeroing of the filter rule below clears the filter valid,
 549          * pending, locked flags, l2t pointer, etc. so it's all we need for
 550          * this operation.
 551          */
 552         memset(f, 0, sizeof(*f));
 553 }
 554
 555 /* Handle a filter write/deletion reply.
 556  */
 557 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
 558 {
 559         unsigned int idx = GET_TID(rpl);
 560         unsigned int nidx = idx - adap->tids.ftid_base;
 561         unsigned int ret;
 562         struct filter_entry *f;
 563
 564         if (idx >= adap->tids.ftid_base && nidx <
 565            (adap->tids.nftids + adap->tids.nsftids)) {
 566                 idx = nidx;
 567                 ret = TCB_COOKIE_G(rpl->cookie);
 568                 f = &adap->tids.ftid_tab[idx];
 569
 570                 if (ret == FW_FILTER_WR_FLT_DELETED) {
 571                         /* Clear the filter when we get confirmation from the
 572                          * hardware that the filter has been deleted.
 573                          */
 574                         clear_filter(adap, f);
 575                 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
 576                         dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
 577                                 idx);
 578                         clear_filter(adap, f);
 579                 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
 580                         f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
 581                         f->pending = 0;  /* asynchronous setup completed */
 582                         f->valid = 1;
 583                 } else {
 584                         /* Something went wrong.  Issue a warning about the
 585                          * problem and clear everything out.
 586                          */
 587                         dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
 588                                 idx, ret);
 589                         clear_filter(adap, f);
 590                 }
 591         }
 592 }
 593
 594 /* Response queue handler for the FW event queue.
 595  */
 596 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 597                           const struct pkt_gl *gl)
 598 {
 599         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 600
 601         rsp++;                                          /* skip RSS header */
 602
 603         /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
 604          */
 605         if (unlikely(opcode == CPL_FW4_MSG &&
 606            ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
 607                 rsp++;
 608                 opcode = ((const struct rss_header *)rsp)->opcode;
 609                 rsp++;
 610                 if (opcode != CPL_SGE_EGR_UPDATE) {
 611                         dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
 612                                 , opcode);
 613                         goto out;
 614                 }
 615         }
 616
 617         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 618                 const struct cpl_sge_egr_update *p = (void *)rsp;
 619                 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
 620                 struct sge_txq *txq;
 621
 622                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 623                 txq->restarts++;
 624                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 625                         struct sge_eth_txq *eq;
 626
 627                         eq = container_of(txq, struct sge_eth_txq, q);
 628                         netif_tx_wake_queue(eq->txq);
 629                 } else {
 630                         struct sge_ofld_txq *oq;
 631
 632                         oq = container_of(txq, struct sge_ofld_txq, q);
 633                         tasklet_schedule(&oq->qresume_tsk);
 634                 }
 635         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 636                 const struct cpl_fw6_msg *p = (void *)rsp;
 637
 638 #ifdef CONFIG_CHELSIO_T4_DCB
 639                 const struct fw_port_cmd *pcmd = (const void *)p->data;
 640                 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
 641                 unsigned int action =
 642                         FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
 643
 644                 if (cmd == FW_PORT_CMD &&
 645                     action == FW_PORT_ACTION_GET_PORT_INFO) {
 646                         int port = FW_PORT_CMD_PORTID_G(
 647                                         be32_to_cpu(pcmd->op_to_portid));
 648                         struct net_device *dev = q->adap->port[port];
 649                         int state_input = ((pcmd->u.info.dcbxdis_pkd &
 650                                             FW_PORT_CMD_DCBXDIS_F)
 651                                            ? CXGB4_DCB_INPUT_FW_DISABLED
 652                                            : CXGB4_DCB_INPUT_FW_ENABLED);
 653
 654                         cxgb4_dcb_state_fsm(dev, state_input);
 655                 }
 656
 657                 if (cmd == FW_PORT_CMD &&
 658                     action == FW_PORT_ACTION_L2_DCB_CFG)
 659                         dcb_rpl(q->adap, pcmd);
 660                 else
 661 #endif
 662                         if (p->type == 0)
 663                                 t4_handle_fw_rpl(q->adap, p->data);
 664         } else if (opcode == CPL_L2T_WRITE_RPL) {
 665                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 666
 667                 do_l2t_write_rpl(q->adap, p);
 668         } else if (opcode == CPL_SET_TCB_RPL) {
 669                 const struct cpl_set_tcb_rpl *p = (void *)rsp;
 670
 671                 filter_rpl(q->adap, p);
 672         } else
 673                 dev_err(q->adap->pdev_dev,
 674                         "unexpected CPL %#x on FW event queue\n", opcode);
 675 out:
 676         return 0;
 677 }
 678
 679 /**
 680  *      uldrx_handler - response queue handler for ULD queues
 681  *      @q: the response queue that received the packet
 682  *      @rsp: the response queue descriptor holding the offload message
 683  *      @gl: the gather list of packet fragments
 684  *
 685  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 686  *      the ULD, we just maintain statistics.
 687  */
 688 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 689                          const struct pkt_gl *gl)
 690 {
 691         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 692
 693         /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
 694          */
 695         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
 696             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
 697                 rsp += 2;
 698
 699         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 700                 rxq->stats.nomem++;
 701                 return -1;
 702         }
 703         if (gl == NULL)
 704                 rxq->stats.imm++;
 705         else if (gl == CXGB4_MSG_AN)
 706                 rxq->stats.an++;
 707         else
 708                 rxq->stats.pkts++;
 709         return 0;
 710 }
 711
 712 static void disable_msi(struct adapter *adapter)
 713 {
 714         if (adapter->flags & USING_MSIX) {
 715                 pci_disable_msix(adapter->pdev);
 716                 adapter->flags &= ~USING_MSIX;
 717         } else if (adapter->flags & USING_MSI) {
 718                 pci_disable_msi(adapter->pdev);
 719                 adapter->flags &= ~USING_MSI;
 720         }
 721 }
 722
 723 /*
 724  * Interrupt handler for non-data events used with MSI-X.
 725  */
 726 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 727 {
 728         struct adapter *adap = cookie;
 729         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
 730
 731         if (v & PFSW_F) {
 732                 adap->swintr = 1;
 733                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
 734         }
 735         t4_slow_intr_handler(adap);
 736         return IRQ_HANDLED;
 737 }
 738
 739 /*
 740  * Name the MSI-X interrupts.
 741  */
 742 static void name_msix_vecs(struct adapter *adap)
 743 {
 744         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 745
 746         /* non-data interrupts */
 747         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 748
 749         /* FW events */
 750         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 751                  adap->port[0]->name);
 752
 753         /* Ethernet queues */
 754         for_each_port(adap, j) {
 755                 struct net_device *d = adap->port[j];
 756                 const struct port_info *pi = netdev_priv(d);
 757
 758                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 759                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 760                                  d->name, i);
 761         }
 762
 763         /* offload queues */
 764         for_each_ofldrxq(&adap->sge, i)
 765                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 766                          adap->port[0]->name, i);
 767
 768         for_each_rdmarxq(&adap->sge, i)
 769                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 770                          adap->port[0]->name, i);
 771
 772         for_each_rdmaciq(&adap->sge, i)
 773                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
 774                          adap->port[0]->name, i);
 775 }
 776
 777 static int request_msix_queue_irqs(struct adapter *adap)
 778 {
 779         struct sge *s = &adap->sge;
 780         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
 781         int msi_index = 2;
 782
 783         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 784                           adap->msix_info[1].desc, &s->fw_evtq);
 785         if (err)
 786                 return err;
 787
 788         for_each_ethrxq(s, ethqidx) {
 789                 err = request_irq(adap->msix_info[msi_index].vec,
 790                                   t4_sge_intr_msix, 0,
 791                                   adap->msix_info[msi_index].desc,
 792                                   &s->ethrxq[ethqidx].rspq);
 793                 if (err)
 794                         goto unwind;
 795                 msi_index++;
 796         }
 797         for_each_ofldrxq(s, ofldqidx) {
 798                 err = request_irq(adap->msix_info[msi_index].vec,
 799                                   t4_sge_intr_msix, 0,
 800                                   adap->msix_info[msi_index].desc,
 801                                   &s->ofldrxq[ofldqidx].rspq);
 802                 if (err)
 803                         goto unwind;
 804                 msi_index++;
 805         }
 806         for_each_rdmarxq(s, rdmaqidx) {
 807                 err = request_irq(adap->msix_info[msi_index].vec,
 808                                   t4_sge_intr_msix, 0,
 809                                   adap->msix_info[msi_index].desc,
 810                                   &s->rdmarxq[rdmaqidx].rspq);
 811                 if (err)
 812                         goto unwind;
 813                 msi_index++;
 814         }
 815         for_each_rdmaciq(s, rdmaciqqidx) {
 816                 err = request_irq(adap->msix_info[msi_index].vec,
 817                                   t4_sge_intr_msix, 0,
 818                                   adap->msix_info[msi_index].desc,
 819                                   &s->rdmaciq[rdmaciqqidx].rspq);
 820                 if (err)
 821                         goto unwind;
 822                 msi_index++;
 823         }
 824         return 0;
 825
 826 unwind:
 827         while (--rdmaciqqidx >= 0)
 828                 free_irq(adap->msix_info[--msi_index].vec,
 829                          &s->rdmaciq[rdmaciqqidx].rspq);
 830         while (--rdmaqidx >= 0)
 831                 free_irq(adap->msix_info[--msi_index].vec,
 832                          &s->rdmarxq[rdmaqidx].rspq);
 833         while (--ofldqidx >= 0)
 834                 free_irq(adap->msix_info[--msi_index].vec,
 835                          &s->ofldrxq[ofldqidx].rspq);
 836         while (--ethqidx >= 0)
 837                 free_irq(adap->msix_info[--msi_index].vec,
 838                          &s->ethrxq[ethqidx].rspq);
 839         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 840         return err;
 841 }
 842
 843 static void free_msix_queue_irqs(struct adapter *adap)
 844 {
 845         int i, msi_index = 2;
 846         struct sge *s = &adap->sge;
 847
 848         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 849         for_each_ethrxq(s, i)
 850                 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
 851         for_each_ofldrxq(s, i)
 852                 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
 853         for_each_rdmarxq(s, i)
 854                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
 855         for_each_rdmaciq(s, i)
 856                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
 857 }
 858
 859 /**
 860  *      write_rss - write the RSS table for a given port
 861  *      @pi: the port
 862  *      @queues: array of queue indices for RSS
 863  *
 864  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 865  *      packets to the Rx queues in @queues.
 866  */
 867 static int write_rss(const struct port_info *pi, const u16 *queues)
 868 {
 869         u16 *rss;
 870         int i, err;
 871         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 872
 873         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 874         if (!rss)
 875                 return -ENOMEM;
 876
 877         /* map the queue indices to queue ids */
 878         for (i = 0; i < pi->rss_size; i++, queues++)
 879                 rss[i] = q[*queues].rspq.abs_id;
 880
 881         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 882                                   pi->rss_size, rss, pi->rss_size);
 883         kfree(rss);
 884         return err;
 885 }
 886
 887 /**
 888  *      setup_rss - configure RSS
 889  *      @adap: the adapter
 890  *
 891  *      Sets up RSS for each port.
 892  */
 893 static int setup_rss(struct adapter *adap)
 894 {
 895         int i, err;
 896
 897         for_each_port(adap, i) {
 898                 const struct port_info *pi = adap2pinfo(adap, i);
 899
 900                 err = write_rss(pi, pi->rss);
 901                 if (err)
 902                         return err;
 903         }
 904         return 0;
 905 }
 906
 907 /*
 908  * Return the channel of the ingress queue with the given qid.
 909  */
 910 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 911 {
 912         qid -= p->ingr_start;
 913         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 914 }
 915
 916 /*
 917  * Wait until all NAPI handlers are descheduled.
 918  */
 919 static void quiesce_rx(struct adapter *adap)
 920 {
 921         int i;
 922
 923         for (i = 0; i < adap->sge.ingr_sz; i++) {
 924                 struct sge_rspq *q = adap->sge.ingr_map[i];
 925
 926                 if (q && q->handler) {
 927                         napi_disable(&q->napi);
 928                         local_bh_disable();
 929                         while (!cxgb_poll_lock_napi(q))
 930                                 mdelay(1);
 931                         local_bh_enable();
 932                 }
 933
 934         }
 935 }
 936
 937 /* Disable interrupt and napi handler */
 938 static void disable_interrupts(struct adapter *adap)
 939 {
 940         if (adap->flags & FULL_INIT_DONE) {
 941                 t4_intr_disable(adap);
 942                 if (adap->flags & USING_MSIX) {
 943                         free_msix_queue_irqs(adap);
 944                         free_irq(adap->msix_info[0].vec, adap);
 945                 } else {
 946                         free_irq(adap->pdev->irq, adap);
 947                 }
 948                 quiesce_rx(adap);
 949         }
 950 }
 951
 952 /*
 953  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 954  */
 955 static void enable_rx(struct adapter *adap)
 956 {
 957         int i;
 958
 959         for (i = 0; i < adap->sge.ingr_sz; i++) {
 960                 struct sge_rspq *q = adap->sge.ingr_map[i];
 961
 962                 if (!q)
 963                         continue;
 964                 if (q->handler) {
 965                         cxgb_busy_poll_init_lock(q);
 966                         napi_enable(&q->napi);
 967                 }
 968                 /* 0-increment GTS to start the timer and enable interrupts */
 969                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
 970                              SEINTARM_V(q->intr_params) |
 971                              INGRESSQID_V(q->cntxt_id));
 972         }
 973 }
 974
 975 /**
 976  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 977  *      @adap: the adapter
 978  *
 979  *      Determines how many sets of SGE queues to use and initializes them.
 980  *      We support multiple queue sets per port if we have MSI-X, otherwise
 981  *      just one queue set per port.
 982  */
 983 static int setup_sge_queues(struct adapter *adap)
 984 {
 985         int err, msi_idx, i, j;
 986         struct sge *s = &adap->sge;
 987
 988         bitmap_zero(s->starving_fl, s->egr_sz);
 989         bitmap_zero(s->txq_maperr, s->egr_sz);
 990
 991         if (adap->flags & USING_MSIX)
 992                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 993         else {
 994                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 995                                        NULL, NULL);
 996                 if (err)
 997                         return err;
 998                 msi_idx = -((int)s->intrq.abs_id + 1);
 999         }
1000
1001         /* NOTE: If you add/delete any Ingress/Egress Queue allocations in here,
1002          * don't forget to update the following which need to be
1003          * synchronized to and changes here.
1004          *
1005          * 1. The calculations of MAX_INGQ in cxgb4.h.
1006          *
1007          * 2. Update enable_msix/name_msix_vecs/request_msix_queue_irqs
1008          *    to accommodate any new/deleted Ingress Queues
1009          *    which need MSI-X Vectors.
1010          *
1011          * 3. Update sge_qinfo_show() to include information on the
1012          *    new/deleted queues.
1013          */
1014         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1015                                msi_idx, NULL, fwevtq_handler);
1016         if (err) {
1017 freeout:        t4_free_sge_resources(adap);
1018                 return err;
1019         }
1020
1021         for_each_port(adap, i) {
1022                 struct net_device *dev = adap->port[i];
1023                 struct port_info *pi = netdev_priv(dev);
1024                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1025                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1026
1027                 for (j = 0; j < pi->nqsets; j++, q++) {
1028                         if (msi_idx > 0)
1029                                 msi_idx++;
1030                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1031                                                msi_idx, &q->fl,
1032                                                t4_ethrx_handler);
1033                         if (err)
1034                                 goto freeout;
1035                         q->rspq.idx = j;
1036                         memset(&q->stats, 0, sizeof(q->stats));
1037                 }
1038                 for (j = 0; j < pi->nqsets; j++, t++) {
1039                         err = t4_sge_alloc_eth_txq(adap, t, dev,
1040                                         netdev_get_tx_queue(dev, j),
1041                                         s->fw_evtq.cntxt_id);
1042                         if (err)
1043                                 goto freeout;
1044                 }
1045         }
1046
1047         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1048         for_each_ofldrxq(s, i) {
1049                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1050                 struct net_device *dev = adap->port[i / j];
1051
1052                 if (msi_idx > 0)
1053                         msi_idx++;
1054                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1055                                        q->fl.size ? &q->fl : NULL,
1056                                        uldrx_handler);
1057                 if (err)
1058                         goto freeout;
1059                 memset(&q->stats, 0, sizeof(q->stats));
1060                 s->ofld_rxq[i] = q->rspq.abs_id;
1061                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1062                                             s->fw_evtq.cntxt_id);
1063                 if (err)
1064                         goto freeout;
1065         }
1066
1067         for_each_rdmarxq(s, i) {
1068                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1069
1070                 if (msi_idx > 0)
1071                         msi_idx++;
1072                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1073                                        msi_idx, q->fl.size ? &q->fl : NULL,
1074                                        uldrx_handler);
1075                 if (err)
1076                         goto freeout;
1077                 memset(&q->stats, 0, sizeof(q->stats));
1078                 s->rdma_rxq[i] = q->rspq.abs_id;
1079         }
1080
1081         for_each_rdmaciq(s, i) {
1082                 struct sge_ofld_rxq *q = &s->rdmaciq[i];
1083
1084                 if (msi_idx > 0)
1085                         msi_idx++;
1086                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1087                                        msi_idx, q->fl.size ? &q->fl : NULL,
1088                                        uldrx_handler);
1089                 if (err)
1090                         goto freeout;
1091                 memset(&q->stats, 0, sizeof(q->stats));
1092                 s->rdma_ciq[i] = q->rspq.abs_id;
1093         }
1094
1095         for_each_port(adap, i) {
1096                 /*
1097                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1098                  * have RDMA queues, and that's the right value.
1099                  */
1100                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1101                                             s->fw_evtq.cntxt_id,
1102                                             s->rdmarxq[i].rspq.cntxt_id);
1103                 if (err)
1104                         goto freeout;
1105         }
1106
1107         t4_write_reg(adap, is_t4(adap->params.chip) ?
1108                                 MPS_TRC_RSS_CONTROL_A :
1109                                 MPS_T5_TRC_RSS_CONTROL_A,
1110                      RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1111                      QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
1112         return 0;
1113 }
1114
1115 /*
1116  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1117  * The allocated memory is cleared.
1118  */
1119 void *t4_alloc_mem(size_t size)
1120 {
1121         void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1122
1123         if (!p)
1124                 p = vzalloc(size);
1125         return p;
1126 }
1127
1128 /*
1129  * Free memory allocated through alloc_mem().
1130  */
1131 void t4_free_mem(void *addr)
1132 {
1133         if (is_vmalloc_addr(addr))
1134                 vfree(addr);
1135         else
1136                 kfree(addr);
1137 }
1138
1139 /* Send a Work Request to write the filter at a specified index.  We construct
1140  * a Firmware Filter Work Request to have the work done and put the indicated
1141  * filter into "pending" mode which will prevent any further actions against
1142  * it till we get a reply from the firmware on the completion status of the
1143  * request.
1144  */
1145 static int set_filter_wr(struct adapter *adapter, int fidx)
1146 {
1147         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1148         struct sk_buff *skb;
1149         struct fw_filter_wr *fwr;
1150         unsigned int ftid;
1151
1152         /* If the new filter requires loopback Destination MAC and/or VLAN
1153          * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1154          * the filter.
1155          */
1156         if (f->fs.newdmac || f->fs.newvlan) {
1157                 /* allocate L2T entry for new filter */
1158                 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1159                 if (f->l2t == NULL)
1160                         return -EAGAIN;
1161                 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1162                                         f->fs.eport, f->fs.dmac)) {
1163                         cxgb4_l2t_release(f->l2t);
1164                         f->l2t = NULL;
1165                         return -ENOMEM;
1166                 }
1167         }
1168
1169         ftid = adapter->tids.ftid_base + fidx;
1170
1171         skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1172         fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1173         memset(fwr, 0, sizeof(*fwr));
1174
1175         /* It would be nice to put most of the following in t4_hw.c but most
1176          * of the work is translating the cxgbtool ch_filter_specification
1177          * into the Work Request and the definition of that structure is
1178          * currently in cxgbtool.h which isn't appropriate to pull into the
1179          * common code.  We may eventually try to come up with a more neutral
1180          * filter specification structure but for now it's easiest to simply
1181          * put this fairly direct code in line ...
1182          */
1183         fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1184         fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
1185         fwr->tid_to_iq =
1186                 htonl(FW_FILTER_WR_TID_V(ftid) |
1187                       FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1188                       FW_FILTER_WR_NOREPLY_V(0) |
1189                       FW_FILTER_WR_IQ_V(f->fs.iq));
1190         fwr->del_filter_to_l2tix =
1191                 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1192                       FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1193                       FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1194                       FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1195                       FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1196                       FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1197                       FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1198                       FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1199                       FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
1200                                              f->fs.newvlan == VLAN_REWRITE) |
1201                       FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
1202                                             f->fs.newvlan == VLAN_REWRITE) |
1203                       FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1204                       FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1205                       FW_FILTER_WR_PRIO_V(f->fs.prio) |
1206                       FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
1207         fwr->ethtype = htons(f->fs.val.ethtype);
1208         fwr->ethtypem = htons(f->fs.mask.ethtype);
1209         fwr->frag_to_ovlan_vldm =
1210                 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1211                  FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1212                  FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1213                  FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1214                  FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1215                  FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
1216         fwr->smac_sel = 0;
1217         fwr->rx_chan_rx_rpl_iq =
1218                 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1219                       FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
1220         fwr->maci_to_matchtypem =
1221                 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1222                       FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1223                       FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1224                       FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1225                       FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1226                       FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1227                       FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1228                       FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
1229         fwr->ptcl = f->fs.val.proto;
1230         fwr->ptclm = f->fs.mask.proto;
1231         fwr->ttyp = f->fs.val.tos;
1232         fwr->ttypm = f->fs.mask.tos;
1233         fwr->ivlan = htons(f->fs.val.ivlan);
1234         fwr->ivlanm = htons(f->fs.mask.ivlan);
1235         fwr->ovlan = htons(f->fs.val.ovlan);
1236         fwr->ovlanm = htons(f->fs.mask.ovlan);
1237         memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1238         memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1239         memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1240         memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1241         fwr->lp = htons(f->fs.val.lport);
1242         fwr->lpm = htons(f->fs.mask.lport);
1243         fwr->fp = htons(f->fs.val.fport);
1244         fwr->fpm = htons(f->fs.mask.fport);
1245         if (f->fs.newsmac)
1246                 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1247
1248         /* Mark the filter as "pending" and ship off the Filter Work Request.
1249          * When we get the Work Request Reply we'll clear the pending status.
1250          */
1251         f->pending = 1;
1252         set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1253         t4_ofld_send(adapter, skb);
1254         return 0;
1255 }
1256
1257 /* Delete the filter at a specified index.
1258  */
1259 static int del_filter_wr(struct adapter *adapter, int fidx)
1260 {
1261         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1262         struct sk_buff *skb;
1263         struct fw_filter_wr *fwr;
1264         unsigned int len, ftid;
1265
1266         len = sizeof(*fwr);
1267         ftid = adapter->tids.ftid_base + fidx;
1268
1269         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1270         fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1271         t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1272
1273         /* Mark the filter as "pending" and ship off the Filter Work Request.
1274          * When we get the Work Request Reply we'll clear the pending status.
1275          */
1276         f->pending = 1;
1277         t4_mgmt_tx(adapter, skb);
1278         return 0;
1279 }
1280
1281 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1282                              void *accel_priv, select_queue_fallback_t fallback)
1283 {
1284         int txq;
1285
1286 #ifdef CONFIG_CHELSIO_T4_DCB
1287         /* If a Data Center Bridging has been successfully negotiated on this
1288          * link then we'll use the skb's priority to map it to a TX Queue.
1289          * The skb's priority is determined via the VLAN Tag Priority Code
1290          * Point field.
1291          */
1292         if (cxgb4_dcb_enabled(dev)) {
1293                 u16 vlan_tci;
1294                 int err;
1295
1296                 err = vlan_get_tag(skb, &vlan_tci);
1297                 if (unlikely(err)) {
1298                         if (net_ratelimit())
1299                                 netdev_warn(dev,
1300                                             "TX Packet without VLAN Tag on DCB Link\n");
1301                         txq = 0;
1302                 } else {
1303                         txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1304                 }
1305                 return txq;
1306         }
1307 #endif /* CONFIG_CHELSIO_T4_DCB */
1308
1309         if (select_queue) {
1310                 txq = (skb_rx_queue_recorded(skb)
1311                         ? skb_get_rx_queue(skb)
1312                         : smp_processor_id());
1313
1314                 while (unlikely(txq >= dev->real_num_tx_queues))
1315                         txq -= dev->real_num_tx_queues;
1316
1317                 return txq;
1318         }
1319
1320         return fallback(dev, skb) % dev->real_num_tx_queues;
1321 }
1322
1323 static inline int is_offload(const struct adapter *adap)
1324 {
1325         return adap->params.offload;
1326 }
1327
1328 /*
1329  * Implementation of ethtool operations.
1330  */
1331
1332 static u32 get_msglevel(struct net_device *dev)
1333 {
1334         return netdev2adap(dev)->msg_enable;
1335 }
1336
1337 static void set_msglevel(struct net_device *dev, u32 val)
1338 {
1339         netdev2adap(dev)->msg_enable = val;
1340 }
1341
1342 static char stats_strings[][ETH_GSTRING_LEN] = {
1343         "TxOctetsOK         ",
1344         "TxFramesOK         ",
1345         "TxBroadcastFrames  ",
1346         "TxMulticastFrames  ",
1347         "TxUnicastFrames    ",
1348         "TxErrorFrames      ",
1349
1350         "TxFrames64         ",
1351         "TxFrames65To127    ",
1352         "TxFrames128To255   ",
1353         "TxFrames256To511   ",
1354         "TxFrames512To1023  ",
1355         "TxFrames1024To1518 ",
1356         "TxFrames1519ToMax  ",
1357
1358         "TxFramesDropped    ",
1359         "TxPauseFrames      ",
1360         "TxPPP0Frames       ",
1361         "TxPPP1Frames       ",
1362         "TxPPP2Frames       ",
1363         "TxPPP3Frames       ",
1364         "TxPPP4Frames       ",
1365         "TxPPP5Frames       ",
1366         "TxPPP6Frames       ",
1367         "TxPPP7Frames       ",
1368
1369         "RxOctetsOK         ",
1370         "RxFramesOK         ",
1371         "RxBroadcastFrames  ",
1372         "RxMulticastFrames  ",
1373         "RxUnicastFrames    ",
1374
1375         "RxFramesTooLong    ",
1376         "RxJabberErrors     ",
1377         "RxFCSErrors        ",
1378         "RxLengthErrors     ",
1379         "RxSymbolErrors     ",
1380         "RxRuntFrames       ",
1381
1382         "RxFrames64         ",
1383         "RxFrames65To127    ",
1384         "RxFrames128To255   ",
1385         "RxFrames256To511   ",
1386         "RxFrames512To1023  ",
1387         "RxFrames1024To1518 ",
1388         "RxFrames1519ToMax  ",
1389
1390         "RxPauseFrames      ",
1391         "RxPPP0Frames       ",
1392         "RxPPP1Frames       ",
1393         "RxPPP2Frames       ",
1394         "RxPPP3Frames       ",
1395         "RxPPP4Frames       ",
1396         "RxPPP5Frames       ",
1397         "RxPPP6Frames       ",
1398         "RxPPP7Frames       ",
1399
1400         "RxBG0FramesDropped ",
1401         "RxBG1FramesDropped ",
1402         "RxBG2FramesDropped ",
1403         "RxBG3FramesDropped ",
1404         "RxBG0FramesTrunc   ",
1405         "RxBG1FramesTrunc   ",
1406         "RxBG2FramesTrunc   ",
1407         "RxBG3FramesTrunc   ",
1408
1409         "TSO                ",
1410         "TxCsumOffload      ",
1411         "RxCsumGood         ",
1412         "VLANextractions    ",
1413         "VLANinsertions     ",
1414         "GROpackets         ",
1415         "GROmerged          ",
1416         "WriteCoalSuccess   ",
1417         "WriteCoalFail      ",
1418 };
1419
1420 static int get_sset_count(struct net_device *dev, int sset)
1421 {
1422         switch (sset) {
1423         case ETH_SS_STATS:
1424                 return ARRAY_SIZE(stats_strings);
1425         default:
1426                 return -EOPNOTSUPP;
1427         }
1428 }
1429
1430 #define T4_REGMAP_SIZE (160 * 1024)
1431 #define T5_REGMAP_SIZE (332 * 1024)
1432
1433 static int get_regs_len(struct net_device *dev)
1434 {
1435         struct adapter *adap = netdev2adap(dev);
1436         if (is_t4(adap->params.chip))
1437                 return T4_REGMAP_SIZE;
1438         else
1439                 return T5_REGMAP_SIZE;
1440 }
1441
1442 static int get_eeprom_len(struct net_device *dev)
1443 {
1444         return EEPROMSIZE;
1445 }
1446
1447 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1448 {
1449         struct adapter *adapter = netdev2adap(dev);
1450         u32 exprom_vers;
1451
1452         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1453         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1454         strlcpy(info->bus_info, pci_name(adapter->pdev),
1455                 sizeof(info->bus_info));
1456
1457         if (adapter->params.fw_vers)
1458                 snprintf(info->fw_version, sizeof(info->fw_version),
1459                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1460                         FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1461                         FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1462                         FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1463                         FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1464                         FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1465                         FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1466                         FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1467                         FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
1468
1469         if (!t4_get_exprom_version(adapter, &exprom_vers))
1470                 snprintf(info->erom_version, sizeof(info->erom_version),
1471                          "%u.%u.%u.%u",
1472                          FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1473                          FW_HDR_FW_VER_MINOR_G(exprom_vers),
1474                          FW_HDR_FW_VER_MICRO_G(exprom_vers),
1475                          FW_HDR_FW_VER_BUILD_G(exprom_vers));
1476 }
1477
1478 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1479 {
1480         if (stringset == ETH_SS_STATS)
1481                 memcpy(data, stats_strings, sizeof(stats_strings));
1482 }
1483
1484 /*
1485  * port stats maintained per queue of the port.  They should be in the same
1486  * order as in stats_strings above.
1487  */
1488 struct queue_port_stats {
1489         u64 tso;
1490         u64 tx_csum;
1491         u64 rx_csum;
1492         u64 vlan_ex;
1493         u64 vlan_ins;
1494         u64 gro_pkts;
1495         u64 gro_merged;
1496 };
1497
1498 static void collect_sge_port_stats(const struct adapter *adap,
1499                 const struct port_info *p, struct queue_port_stats *s)
1500 {
1501         int i;
1502         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1503         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1504
1505         memset(s, 0, sizeof(*s));
1506         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1507                 s->tso += tx->tso;
1508                 s->tx_csum += tx->tx_cso;
1509                 s->rx_csum += rx->stats.rx_cso;
1510                 s->vlan_ex += rx->stats.vlan_ex;
1511                 s->vlan_ins += tx->vlan_ins;
1512                 s->gro_pkts += rx->stats.lro_pkts;
1513                 s->gro_merged += rx->stats.lro_merged;
1514         }
1515 }
1516
1517 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1518                       u64 *data)
1519 {
1520         struct port_info *pi = netdev_priv(dev);
1521         struct adapter *adapter = pi->adapter;
1522         u32 val1, val2;
1523
1524         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1525
1526         data += sizeof(struct port_stats) / sizeof(u64);
1527         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1528         data += sizeof(struct queue_port_stats) / sizeof(u64);
1529         if (!is_t4(adapter->params.chip)) {
1530                 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1531                 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1532                 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
1533                 *data = val1 - val2;
1534                 data++;
1535                 *data = val2;
1536                 data++;
1537         } else {
1538                 memset(data, 0, 2 * sizeof(u64));
1539                 *data += 2;
1540         }
1541 }
1542
1543 /*
1544  * Return a version number to identify the type of adapter.  The scheme is:
1545  * - bits 0..9: chip version
1546  * - bits 10..15: chip revision
1547  * - bits 16..23: register dump version
1548  */
1549 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1550 {
1551         return CHELSIO_CHIP_VERSION(ap->params.chip) |
1552                 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1553 }
1554
1555 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1556                            unsigned int end)
1557 {
1558         u32 *p = buf + start;
1559
1560         for ( ; start <= end; start += sizeof(u32))
1561                 *p++ = t4_read_reg(ap, start);
1562 }
1563
1564 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1565                      void *buf)
1566 {
1567         static const unsigned int t4_reg_ranges[] = {
1568                 0x1008, 0x1108,
1569                 0x1180, 0x11b4,
1570                 0x11fc, 0x123c,
1571                 0x1300, 0x173c,
1572                 0x1800, 0x18fc,
1573                 0x3000, 0x30d8,
1574                 0x30e0, 0x5924,
1575                 0x5960, 0x59d4,
1576                 0x5a00, 0x5af8,
1577                 0x6000, 0x6098,
1578                 0x6100, 0x6150,
1579                 0x6200, 0x6208,
1580                 0x6240, 0x6248,
1581                 0x6280, 0x6338,
1582                 0x6370, 0x638c,
1583                 0x6400, 0x643c,
1584                 0x6500, 0x6524,
1585                 0x6a00, 0x6a38,
1586                 0x6a60, 0x6a78,
1587                 0x6b00, 0x6b84,
1588                 0x6bf0, 0x6c84,
1589                 0x6cf0, 0x6d84,
1590                 0x6df0, 0x6e84,
1591                 0x6ef0, 0x6f84,
1592                 0x6ff0, 0x7084,
1593                 0x70f0, 0x7184,
1594                 0x71f0, 0x7284,
1595                 0x72f0, 0x7384,
1596                 0x73f0, 0x7450,
1597                 0x7500, 0x7530,
1598                 0x7600, 0x761c,
1599                 0x7680, 0x76cc,
1600                 0x7700, 0x7798,
1601                 0x77c0, 0x77fc,
1602                 0x7900, 0x79fc,
1603                 0x7b00, 0x7c38,
1604                 0x7d00, 0x7efc,
1605                 0x8dc0, 0x8e1c,
1606                 0x8e30, 0x8e78,
1607                 0x8ea0, 0x8f6c,
1608                 0x8fc0, 0x9074,
1609                 0x90fc, 0x90fc,
1610                 0x9400, 0x9458,
1611                 0x9600, 0x96bc,
1612                 0x9800, 0x9808,
1613                 0x9820, 0x983c,
1614                 0x9850, 0x9864,
1615                 0x9c00, 0x9c6c,
1616                 0x9c80, 0x9cec,
1617                 0x9d00, 0x9d6c,
1618                 0x9d80, 0x9dec,
1619                 0x9e00, 0x9e6c,
1620                 0x9e80, 0x9eec,
1621                 0x9f00, 0x9f6c,
1622                 0x9f80, 0x9fec,
1623                 0xd004, 0xd03c,
1624                 0xdfc0, 0xdfe0,
1625                 0xe000, 0xea7c,
1626                 0xf000, 0x11110,
1627                 0x11118, 0x11190,
1628                 0x19040, 0x1906c,
1629                 0x19078, 0x19080,
1630                 0x1908c, 0x19124,
1631                 0x19150, 0x191b0,
1632                 0x191d0, 0x191e8,
1633                 0x19238, 0x1924c,
1634                 0x193f8, 0x19474,
1635                 0x19490, 0x194f8,
1636                 0x19800, 0x19f30,
1637                 0x1a000, 0x1a06c,
1638                 0x1a0b0, 0x1a120,
1639                 0x1a128, 0x1a138,
1640                 0x1a190, 0x1a1c4,
1641                 0x1a1fc, 0x1a1fc,
1642                 0x1e040, 0x1e04c,
1643                 0x1e284, 0x1e28c,
1644                 0x1e2c0, 0x1e2c0,
1645                 0x1e2e0, 0x1e2e0,
1646                 0x1e300, 0x1e384,
1647                 0x1e3c0, 0x1e3c8,
1648                 0x1e440, 0x1e44c,
1649                 0x1e684, 0x1e68c,
1650                 0x1e6c0, 0x1e6c0,
1651                 0x1e6e0, 0x1e6e0,
1652                 0x1e700, 0x1e784,
1653                 0x1e7c0, 0x1e7c8,
1654                 0x1e840, 0x1e84c,
1655                 0x1ea84, 0x1ea8c,
1656                 0x1eac0, 0x1eac0,
1657                 0x1eae0, 0x1eae0,
1658                 0x1eb00, 0x1eb84,
1659                 0x1ebc0, 0x1ebc8,
1660                 0x1ec40, 0x1ec4c,
1661                 0x1ee84, 0x1ee8c,
1662                 0x1eec0, 0x1eec0,
1663                 0x1eee0, 0x1eee0,
1664                 0x1ef00, 0x1ef84,
1665                 0x1efc0, 0x1efc8,
1666                 0x1f040, 0x1f04c,
1667                 0x1f284, 0x1f28c,
1668                 0x1f2c0, 0x1f2c0,
1669                 0x1f2e0, 0x1f2e0,
1670                 0x1f300, 0x1f384,
1671                 0x1f3c0, 0x1f3c8,
1672                 0x1f440, 0x1f44c,
1673                 0x1f684, 0x1f68c,
1674                 0x1f6c0, 0x1f6c0,
1675                 0x1f6e0, 0x1f6e0,
1676                 0x1f700, 0x1f784,
1677                 0x1f7c0, 0x1f7c8,
1678                 0x1f840, 0x1f84c,
1679                 0x1fa84, 0x1fa8c,
1680                 0x1fac0, 0x1fac0,
1681                 0x1fae0, 0x1fae0,
1682                 0x1fb00, 0x1fb84,
1683                 0x1fbc0, 0x1fbc8,
1684                 0x1fc40, 0x1fc4c,
1685                 0x1fe84, 0x1fe8c,
1686                 0x1fec0, 0x1fec0,
1687                 0x1fee0, 0x1fee0,
1688                 0x1ff00, 0x1ff84,
1689                 0x1ffc0, 0x1ffc8,
1690                 0x20000, 0x2002c,
1691                 0x20100, 0x2013c,
1692                 0x20190, 0x201c8,
1693                 0x20200, 0x20318,
1694                 0x20400, 0x20528,
1695                 0x20540, 0x20614,
1696                 0x21000, 0x21040,
1697                 0x2104c, 0x21060,
1698                 0x210c0, 0x210ec,
1699                 0x21200, 0x21268,
1700                 0x21270, 0x21284,
1701                 0x212fc, 0x21388,
1702                 0x21400, 0x21404,
1703                 0x21500, 0x21518,
1704                 0x2152c, 0x2153c,
1705                 0x21550, 0x21554,
1706                 0x21600, 0x21600,
1707                 0x21608, 0x21628,
1708                 0x21630, 0x2163c,
1709                 0x21700, 0x2171c,
1710                 0x21780, 0x2178c,
1711                 0x21800, 0x21c38,
1712                 0x21c80, 0x21d7c,
1713                 0x21e00, 0x21e04,
1714                 0x22000, 0x2202c,
1715                 0x22100, 0x2213c,
1716                 0x22190, 0x221c8,
1717                 0x22200, 0x22318,
1718                 0x22400, 0x22528,
1719                 0x22540, 0x22614,
1720                 0x23000, 0x23040,
1721                 0x2304c, 0x23060,
1722                 0x230c0, 0x230ec,
1723                 0x23200, 0x23268,
1724                 0x23270, 0x23284,
1725                 0x232fc, 0x23388,
1726                 0x23400, 0x23404,
1727                 0x23500, 0x23518,
1728                 0x2352c, 0x2353c,
1729                 0x23550, 0x23554,
1730                 0x23600, 0x23600,
1731                 0x23608, 0x23628,
1732                 0x23630, 0x2363c,
1733                 0x23700, 0x2371c,
1734                 0x23780, 0x2378c,
1735                 0x23800, 0x23c38,
1736                 0x23c80, 0x23d7c,
1737                 0x23e00, 0x23e04,
1738                 0x24000, 0x2402c,
1739                 0x24100, 0x2413c,
1740                 0x24190, 0x241c8,
1741                 0x24200, 0x24318,
1742                 0x24400, 0x24528,
1743                 0x24540, 0x24614,
1744                 0x25000, 0x25040,
1745                 0x2504c, 0x25060,
1746                 0x250c0, 0x250ec,
1747                 0x25200, 0x25268,
1748                 0x25270, 0x25284,
1749                 0x252fc, 0x25388,
1750                 0x25400, 0x25404,
1751                 0x25500, 0x25518,
1752                 0x2552c, 0x2553c,
1753                 0x25550, 0x25554,
1754                 0x25600, 0x25600,
1755                 0x25608, 0x25628,
1756                 0x25630, 0x2563c,
1757                 0x25700, 0x2571c,
1758                 0x25780, 0x2578c,
1759                 0x25800, 0x25c38,
1760                 0x25c80, 0x25d7c,
1761                 0x25e00, 0x25e04,
1762                 0x26000, 0x2602c,
1763                 0x26100, 0x2613c,
1764                 0x26190, 0x261c8,
1765                 0x26200, 0x26318,
1766                 0x26400, 0x26528,
1767                 0x26540, 0x26614,
1768                 0x27000, 0x27040,
1769                 0x2704c, 0x27060,
1770                 0x270c0, 0x270ec,
1771                 0x27200, 0x27268,
1772                 0x27270, 0x27284,
1773                 0x272fc, 0x27388,
1774                 0x27400, 0x27404,
1775                 0x27500, 0x27518,
1776                 0x2752c, 0x2753c,
1777                 0x27550, 0x27554,
1778                 0x27600, 0x27600,
1779                 0x27608, 0x27628,
1780                 0x27630, 0x2763c,
1781                 0x27700, 0x2771c,
1782                 0x27780, 0x2778c,
1783                 0x27800, 0x27c38,
1784                 0x27c80, 0x27d7c,
1785                 0x27e00, 0x27e04
1786         };
1787
1788         static const unsigned int t5_reg_ranges[] = {
1789                 0x1008, 0x1148,
1790                 0x1180, 0x11b4,
1791                 0x11fc, 0x123c,
1792                 0x1280, 0x173c,
1793                 0x1800, 0x18fc,
1794                 0x3000, 0x3028,
1795                 0x3060, 0x30d8,
1796                 0x30e0, 0x30fc,
1797                 0x3140, 0x357c,
1798                 0x35a8, 0x35cc,
1799                 0x35ec, 0x35ec,
1800                 0x3600, 0x5624,
1801                 0x56cc, 0x575c,
1802                 0x580c, 0x5814,
1803                 0x5890, 0x58bc,
1804                 0x5940, 0x59dc,
1805                 0x59fc, 0x5a18,
1806                 0x5a60, 0x5a9c,
1807                 0x5b9c, 0x5bfc,
1808                 0x6000, 0x6040,
1809                 0x6058, 0x614c,
1810                 0x7700, 0x7798,
1811                 0x77c0, 0x78fc,
1812                 0x7b00, 0x7c54,
1813                 0x7d00, 0x7efc,
1814                 0x8dc0, 0x8de0,
1815                 0x8df8, 0x8e84,
1816                 0x8ea0, 0x8f84,
1817                 0x8fc0, 0x90f8,
1818                 0x9400, 0x9470,
1819                 0x9600, 0x96f4,
1820                 0x9800, 0x9808,
1821                 0x9820, 0x983c,
1822                 0x9850, 0x9864,
1823                 0x9c00, 0x9c6c,
1824                 0x9c80, 0x9cec,
1825                 0x9d00, 0x9d6c,
1826                 0x9d80, 0x9dec,
1827                 0x9e00, 0x9e6c,
1828                 0x9e80, 0x9eec,
1829                 0x9f00, 0x9f6c,
1830                 0x9f80, 0xa020,
1831                 0xd004, 0xd03c,
1832                 0xdfc0, 0xdfe0,
1833                 0xe000, 0x11088,
1834                 0x1109c, 0x11110,
1835                 0x11118, 0x1117c,
1836                 0x11190, 0x11204,
1837                 0x19040, 0x1906c,
1838                 0x19078, 0x19080,
1839                 0x1908c, 0x19124,
1840                 0x19150, 0x191b0,
1841                 0x191d0, 0x191e8,
1842                 0x19238, 0x19290,
1843                 0x193f8, 0x19474,
1844                 0x19490, 0x194cc,
1845                 0x194f0, 0x194f8,
1846                 0x19c00, 0x19c60,
1847                 0x19c94, 0x19e10,
1848                 0x19e50, 0x19f34,
1849                 0x19f40, 0x19f50,
1850                 0x19f90, 0x19fe4,
1851                 0x1a000, 0x1a06c,
1852                 0x1a0b0, 0x1a120,
1853                 0x1a128, 0x1a138,
1854                 0x1a190, 0x1a1c4,
1855                 0x1a1fc, 0x1a1fc,
1856                 0x1e008, 0x1e00c,
1857                 0x1e040, 0x1e04c,
1858                 0x1e284, 0x1e290,
1859                 0x1e2c0, 0x1e2c0,
1860                 0x1e2e0, 0x1e2e0,
1861                 0x1e300, 0x1e384,
1862                 0x1e3c0, 0x1e3c8,
1863                 0x1e408, 0x1e40c,
1864                 0x1e440, 0x1e44c,
1865                 0x1e684, 0x1e690,
1866                 0x1e6c0, 0x1e6c0,
1867                 0x1e6e0, 0x1e6e0,
1868                 0x1e700, 0x1e784,
1869                 0x1e7c0, 0x1e7c8,
1870                 0x1e808, 0x1e80c,
1871                 0x1e840, 0x1e84c,
1872                 0x1ea84, 0x1ea90,
1873                 0x1eac0, 0x1eac0,
1874                 0x1eae0, 0x1eae0,
1875                 0x1eb00, 0x1eb84,
1876                 0x1ebc0, 0x1ebc8,
1877                 0x1ec08, 0x1ec0c,
1878                 0x1ec40, 0x1ec4c,
1879                 0x1ee84, 0x1ee90,
1880                 0x1eec0, 0x1eec0,
1881                 0x1eee0, 0x1eee0,
1882                 0x1ef00, 0x1ef84,
1883                 0x1efc0, 0x1efc8,
1884                 0x1f008, 0x1f00c,
1885                 0x1f040, 0x1f04c,
1886                 0x1f284, 0x1f290,
1887                 0x1f2c0, 0x1f2c0,
1888                 0x1f2e0, 0x1f2e0,
1889                 0x1f300, 0x1f384,
1890                 0x1f3c0, 0x1f3c8,
1891                 0x1f408, 0x1f40c,
1892                 0x1f440, 0x1f44c,
1893                 0x1f684, 0x1f690,
1894                 0x1f6c0, 0x1f6c0,
1895                 0x1f6e0, 0x1f6e0,
1896                 0x1f700, 0x1f784,
1897                 0x1f7c0, 0x1f7c8,
1898                 0x1f808, 0x1f80c,
1899                 0x1f840, 0x1f84c,
1900                 0x1fa84, 0x1fa90,
1901                 0x1fac0, 0x1fac0,
1902                 0x1fae0, 0x1fae0,
1903                 0x1fb00, 0x1fb84,
1904                 0x1fbc0, 0x1fbc8,
1905                 0x1fc08, 0x1fc0c,
1906                 0x1fc40, 0x1fc4c,
1907                 0x1fe84, 0x1fe90,
1908                 0x1fec0, 0x1fec0,
1909                 0x1fee0, 0x1fee0,
1910                 0x1ff00, 0x1ff84,
1911                 0x1ffc0, 0x1ffc8,
1912                 0x30000, 0x30030,
1913                 0x30100, 0x30144,
1914                 0x30190, 0x301d0,
1915                 0x30200, 0x30318,
1916                 0x30400, 0x3052c,
1917                 0x30540, 0x3061c,
1918                 0x30800, 0x30834,
1919                 0x308c0, 0x30908,
1920                 0x30910, 0x309ac,
1921                 0x30a00, 0x30a04,
1922                 0x30a0c, 0x30a2c,
1923                 0x30a44, 0x30a50,
1924                 0x30a74, 0x30c24,
1925                 0x30d08, 0x30d14,
1926                 0x30d1c, 0x30d20,
1927                 0x30d3c, 0x30d50,
1928                 0x31200, 0x3120c,
1929                 0x31220, 0x31220,
1930                 0x31240, 0x31240,
1931                 0x31600, 0x31600,
1932                 0x31608, 0x3160c,
1933                 0x31a00, 0x31a1c,
1934                 0x31e04, 0x31e20,
1935                 0x31e38, 0x31e3c,
1936                 0x31e80, 0x31e80,
1937                 0x31e88, 0x31ea8,
1938                 0x31eb0, 0x31eb4,
1939                 0x31ec8, 0x31ed4,
1940                 0x31fb8, 0x32004,
1941                 0x32208, 0x3223c,
1942                 0x32600, 0x32630,
1943                 0x32a00, 0x32abc,
1944                 0x32b00, 0x32b70,
1945                 0x33000, 0x33048,
1946                 0x33060, 0x3309c,
1947                 0x330f0, 0x33148,
1948                 0x33160, 0x3319c,
1949                 0x331f0, 0x332e4,
1950                 0x332f8, 0x333e4,
1951                 0x333f8, 0x33448,
1952                 0x33460, 0x3349c,
1953                 0x334f0, 0x33548,
1954                 0x33560, 0x3359c,
1955                 0x335f0, 0x336e4,
1956                 0x336f8, 0x337e4,
1957                 0x337f8, 0x337fc,
1958                 0x33814, 0x33814,
1959                 0x3382c, 0x3382c,
1960                 0x33880, 0x3388c,
1961                 0x338e8, 0x338ec,
1962                 0x33900, 0x33948,
1963                 0x33960, 0x3399c,
1964                 0x339f0, 0x33ae4,
1965                 0x33af8, 0x33b10,
1966                 0x33b28, 0x33b28,
1967                 0x33b3c, 0x33b50,
1968                 0x33bf0, 0x33c10,
1969                 0x33c28, 0x33c28,
1970                 0x33c3c, 0x33c50,
1971                 0x33cf0, 0x33cfc,
1972                 0x34000, 0x34030,
1973                 0x34100, 0x34144,
1974                 0x34190, 0x341d0,
1975                 0x34200, 0x34318,
1976                 0x34400, 0x3452c,
1977                 0x34540, 0x3461c,
1978                 0x34800, 0x34834,
1979                 0x348c0, 0x34908,
1980                 0x34910, 0x349ac,
1981                 0x34a00, 0x34a04,
1982                 0x34a0c, 0x34a2c,
1983                 0x34a44, 0x34a50,
1984                 0x34a74, 0x34c24,
1985                 0x34d08, 0x34d14,
1986                 0x34d1c, 0x34d20,
1987                 0x34d3c, 0x34d50,
1988                 0x35200, 0x3520c,
1989                 0x35220, 0x35220,
1990                 0x35240, 0x35240,
1991                 0x35600, 0x35600,
1992                 0x35608, 0x3560c,
1993                 0x35a00, 0x35a1c,
1994                 0x35e04, 0x35e20,
1995                 0x35e38, 0x35e3c,
1996                 0x35e80, 0x35e80,
1997                 0x35e88, 0x35ea8,
1998                 0x35eb0, 0x35eb4,
1999                 0x35ec8, 0x35ed4,
2000                 0x35fb8, 0x36004,
2001                 0x36208, 0x3623c,
2002                 0x36600, 0x36630,
2003                 0x36a00, 0x36abc,
2004                 0x36b00, 0x36b70,
2005                 0x37000, 0x37048,
2006                 0x37060, 0x3709c,
2007                 0x370f0, 0x37148,
2008                 0x37160, 0x3719c,
2009                 0x371f0, 0x372e4,
2010                 0x372f8, 0x373e4,
2011                 0x373f8, 0x37448,
2012                 0x37460, 0x3749c,
2013                 0x374f0, 0x37548,
2014                 0x37560, 0x3759c,
2015                 0x375f0, 0x376e4,
2016                 0x376f8, 0x377e4,
2017                 0x377f8, 0x377fc,
2018                 0x37814, 0x37814,
2019                 0x3782c, 0x3782c,
2020                 0x37880, 0x3788c,
2021                 0x378e8, 0x378ec,
2022                 0x37900, 0x37948,
2023                 0x37960, 0x3799c,
2024                 0x379f0, 0x37ae4,
2025                 0x37af8, 0x37b10,
2026                 0x37b28, 0x37b28,
2027                 0x37b3c, 0x37b50,
2028                 0x37bf0, 0x37c10,
2029                 0x37c28, 0x37c28,
2030                 0x37c3c, 0x37c50,
2031                 0x37cf0, 0x37cfc,
2032                 0x38000, 0x38030,
2033                 0x38100, 0x38144,
2034                 0x38190, 0x381d0,
2035                 0x38200, 0x38318,
2036                 0x38400, 0x3852c,
2037                 0x38540, 0x3861c,
2038                 0x38800, 0x38834,
2039                 0x388c0, 0x38908,
2040                 0x38910, 0x389ac,
2041                 0x38a00, 0x38a04,
2042                 0x38a0c, 0x38a2c,
2043                 0x38a44, 0x38a50,
2044                 0x38a74, 0x38c24,
2045                 0x38d08, 0x38d14,
2046                 0x38d1c, 0x38d20,
2047                 0x38d3c, 0x38d50,
2048                 0x39200, 0x3920c,
2049                 0x39220, 0x39220,
2050                 0x39240, 0x39240,
2051                 0x39600, 0x39600,
2052                 0x39608, 0x3960c,
2053                 0x39a00, 0x39a1c,
2054                 0x39e04, 0x39e20,
2055                 0x39e38, 0x39e3c,
2056                 0x39e80, 0x39e80,
2057                 0x39e88, 0x39ea8,
2058                 0x39eb0, 0x39eb4,
2059                 0x39ec8, 0x39ed4,
2060                 0x39fb8, 0x3a004,
2061                 0x3a208, 0x3a23c,
2062                 0x3a600, 0x3a630,
2063                 0x3aa00, 0x3aabc,
2064                 0x3ab00, 0x3ab70,
2065                 0x3b000, 0x3b048,
2066                 0x3b060, 0x3b09c,
2067                 0x3b0f0, 0x3b148,
2068                 0x3b160, 0x3b19c,
2069                 0x3b1f0, 0x3b2e4,
2070                 0x3b2f8, 0x3b3e4,
2071                 0x3b3f8, 0x3b448,
2072                 0x3b460, 0x3b49c,
2073                 0x3b4f0, 0x3b548,
2074                 0x3b560, 0x3b59c,
2075                 0x3b5f0, 0x3b6e4,
2076                 0x3b6f8, 0x3b7e4,
2077                 0x3b7f8, 0x3b7fc,
2078                 0x3b814, 0x3b814,
2079                 0x3b82c, 0x3b82c,
2080                 0x3b880, 0x3b88c,
2081                 0x3b8e8, 0x3b8ec,
2082                 0x3b900, 0x3b948,
2083                 0x3b960, 0x3b99c,
2084                 0x3b9f0, 0x3bae4,
2085                 0x3baf8, 0x3bb10,
2086                 0x3bb28, 0x3bb28,
2087                 0x3bb3c, 0x3bb50,
2088                 0x3bbf0, 0x3bc10,
2089                 0x3bc28, 0x3bc28,
2090                 0x3bc3c, 0x3bc50,
2091                 0x3bcf0, 0x3bcfc,
2092                 0x3c000, 0x3c030,
2093                 0x3c100, 0x3c144,
2094                 0x3c190, 0x3c1d0,
2095                 0x3c200, 0x3c318,
2096                 0x3c400, 0x3c52c,
2097                 0x3c540, 0x3c61c,
2098                 0x3c800, 0x3c834,
2099                 0x3c8c0, 0x3c908,
2100                 0x3c910, 0x3c9ac,
2101                 0x3ca00, 0x3ca04,
2102                 0x3ca0c, 0x3ca2c,
2103                 0x3ca44, 0x3ca50,
2104                 0x3ca74, 0x3cc24,
2105                 0x3cd08, 0x3cd14,
2106                 0x3cd1c, 0x3cd20,
2107                 0x3cd3c, 0x3cd50,
2108                 0x3d200, 0x3d20c,
2109                 0x3d220, 0x3d220,
2110                 0x3d240, 0x3d240,
2111                 0x3d600, 0x3d600,
2112                 0x3d608, 0x3d60c,
2113                 0x3da00, 0x3da1c,
2114                 0x3de04, 0x3de20,
2115                 0x3de38, 0x3de3c,
2116                 0x3de80, 0x3de80,
2117                 0x3de88, 0x3dea8,
2118                 0x3deb0, 0x3deb4,
2119                 0x3dec8, 0x3ded4,
2120                 0x3dfb8, 0x3e004,
2121                 0x3e208, 0x3e23c,
2122                 0x3e600, 0x3e630,
2123                 0x3ea00, 0x3eabc,
2124                 0x3eb00, 0x3eb70,
2125                 0x3f000, 0x3f048,
2126                 0x3f060, 0x3f09c,
2127                 0x3f0f0, 0x3f148,
2128                 0x3f160, 0x3f19c,
2129                 0x3f1f0, 0x3f2e4,
2130                 0x3f2f8, 0x3f3e4,
2131                 0x3f3f8, 0x3f448,
2132                 0x3f460, 0x3f49c,
2133                 0x3f4f0, 0x3f548,
2134                 0x3f560, 0x3f59c,
2135                 0x3f5f0, 0x3f6e4,
2136                 0x3f6f8, 0x3f7e4,
2137                 0x3f7f8, 0x3f7fc,
2138                 0x3f814, 0x3f814,
2139                 0x3f82c, 0x3f82c,
2140                 0x3f880, 0x3f88c,
2141                 0x3f8e8, 0x3f8ec,
2142                 0x3f900, 0x3f948,
2143                 0x3f960, 0x3f99c,
2144                 0x3f9f0, 0x3fae4,
2145                 0x3faf8, 0x3fb10,
2146                 0x3fb28, 0x3fb28,
2147                 0x3fb3c, 0x3fb50,
2148                 0x3fbf0, 0x3fc10,
2149                 0x3fc28, 0x3fc28,
2150                 0x3fc3c, 0x3fc50,
2151                 0x3fcf0, 0x3fcfc,
2152                 0x40000, 0x4000c,
2153                 0x40040, 0x40068,
2154                 0x40080, 0x40144,
2155                 0x40180, 0x4018c,
2156                 0x40200, 0x40298,
2157                 0x402ac, 0x4033c,
2158                 0x403f8, 0x403fc,
2159                 0x41304, 0x413c4,
2160                 0x41400, 0x4141c,
2161                 0x41480, 0x414d0,
2162                 0x44000, 0x44078,
2163                 0x440c0, 0x44278,
2164                 0x442c0, 0x44478,
2165                 0x444c0, 0x44678,
2166                 0x446c0, 0x44878,
2167                 0x448c0, 0x449fc,
2168                 0x45000, 0x45068,
2169                 0x45080, 0x45084,
2170                 0x450a0, 0x450b0,
2171                 0x45200, 0x45268,
2172                 0x45280, 0x45284,
2173                 0x452a0, 0x452b0,
2174                 0x460c0, 0x460e4,
2175                 0x47000, 0x4708c,
2176                 0x47200, 0x47250,
2177                 0x47400, 0x47420,
2178                 0x47600, 0x47618,
2179                 0x47800, 0x47814,
2180                 0x48000, 0x4800c,
2181                 0x48040, 0x48068,
2182                 0x48080, 0x48144,
2183                 0x48180, 0x4818c,
2184                 0x48200, 0x48298,
2185                 0x482ac, 0x4833c,
2186                 0x483f8, 0x483fc,
2187                 0x49304, 0x493c4,
2188                 0x49400, 0x4941c,
2189                 0x49480, 0x494d0,
2190                 0x4c000, 0x4c078,
2191                 0x4c0c0, 0x4c278,
2192                 0x4c2c0, 0x4c478,
2193                 0x4c4c0, 0x4c678,
2194                 0x4c6c0, 0x4c878,
2195                 0x4c8c0, 0x4c9fc,
2196                 0x4d000, 0x4d068,
2197                 0x4d080, 0x4d084,
2198                 0x4d0a0, 0x4d0b0,
2199                 0x4d200, 0x4d268,
2200                 0x4d280, 0x4d284,
2201                 0x4d2a0, 0x4d2b0,
2202                 0x4e0c0, 0x4e0e4,
2203                 0x4f000, 0x4f08c,
2204                 0x4f200, 0x4f250,
2205                 0x4f400, 0x4f420,
2206                 0x4f600, 0x4f618,
2207                 0x4f800, 0x4f814,
2208                 0x50000, 0x500cc,
2209                 0x50400, 0x50400,
2210                 0x50800, 0x508cc,
2211                 0x50c00, 0x50c00,
2212                 0x51000, 0x5101c,
2213                 0x51300, 0x51308,
2214         };
2215
2216         int i;
2217         struct adapter *ap = netdev2adap(dev);
2218         static const unsigned int *reg_ranges;
2219         int arr_size = 0, buf_size = 0;
2220
2221         if (is_t4(ap->params.chip)) {
2222                 reg_ranges = &t4_reg_ranges[0];
2223                 arr_size = ARRAY_SIZE(t4_reg_ranges);
2224                 buf_size = T4_REGMAP_SIZE;
2225         } else {
2226                 reg_ranges = &t5_reg_ranges[0];
2227                 arr_size = ARRAY_SIZE(t5_reg_ranges);
2228                 buf_size = T5_REGMAP_SIZE;
2229         }
2230
2231         regs->version = mk_adap_vers(ap);
2232
2233         memset(buf, 0, buf_size);
2234         for (i = 0; i < arr_size; i += 2)
2235                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2236 }
2237
2238 static int restart_autoneg(struct net_device *dev)
2239 {
2240         struct port_info *p = netdev_priv(dev);
2241
2242         if (!netif_running(dev))
2243                 return -EAGAIN;
2244         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2245                 return -EINVAL;
2246         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2247         return 0;
2248 }
2249
2250 static int identify_port(struct net_device *dev,
2251                          enum ethtool_phys_id_state state)
2252 {
2253         unsigned int val;
2254         struct adapter *adap = netdev2adap(dev);
2255
2256         if (state == ETHTOOL_ID_ACTIVE)
2257                 val = 0xffff;
2258         else if (state == ETHTOOL_ID_INACTIVE)
2259                 val = 0;
2260         else
2261                 return -EINVAL;
2262
2263         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2264 }
2265
2266 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
2267 {
2268         unsigned int v = 0;
2269
2270         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2271             type == FW_PORT_TYPE_BT_XAUI) {
2272                 v |= SUPPORTED_TP;
2273                 if (caps & FW_PORT_CAP_SPEED_100M)
2274                         v |= SUPPORTED_100baseT_Full;
2275                 if (caps & FW_PORT_CAP_SPEED_1G)
2276                         v |= SUPPORTED_1000baseT_Full;
2277                 if (caps & FW_PORT_CAP_SPEED_10G)
2278                         v |= SUPPORTED_10000baseT_Full;
2279         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2280                 v |= SUPPORTED_Backplane;
2281                 if (caps & FW_PORT_CAP_SPEED_1G)
2282                         v |= SUPPORTED_1000baseKX_Full;
2283                 if (caps & FW_PORT_CAP_SPEED_10G)
2284                         v |= SUPPORTED_10000baseKX4_Full;
2285         } else if (type == FW_PORT_TYPE_KR)
2286                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2287         else if (type == FW_PORT_TYPE_BP_AP)
2288                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2289                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2290         else if (type == FW_PORT_TYPE_BP4_AP)
2291                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2292                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2293                      SUPPORTED_10000baseKX4_Full;
2294         else if (type == FW_PORT_TYPE_FIBER_XFI ||
2295                  type == FW_PORT_TYPE_FIBER_XAUI ||
2296                  type == FW_PORT_TYPE_SFP ||
2297                  type == FW_PORT_TYPE_QSFP_10G ||
2298                  type == FW_PORT_TYPE_QSA) {
2299                 v |= SUPPORTED_FIBRE;
2300                 if (caps & FW_PORT_CAP_SPEED_1G)
2301                         v |= SUPPORTED_1000baseT_Full;
2302                 if (caps & FW_PORT_CAP_SPEED_10G)
2303                         v |= SUPPORTED_10000baseT_Full;
2304         } else if (type == FW_PORT_TYPE_BP40_BA ||
2305                    type == FW_PORT_TYPE_QSFP) {
2306                 v |= SUPPORTED_40000baseSR4_Full;
2307                 v |= SUPPORTED_FIBRE;
2308         }
2309
2310         if (caps & FW_PORT_CAP_ANEG)
2311                 v |= SUPPORTED_Autoneg;
2312         return v;
2313 }
2314
2315 static unsigned int to_fw_linkcaps(unsigned int caps)
2316 {
2317         unsigned int v = 0;
2318
2319         if (caps & ADVERTISED_100baseT_Full)
2320                 v |= FW_PORT_CAP_SPEED_100M;
2321         if (caps & ADVERTISED_1000baseT_Full)
2322                 v |= FW_PORT_CAP_SPEED_1G;
2323         if (caps & ADVERTISED_10000baseT_Full)
2324                 v |= FW_PORT_CAP_SPEED_10G;
2325         if (caps & ADVERTISED_40000baseSR4_Full)
2326                 v |= FW_PORT_CAP_SPEED_40G;
2327         return v;
2328 }
2329
2330 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2331 {
2332         const struct port_info *p = netdev_priv(dev);
2333
2334         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2335             p->port_type == FW_PORT_TYPE_BT_XFI ||
2336             p->port_type == FW_PORT_TYPE_BT_XAUI)
2337                 cmd->port = PORT_TP;
2338         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2339                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2340                 cmd->port = PORT_FIBRE;
2341         else if (p->port_type == FW_PORT_TYPE_SFP ||
2342                  p->port_type == FW_PORT_TYPE_QSFP_10G ||
2343                  p->port_type == FW_PORT_TYPE_QSA ||
2344                  p->port_type == FW_PORT_TYPE_QSFP) {
2345                 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2346                     p->mod_type == FW_PORT_MOD_TYPE_SR ||
2347                     p->mod_type == FW_PORT_MOD_TYPE_ER ||
2348                     p->mod_type == FW_PORT_MOD_TYPE_LRM)
2349                         cmd->port = PORT_FIBRE;
2350                 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2351                          p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2352                         cmd->port = PORT_DA;
2353                 else
2354                         cmd->port = PORT_OTHER;
2355         } else
2356                 cmd->port = PORT_OTHER;
2357
2358         if (p->mdio_addr >= 0) {
2359                 cmd->phy_address = p->mdio_addr;
2360                 cmd->transceiver = XCVR_EXTERNAL;
2361                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2362                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2363         } else {
2364                 cmd->phy_address = 0;  /* not really, but no better option */
2365                 cmd->transceiver = XCVR_INTERNAL;
2366                 cmd->mdio_support = 0;
2367         }
2368
2369         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2370         cmd->advertising = from_fw_linkcaps(p->port_type,
2371                                             p->link_cfg.advertising);
2372         ethtool_cmd_speed_set(cmd,
2373                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2374         cmd->duplex = DUPLEX_FULL;
2375         cmd->autoneg = p->link_cfg.autoneg;
2376         cmd->maxtxpkt = 0;
2377         cmd->maxrxpkt = 0;
2378         return 0;
2379 }
2380
2381 static unsigned int speed_to_caps(int speed)
2382 {
2383         if (speed == 100)
2384                 return FW_PORT_CAP_SPEED_100M;
2385         if (speed == 1000)
2386                 return FW_PORT_CAP_SPEED_1G;
2387         if (speed == 10000)
2388                 return FW_PORT_CAP_SPEED_10G;
2389         if (speed == 40000)
2390                 return FW_PORT_CAP_SPEED_40G;
2391         return 0;
2392 }
2393
2394 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2395 {
2396         unsigned int cap;
2397         struct port_info *p = netdev_priv(dev);
2398         struct link_config *lc = &p->link_cfg;
2399         u32 speed = ethtool_cmd_speed(cmd);
2400
2401         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
2402                 return -EINVAL;
2403
2404         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2405                 /*
2406                  * PHY offers a single speed.  See if that's what's
2407                  * being requested.
2408                  */
2409                 if (cmd->autoneg == AUTONEG_DISABLE &&
2410                     (lc->supported & speed_to_caps(speed)))
2411                         return 0;
2412                 return -EINVAL;
2413         }
2414
2415         if (cmd->autoneg == AUTONEG_DISABLE) {
2416                 cap = speed_to_caps(speed);
2417
2418                 if (!(lc->supported & cap) ||
2419                     (speed == 1000) ||
2420                     (speed == 10000) ||
2421                     (speed == 40000))
2422                         return -EINVAL;
2423                 lc->requested_speed = cap;
2424                 lc->advertising = 0;
2425         } else {
2426                 cap = to_fw_linkcaps(cmd->advertising);
2427                 if (!(lc->supported & cap))
2428                         return -EINVAL;
2429                 lc->requested_speed = 0;
2430                 lc->advertising = cap | FW_PORT_CAP_ANEG;
2431         }
2432         lc->autoneg = cmd->autoneg;
2433
2434         if (netif_running(dev))
2435                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2436                                      lc);
2437         return 0;
2438 }
2439
2440 static void get_pauseparam(struct net_device *dev,
2441                            struct ethtool_pauseparam *epause)
2442 {
2443         struct port_info *p = netdev_priv(dev);
2444
2445         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2446         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2447         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2448 }
2449
2450 static int set_pauseparam(struct net_device *dev,
2451                           struct ethtool_pauseparam *epause)
2452 {
2453         struct port_info *p = netdev_priv(dev);
2454         struct link_config *lc = &p->link_cfg;
2455
2456         if (epause->autoneg == AUTONEG_DISABLE)
2457                 lc->requested_fc = 0;
2458         else if (lc->supported & FW_PORT_CAP_ANEG)
2459                 lc->requested_fc = PAUSE_AUTONEG;
2460         else
2461                 return -EINVAL;
2462
2463         if (epause->rx_pause)
2464                 lc->requested_fc |= PAUSE_RX;
2465         if (epause->tx_pause)
2466                 lc->requested_fc |= PAUSE_TX;
2467         if (netif_running(dev))
2468                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2469                                      lc);
2470         return 0;
2471 }
2472
2473 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2474 {
2475         const struct port_info *pi = netdev_priv(dev);
2476         const struct sge *s = &pi->adapter->sge;
2477
2478         e->rx_max_pending = MAX_RX_BUFFERS;
2479         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2480         e->rx_jumbo_max_pending = 0;
2481         e->tx_max_pending = MAX_TXQ_ENTRIES;
2482
2483         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2484         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2485         e->rx_jumbo_pending = 0;
2486         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2487 }
2488
2489 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2490 {
2491         int i;
2492         const struct port_info *pi = netdev_priv(dev);
2493         struct adapter *adapter = pi->adapter;
2494         struct sge *s = &adapter->sge;
2495
2496         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2497             e->tx_pending > MAX_TXQ_ENTRIES ||
2498             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2499             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2500             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2501                 return -EINVAL;
2502
2503         if (adapter->flags & FULL_INIT_DONE)
2504                 return -EBUSY;
2505
2506         for (i = 0; i < pi->nqsets; ++i) {
2507                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2508                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2509                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2510         }
2511         return 0;
2512 }
2513
2514 static int closest_timer(const struct sge *s, int time)
2515 {
2516         int i, delta, match = 0, min_delta = INT_MAX;
2517
2518         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2519                 delta = time - s->timer_val[i];
2520                 if (delta < 0)
2521                         delta = -delta;
2522                 if (delta < min_delta) {
2523                         min_delta = delta;
2524                         match = i;
2525                 }
2526         }
2527         return match;
2528 }
2529
2530 static int closest_thres(const struct sge *s, int thres)
2531 {
2532         int i, delta, match = 0, min_delta = INT_MAX;
2533
2534         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2535                 delta = thres - s->counter_val[i];
2536                 if (delta < 0)
2537                         delta = -delta;
2538                 if (delta < min_delta) {
2539                         min_delta = delta;
2540                         match = i;
2541                 }
2542         }
2543         return match;
2544 }
2545
2546 /*
2547  * Return a queue's interrupt hold-off time in us.  0 means no timer.
2548  */
2549 unsigned int qtimer_val(const struct adapter *adap,
2550                         const struct sge_rspq *q)
2551 {
2552         unsigned int idx = q->intr_params >> 1;
2553
2554         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2555 }
2556
2557 /**
2558  *      set_rspq_intr_params - set a queue's interrupt holdoff parameters
2559  *      @q: the Rx queue
2560  *      @us: the hold-off time in us, or 0 to disable timer
2561  *      @cnt: the hold-off packet count, or 0 to disable counter
2562  *
2563  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
2564  *      one of the two needs to be enabled for the queue to generate interrupts.
2565  */
2566 static int set_rspq_intr_params(struct sge_rspq *q,
2567                                 unsigned int us, unsigned int cnt)
2568 {
2569         struct adapter *adap = q->adap;
2570
2571         if ((us | cnt) == 0)
2572                 cnt = 1;
2573
2574         if (cnt) {
2575                 int err;
2576                 u32 v, new_idx;
2577
2578                 new_idx = closest_thres(&adap->sge, cnt);
2579                 if (q->desc && q->pktcnt_idx != new_idx) {
2580                         /* the queue has already been created, update it */
2581                         v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2582                             FW_PARAMS_PARAM_X_V(
2583                                         FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2584                             FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
2585                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2586                                             &new_idx);
2587                         if (err)
2588                                 return err;
2589                 }
2590                 q->pktcnt_idx = new_idx;
2591         }
2592
2593         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2594         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2595         return 0;
2596 }
2597
2598 /**
2599  * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2600  * @dev: the network device
2601  * @us: the hold-off time in us, or 0 to disable timer
2602  * @cnt: the hold-off packet count, or 0 to disable counter
2603  *
2604  * Set the RX interrupt hold-off parameters for a network device.
2605  */
2606 static int set_rx_intr_params(struct net_device *dev,
2607                               unsigned int us, unsigned int cnt)
2608 {
2609         int i, err;
2610         struct port_info *pi = netdev_priv(dev);
2611         struct adapter *adap = pi->adapter;
2612         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2613
2614         for (i = 0; i < pi->nqsets; i++, q++) {
2615                 err = set_rspq_intr_params(&q->rspq, us, cnt);
2616                 if (err)
2617                         return err;
2618         }
2619         return 0;
2620 }
2621
2622 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2623 {
2624         int i;
2625         struct port_info *pi = netdev_priv(dev);
2626         struct adapter *adap = pi->adapter;
2627         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2628
2629         for (i = 0; i < pi->nqsets; i++, q++)
2630                 q->rspq.adaptive_rx = adaptive_rx;
2631
2632         return 0;
2633 }
2634
2635 static int get_adaptive_rx_setting(struct net_device *dev)
2636 {
2637         struct port_info *pi = netdev_priv(dev);
2638         struct adapter *adap = pi->adapter;
2639         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2640
2641         return q->rspq.adaptive_rx;
2642 }
2643
2644 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2645 {
2646         set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
2647         return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2648                                   c->rx_max_coalesced_frames);
2649 }
2650
2651 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2652 {
2653         const struct port_info *pi = netdev_priv(dev);
2654         const struct adapter *adap = pi->adapter;
2655         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2656
2657         c->rx_coalesce_usecs = qtimer_val(adap, rq);
2658         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2659                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2660         c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
2661         return 0;
2662 }
2663
2664 /**
2665  *      eeprom_ptov - translate a physical EEPROM address to virtual
2666  *      @phys_addr: the physical EEPROM address
2667  *      @fn: the PCI function number
2668  *      @sz: size of function-specific area
2669  *
2670  *      Translate a physical EEPROM address to virtual.  The first 1K is
2671  *      accessed through virtual addresses starting at 31K, the rest is
2672  *      accessed through virtual addresses starting at 0.
2673  *
2674  *      The mapping is as follows:
2675  *      [0..1K) -> [31K..32K)
2676  *      [1K..1K+A) -> [31K-A..31K)
2677  *      [1K+A..ES) -> [0..ES-A-1K)
2678  *
2679  *      where A = @fn * @sz, and ES = EEPROM size.
2680  */
2681 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2682 {
2683         fn *= sz;
2684         if (phys_addr < 1024)
2685                 return phys_addr + (31 << 10);
2686         if (phys_addr < 1024 + fn)
2687                 return 31744 - fn + phys_addr - 1024;
2688         if (phys_addr < EEPROMSIZE)
2689                 return phys_addr - 1024 - fn;
2690         return -EINVAL;
2691 }
2692
2693 /*
2694  * The next two routines implement eeprom read/write from physical addresses.
2695  */
2696 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2697 {
2698         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2699
2700         if (vaddr >= 0)
2701                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2702         return vaddr < 0 ? vaddr : 0;
2703 }
2704
2705 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2706 {
2707         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2708
2709         if (vaddr >= 0)
2710                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2711         return vaddr < 0 ? vaddr : 0;
2712 }
2713
2714 #define EEPROM_MAGIC 0x38E2F10C
2715
2716 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2717                       u8 *data)
2718 {
2719         int i, err = 0;
2720         struct adapter *adapter = netdev2adap(dev);
2721
2722         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2723         if (!buf)
2724                 return -ENOMEM;
2725
2726         e->magic = EEPROM_MAGIC;
2727         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2728                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2729
2730         if (!err)
2731                 memcpy(data, buf + e->offset, e->len);
2732         kfree(buf);
2733         return err;
2734 }
2735
2736 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2737                       u8 *data)
2738 {
2739         u8 *buf;
2740         int err = 0;
2741         u32 aligned_offset, aligned_len, *p;
2742         struct adapter *adapter = netdev2adap(dev);
2743
2744         if (eeprom->magic != EEPROM_MAGIC)
2745                 return -EINVAL;
2746
2747         aligned_offset = eeprom->offset & ~3;
2748         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2749
2750         if (adapter->fn > 0) {
2751                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2752
2753                 if (aligned_offset < start ||
2754                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
2755                         return -EPERM;
2756         }
2757
2758         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2759                 /*
2760                  * RMW possibly needed for first or last words.
2761                  */
2762                 buf = kmalloc(aligned_len, GFP_KERNEL);
2763                 if (!buf)
2764                         return -ENOMEM;
2765                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2766                 if (!err && aligned_len > 4)
2767                         err = eeprom_rd_phys(adapter,
2768                                              aligned_offset + aligned_len - 4,
2769                                              (u32 *)&buf[aligned_len - 4]);
2770                 if (err)
2771                         goto out;
2772                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2773         } else
2774                 buf = data;
2775
2776         err = t4_seeprom_wp(adapter, false);
2777         if (err)
2778                 goto out;
2779
2780         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2781                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2782                 aligned_offset += 4;
2783         }
2784
2785         if (!err)
2786                 err = t4_seeprom_wp(adapter, true);
2787 out:
2788         if (buf != data)
2789                 kfree(buf);
2790         return err;
2791 }
2792
2793 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2794 {
2795         int ret;
2796         const struct firmware *fw;
2797         struct adapter *adap = netdev2adap(netdev);
2798         unsigned int mbox = PCIE_FW_MASTER_M + 1;
2799
2800         ef->data[sizeof(ef->data) - 1] = '\0';
2801         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2802         if (ret < 0)
2803                 return ret;
2804
2805         /* If the adapter has been fully initialized then we'll go ahead and
2806          * try to get the firmware's cooperation in upgrading to the new
2807          * firmware image otherwise we'll try to do the entire job from the
2808          * host ... and we always "force" the operation in this path.
2809          */
2810         if (adap->flags & FULL_INIT_DONE)
2811                 mbox = adap->mbox;
2812
2813         ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
2814         release_firmware(fw);
2815         if (!ret)
2816                 dev_info(adap->pdev_dev, "loaded firmware %s,"
2817                          " reload cxgb4 driver\n", ef->data);
2818         return ret;
2819 }
2820
2821 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2822 #define BCAST_CRC 0xa0ccc1a6
2823
2824 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2825 {
2826         wol->supported = WAKE_BCAST | WAKE_MAGIC;
2827         wol->wolopts = netdev2adap(dev)->wol;
2828         memset(&wol->sopass, 0, sizeof(wol->sopass));
2829 }
2830
2831 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2832 {
2833         int err = 0;
2834         struct port_info *pi = netdev_priv(dev);
2835
2836         if (wol->wolopts & ~WOL_SUPPORTED)
2837                 return -EINVAL;
2838         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2839                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2840         if (wol->wolopts & WAKE_BCAST) {
2841                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2842                                         ~0ULL, 0, false);
2843                 if (!err)
2844                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2845                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
2846         } else
2847                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2848         return err;
2849 }
2850
2851 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2852 {
2853         const struct port_info *pi = netdev_priv(dev);
2854         netdev_features_t changed = dev->features ^ features;
2855         int err;
2856
2857         if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2858                 return 0;
2859
2860         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2861                             -1, -1, -1,
2862                             !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2863         if (unlikely(err))
2864                 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2865         return err;
2866 }
2867
2868 static u32 get_rss_table_size(struct net_device *dev)
2869 {
2870         const struct port_info *pi = netdev_priv(dev);
2871
2872         return pi->rss_size;
2873 }
2874
2875 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
2876 {
2877         const struct port_info *pi = netdev_priv(dev);
2878         unsigned int n = pi->rss_size;
2879
2880         if (hfunc)
2881                 *hfunc = ETH_RSS_HASH_TOP;
2882         if (!p)
2883                 return 0;
2884         while (n--)
2885                 p[n] = pi->rss[n];
2886         return 0;
2887 }
2888
2889 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2890                          const u8 hfunc)
2891 {
2892         unsigned int i;
2893         struct port_info *pi = netdev_priv(dev);
2894
2895         /* We require at least one supported parameter to be changed and no
2896          * change in any of the unsupported parameters
2897          */
2898         if (key ||
2899             (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2900                 return -EOPNOTSUPP;
2901         if (!p)
2902                 return 0;
2903
2904         for (i = 0; i < pi->rss_size; i++)
2905                 pi->rss[i] = p[i];
2906         if (pi->adapter->flags & FULL_INIT_DONE)
2907                 return write_rss(pi, pi->rss);
2908         return 0;
2909 }
2910
2911 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2912                      u32 *rules)
2913 {
2914         const struct port_info *pi = netdev_priv(dev);
2915
2916         switch (info->cmd) {
2917         case ETHTOOL_GRXFH: {
2918                 unsigned int v = pi->rss_mode;
2919
2920                 info->data = 0;
2921                 switch (info->flow_type) {
2922                 case TCP_V4_FLOW:
2923                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
2924                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2925                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2926                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2927                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2928                         break;
2929                 case UDP_V4_FLOW:
2930                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2931                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2932                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2933                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2934                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2935                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2936                         break;
2937                 case SCTP_V4_FLOW:
2938                 case AH_ESP_V4_FLOW:
2939                 case IPV4_FLOW:
2940                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2941                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2942                         break;
2943                 case TCP_V6_FLOW:
2944                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
2945                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2946                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2947                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2948                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2949                         break;
2950                 case UDP_V6_FLOW:
2951                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2952                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2953                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2954                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2955                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2956                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2957                         break;
2958                 case SCTP_V6_FLOW:
2959                 case AH_ESP_V6_FLOW:
2960                 case IPV6_FLOW:
2961                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2962                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2963                         break;
2964                 }
2965                 return 0;
2966         }
2967         case ETHTOOL_GRXRINGS:
2968                 info->data = pi->nqsets;
2969                 return 0;
2970         }
2971         return -EOPNOTSUPP;
2972 }
2973
2974 static const struct ethtool_ops cxgb_ethtool_ops = {
2975         .get_settings      = get_settings,
2976         .set_settings      = set_settings,
2977         .get_drvinfo       = get_drvinfo,
2978         .get_msglevel      = get_msglevel,
2979         .set_msglevel      = set_msglevel,
2980         .get_ringparam     = get_sge_param,
2981         .set_ringparam     = set_sge_param,
2982         .get_coalesce      = get_coalesce,
2983         .set_coalesce      = set_coalesce,
2984         .get_eeprom_len    = get_eeprom_len,
2985         .get_eeprom        = get_eeprom,
2986         .set_eeprom        = set_eeprom,
2987         .get_pauseparam    = get_pauseparam,
2988         .set_pauseparam    = set_pauseparam,
2989         .get_link          = ethtool_op_get_link,
2990         .get_strings       = get_strings,
2991         .set_phys_id       = identify_port,
2992         .nway_reset        = restart_autoneg,
2993         .get_sset_count    = get_sset_count,
2994         .get_ethtool_stats = get_stats,
2995         .get_regs_len      = get_regs_len,
2996         .get_regs          = get_regs,
2997         .get_wol           = get_wol,
2998         .set_wol           = set_wol,
2999         .get_rxnfc         = get_rxnfc,
3000         .get_rxfh_indir_size = get_rss_table_size,
3001         .get_rxfh          = get_rss_table,
3002         .set_rxfh          = set_rss_table,
3003         .flash_device      = set_flash,
3004 };
3005
3006 static int setup_debugfs(struct adapter *adap)
3007 {
3008         if (IS_ERR_OR_NULL(adap->debugfs_root))
3009                 return -1;
3010
3011 #ifdef CONFIG_DEBUG_FS
3012         t4_setup_debugfs(adap);
3013 #endif
3014         return 0;
3015 }
3016
3017 /*
3018  * upper-layer driver support
3019  */
3020
3021 /*
3022  * Allocate an active-open TID and set it to the supplied value.
3023  */
3024 int cxgb4_alloc_atid(struct tid_info *t, void *data)
3025 {
3026         int atid = -1;
3027
3028         spin_lock_bh(&t->atid_lock);
3029         if (t->afree) {
3030                 union aopen_entry *p = t->afree;
3031
3032                 atid = (p - t->atid_tab) + t->atid_base;
3033                 t->afree = p->next;
3034                 p->data = data;
3035                 t->atids_in_use++;
3036         }
3037         spin_unlock_bh(&t->atid_lock);
3038         return atid;
3039 }
3040 EXPORT_SYMBOL(cxgb4_alloc_atid);
3041
3042 /*
3043  * Release an active-open TID.
3044  */
3045 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3046 {
3047         union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
3048
3049         spin_lock_bh(&t->atid_lock);
3050         p->next = t->afree;
3051         t->afree = p;
3052         t->atids_in_use--;
3053         spin_unlock_bh(&t->atid_lock);
3054 }
3055 EXPORT_SYMBOL(cxgb4_free_atid);
3056
3057 /*
3058  * Allocate a server TID and set it to the supplied value.
3059  */
3060 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3061 {
3062         int stid;
3063
3064         spin_lock_bh(&t->stid_lock);
3065         if (family == PF_INET) {
3066                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3067                 if (stid < t->nstids)
3068                         __set_bit(stid, t->stid_bmap);
3069                 else
3070                         stid = -1;
3071         } else {
3072                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3073                 if (stid < 0)
3074                         stid = -1;
3075         }
3076         if (stid >= 0) {
3077                 t->stid_tab[stid].data = data;
3078                 stid += t->stid_base;
3079                 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3080                  * This is equivalent to 4 TIDs. With CLIP enabled it
3081                  * needs 2 TIDs.
3082                  */
3083                 if (family == PF_INET)
3084                         t->stids_in_use++;
3085                 else
3086                         t->stids_in_use += 4;
3087         }
3088         spin_unlock_bh(&t->stid_lock);
3089         return stid;
3090 }
3091 EXPORT_SYMBOL(cxgb4_alloc_stid);
3092
3093 /* Allocate a server filter TID and set it to the supplied value.
3094  */
3095 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3096 {
3097         int stid;
3098
3099         spin_lock_bh(&t->stid_lock);
3100         if (family == PF_INET) {
3101                 stid = find_next_zero_bit(t->stid_bmap,
3102                                 t->nstids + t->nsftids, t->nstids);
3103                 if (stid < (t->nstids + t->nsftids))
3104                         __set_bit(stid, t->stid_bmap);
3105                 else
3106                         stid = -1;
3107         } else {
3108                 stid = -1;
3109         }
3110         if (stid >= 0) {
3111                 t->stid_tab[stid].data = data;
3112                 stid -= t->nstids;
3113                 stid += t->sftid_base;
3114                 t->stids_in_use++;
3115         }
3116         spin_unlock_bh(&t->stid_lock);
3117         return stid;
3118 }
3119 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3120
3121 /* Release a server TID.
3122  */
3123 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3124 {
3125         /* Is it a server filter TID? */
3126         if (t->nsftids && (stid >= t->sftid_base)) {
3127                 stid -= t->sftid_base;
3128                 stid += t->nstids;
3129         } else {
3130                 stid -= t->stid_base;
3131         }
3132
3133         spin_lock_bh(&t->stid_lock);
3134         if (family == PF_INET)
3135                 __clear_bit(stid, t->stid_bmap);
3136         else
3137                 bitmap_release_region(t->stid_bmap, stid, 2);
3138         t->stid_tab[stid].data = NULL;
3139         if (family == PF_INET)
3140                 t->stids_in_use--;
3141         else
3142                 t->stids_in_use -= 4;
3143         spin_unlock_bh(&t->stid_lock);
3144 }
3145 EXPORT_SYMBOL(cxgb4_free_stid);
3146
3147 /*
3148  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
3149  */
3150 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3151                            unsigned int tid)
3152 {
3153         struct cpl_tid_release *req;
3154
3155         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3156         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3157         INIT_TP_WR(req, tid);
3158         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3159 }
3160
3161 /*
3162  * Queue a TID release request and if necessary schedule a work queue to
3163  * process it.
3164  */
3165 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3166                                     unsigned int tid)
3167 {
3168         void **p = &t->tid_tab[tid];
3169         struct adapter *adap = container_of(t, struct adapter, tids);
3170
3171         spin_lock_bh(&adap->tid_release_lock);
3172         *p = adap->tid_release_head;
3173         /* Low 2 bits encode the Tx channel number */
3174         adap->tid_release_head = (void **)((uintptr_t)p | chan);
3175         if (!adap->tid_release_task_busy) {
3176                 adap->tid_release_task_busy = true;
3177                 queue_work(adap->workq, &adap->tid_release_task);
3178         }
3179         spin_unlock_bh(&adap->tid_release_lock);
3180 }
3181
3182 /*
3183  * Process the list of pending TID release requests.
3184  */
3185 static void process_tid_release_list(struct work_struct *work)
3186 {
3187         struct sk_buff *skb;
3188         struct adapter *adap;
3189
3190         adap = container_of(work, struct adapter, tid_release_task);
3191
3192         spin_lock_bh(&adap->tid_release_lock);
3193         while (adap->tid_release_head) {
3194                 void **p = adap->tid_release_head;
3195                 unsigned int chan = (uintptr_t)p & 3;
3196                 p = (void *)p - chan;
3197
3198                 adap->tid_release_head = *p;
3199                 *p = NULL;
3200                 spin_unlock_bh(&adap->tid_release_lock);
3201
3202                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3203                                          GFP_KERNEL)))
3204                         schedule_timeout_uninterruptible(1);
3205
3206                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3207                 t4_ofld_send(adap, skb);
3208                 spin_lock_bh(&adap->tid_release_lock);
3209         }
3210         adap->tid_release_task_busy = false;
3211         spin_unlock_bh(&adap->tid_release_lock);
3212 }
3213
3214 /*
3215  * Release a TID and inform HW.  If we are unable to allocate the release
3216  * message we defer to a work queue.
3217  */
3218 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3219 {
3220         void *old;
3221         struct sk_buff *skb;
3222         struct adapter *adap = container_of(t, struct adapter, tids);
3223
3224         old = t->tid_tab[tid];
3225         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3226         if (likely(skb)) {
3227                 t->tid_tab[tid] = NULL;
3228                 mk_tid_release(skb, chan, tid);
3229                 t4_ofld_send(adap, skb);
3230         } else
3231                 cxgb4_queue_tid_release(t, chan, tid);
3232         if (old)
3233                 atomic_dec(&t->tids_in_use);
3234 }
3235 EXPORT_SYMBOL(cxgb4_remove_tid);
3236
3237 /*
3238  * Allocate and initialize the TID tables.  Returns 0 on success.
3239  */
3240 static int tid_init(struct tid_info *t)
3241 {
3242         size_t size;
3243         unsigned int stid_bmap_size;
3244         unsigned int natids = t->natids;
3245         struct adapter *adap = container_of(t, struct adapter, tids);
3246
3247         stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3248         size = t->ntids * sizeof(*t->tid_tab) +
3249                natids * sizeof(*t->atid_tab) +
3250                t->nstids * sizeof(*t->stid_tab) +
3251                t->nsftids * sizeof(*t->stid_tab) +
3252                stid_bmap_size * sizeof(long) +
3253                t->nftids * sizeof(*t->ftid_tab) +
3254                t->nsftids * sizeof(*t->ftid_tab);
3255
3256         t->tid_tab = t4_alloc_mem(size);
3257         if (!t->tid_tab)
3258                 return -ENOMEM;
3259
3260         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3261         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3262         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3263         t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3264         spin_lock_init(&t->stid_lock);
3265         spin_lock_init(&t->atid_lock);
3266
3267         t->stids_in_use = 0;
3268         t->afree = NULL;
3269         t->atids_in_use = 0;
3270         atomic_set(&t->tids_in_use, 0);
3271
3272         /* Setup the free list for atid_tab and clear the stid bitmap. */
3273         if (natids) {
3274                 while (--natids)
3275                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3276                 t->afree = t->atid_tab;
3277         }
3278         bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3279         /* Reserve stid 0 for T4/T5 adapters */
3280         if (!t->stid_base &&
3281             (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3282                 __set_bit(0, t->stid_bmap);
3283
3284         return 0;
3285 }
3286
3287 /**
3288  *      cxgb4_create_server - create an IP server
3289  *      @dev: the device
3290  *      @stid: the server TID
3291  *      @sip: local IP address to bind server to
3292  *      @sport: the server's TCP port
3293  *      @queue: queue to direct messages from this server to
3294  *
3295  *      Create an IP server for the given port and address.
3296  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3297  */
3298 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3299                         __be32 sip, __be16 sport, __be16 vlan,
3300                         unsigned int queue)
3301 {
3302         unsigned int chan;
3303         struct sk_buff *skb;
3304         struct adapter *adap;
3305         struct cpl_pass_open_req *req;
3306         int ret;
3307
3308         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3309         if (!skb)
3310                 return -ENOMEM;
3311
3312         adap = netdev2adap(dev);
3313         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3314         INIT_TP_WR(req, 0);
3315         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3316         req->local_port = sport;
3317         req->peer_port = htons(0);
3318         req->local_ip = sip;
3319         req->peer_ip = htonl(0);
3320         chan = rxq_to_chan(&adap->sge, queue);
3321         req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3322         req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3323                                 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3324         ret = t4_mgmt_tx(adap, skb);
3325         return net_xmit_eval(ret);
3326 }
3327 EXPORT_SYMBOL(cxgb4_create_server);
3328
3329 /*      cxgb4_create_server6 - create an IPv6 server
3330  *      @dev: the device
3331  *      @stid: the server TID
3332  *      @sip: local IPv6 address to bind server to
3333  *      @sport: the server's TCP port
3334  *      @queue: queue to direct messages from this server to
3335  *
3336  *      Create an IPv6 server for the given port and address.
3337  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3338  */
3339 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3340                          const struct in6_addr *sip, __be16 sport,
3341                          unsigned int queue)
3342 {
3343         unsigned int chan;
3344         struct sk_buff *skb;
3345         struct adapter *adap;
3346         struct cpl_pass_open_req6 *req;
3347         int ret;
3348
3349         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3350         if (!skb)
3351                 return -ENOMEM;
3352
3353         adap = netdev2adap(dev);
3354         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3355         INIT_TP_WR(req, 0);
3356         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3357         req->local_port = sport;
3358         req->peer_port = htons(0);
3359         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3360         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3361         req->peer_ip_hi = cpu_to_be64(0);
3362         req->peer_ip_lo = cpu_to_be64(0);
3363         chan = rxq_to_chan(&adap->sge, queue);
3364         req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3365         req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3366                                 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3367         ret = t4_mgmt_tx(adap, skb);
3368         return net_xmit_eval(ret);
3369 }
3370 EXPORT_SYMBOL(cxgb4_create_server6);
3371
3372 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3373                         unsigned int queue, bool ipv6)
3374 {
3375         struct sk_buff *skb;
3376         struct adapter *adap;
3377         struct cpl_close_listsvr_req *req;
3378         int ret;
3379
3380         adap = netdev2adap(dev);
3381
3382         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3383         if (!skb)
3384                 return -ENOMEM;
3385
3386         req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3387         INIT_TP_WR(req, 0);
3388         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3389         req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3390                                 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
3391         ret = t4_mgmt_tx(adap, skb);
3392         return net_xmit_eval(ret);
3393 }
3394 EXPORT_SYMBOL(cxgb4_remove_server);
3395
3396 /**
3397  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3398  *      @mtus: the HW MTU table
3399  *      @mtu: the target MTU
3400  *      @idx: index of selected entry in the MTU table
3401  *
3402  *      Returns the index and the value in the HW MTU table that is closest to
3403  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
3404  *      table, in which case that smallest available value is selected.
3405  */
3406 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3407                             unsigned int *idx)
3408 {
3409         unsigned int i = 0;
3410
3411         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3412                 ++i;
3413         if (idx)
3414                 *idx = i;
3415         return mtus[i];
3416 }
3417 EXPORT_SYMBOL(cxgb4_best_mtu);
3418
3419 /**
3420  *     cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3421  *     @mtus: the HW MTU table
3422  *     @header_size: Header Size
3423  *     @data_size_max: maximum Data Segment Size
3424  *     @data_size_align: desired Data Segment Size Alignment (2^N)
3425  *     @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3426  *
3427  *     Similar to cxgb4_best_mtu() but instead of searching the Hardware
3428  *     MTU Table based solely on a Maximum MTU parameter, we break that
3429  *     parameter up into a Header Size and Maximum Data Segment Size, and
3430  *     provide a desired Data Segment Size Alignment.  If we find an MTU in
3431  *     the Hardware MTU Table which will result in a Data Segment Size with
3432  *     the requested alignment _and_ that MTU isn't "too far" from the
3433  *     closest MTU, then we'll return that rather than the closest MTU.
3434  */
3435 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3436                                     unsigned short header_size,
3437                                     unsigned short data_size_max,
3438                                     unsigned short data_size_align,
3439                                     unsigned int *mtu_idxp)
3440 {
3441         unsigned short max_mtu = header_size + data_size_max;
3442         unsigned short data_size_align_mask = data_size_align - 1;
3443         int mtu_idx, aligned_mtu_idx;
3444
3445         /* Scan the MTU Table till we find an MTU which is larger than our
3446          * Maximum MTU or we reach the end of the table.  Along the way,
3447          * record the last MTU found, if any, which will result in a Data
3448          * Segment Length matching the requested alignment.
3449          */
3450         for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3451                 unsigned short data_size = mtus[mtu_idx] - header_size;
3452
3453                 /* If this MTU minus the Header Size would result in a
3454                  * Data Segment Size of the desired alignment, remember it.
3455                  */
3456                 if ((data_size & data_size_align_mask) == 0)
3457                         aligned_mtu_idx = mtu_idx;
3458
3459                 /* If we're not at the end of the Hardware MTU Table and the
3460                  * next element is larger than our Maximum MTU, drop out of
3461                  * the loop.
3462                  */
3463                 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3464                         break;
3465         }
3466
3467         /* If we fell out of the loop because we ran to the end of the table,
3468          * then we just have to use the last [largest] entry.
3469          */
3470         if (mtu_idx == NMTUS)
3471                 mtu_idx--;
3472
3473         /* If we found an MTU which resulted in the requested Data Segment
3474          * Length alignment and that's "not far" from the largest MTU which is
3475          * less than or equal to the maximum MTU, then use that.
3476          */
3477         if (aligned_mtu_idx >= 0 &&
3478             mtu_idx - aligned_mtu_idx <= 1)
3479                 mtu_idx = aligned_mtu_idx;
3480
3481         /* If the caller has passed in an MTU Index pointer, pass the
3482          * MTU Index back.  Return the MTU value.
3483          */
3484         if (mtu_idxp)
3485                 *mtu_idxp = mtu_idx;
3486         return mtus[mtu_idx];
3487 }
3488 EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3489
3490 /**
3491  *      cxgb4_port_chan - get the HW channel of a port
3492  *      @dev: the net device for the port
3493  *
3494  *      Return the HW Tx channel of the given port.
3495  */
3496 unsigned int cxgb4_port_chan(const struct net_device *dev)
3497 {
3498         return netdev2pinfo(dev)->tx_chan;
3499 }
3500 EXPORT_SYMBOL(cxgb4_port_chan);
3501
3502 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3503 {
3504         struct adapter *adap = netdev2adap(dev);
3505         u32 v1, v2, lp_count, hp_count;
3506
3507         v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3508         v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3509         if (is_t4(adap->params.chip)) {
3510                 lp_count = LP_COUNT_G(v1);
3511                 hp_count = HP_COUNT_G(v1);
3512         } else {
3513                 lp_count = LP_COUNT_T5_G(v1);
3514                 hp_count = HP_COUNT_T5_G(v2);
3515         }
3516         return lpfifo ? lp_count : hp_count;
3517 }
3518 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3519
3520 /**
3521  *      cxgb4_port_viid - get the VI id of a port
3522  *      @dev: the net device for the port
3523  *
3524  *      Return the VI id of the given port.
3525  */
3526 unsigned int cxgb4_port_viid(const struct net_device *dev)
3527 {
3528         return netdev2pinfo(dev)->viid;
3529 }
3530 EXPORT_SYMBOL(cxgb4_port_viid);
3531
3532 /**
3533  *      cxgb4_port_idx - get the index of a port
3534  *      @dev: the net device for the port
3535  *
3536  *      Return the index of the given port.
3537  */
3538 unsigned int cxgb4_port_idx(const struct net_device *dev)
3539 {
3540         return netdev2pinfo(dev)->port_id;
3541 }
3542 EXPORT_SYMBOL(cxgb4_port_idx);
3543
3544 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3545                          struct tp_tcp_stats *v6)
3546 {
3547         struct adapter *adap = pci_get_drvdata(pdev);
3548
3549         spin_lock(&adap->stats_lock);
3550         t4_tp_get_tcp_stats(adap, v4, v6);
3551         spin_unlock(&adap->stats_lock);
3552 }
3553 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3554
3555 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3556                       const unsigned int *pgsz_order)
3557 {
3558         struct adapter *adap = netdev2adap(dev);
3559
3560         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3561         t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3562                      HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3563                      HPZ3_V(pgsz_order[3]));
3564 }
3565 EXPORT_SYMBOL(cxgb4_iscsi_init);
3566
3567 int cxgb4_flush_eq_cache(struct net_device *dev)
3568 {
3569         struct adapter *adap = netdev2adap(dev);
3570         int ret;
3571
3572         ret = t4_fwaddrspace_write(adap, adap->mbox,
3573                                    0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3574         return ret;
3575 }
3576 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3577
3578 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3579 {
3580         u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3581         __be64 indices;
3582         int ret;
3583
3584         spin_lock(&adap->win0_lock);
3585         ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3586                            sizeof(indices), (__be32 *)&indices,
3587                            T4_MEMORY_READ);
3588         spin_unlock(&adap->win0_lock);
3589         if (!ret) {
3590                 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3591                 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3592         }
3593         return ret;
3594 }
3595
3596 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3597                         u16 size)
3598 {
3599         struct adapter *adap = netdev2adap(dev);
3600         u16 hw_pidx, hw_cidx;
3601         int ret;
3602
3603         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3604         if (ret)
3605                 goto out;
3606
3607         if (pidx != hw_pidx) {
3608                 u16 delta;
3609                 u32 val;
3610
3611                 if (pidx >= hw_pidx)
3612                         delta = pidx - hw_pidx;
3613                 else
3614                         delta = size - hw_pidx + pidx;
3615
3616                 if (is_t4(adap->params.chip))
3617                         val = PIDX_V(delta);
3618                 else
3619                         val = PIDX_T5_V(delta);
3620                 wmb();
3621                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3622                              QID_V(qid) | val);
3623         }
3624 out:
3625         return ret;
3626 }
3627 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3628
3629 void cxgb4_disable_db_coalescing(struct net_device *dev)
3630 {
3631         struct adapter *adap;
3632
3633         adap = netdev2adap(dev);
3634         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
3635                          NOCOALESCE_F);
3636 }
3637 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3638
3639 void cxgb4_enable_db_coalescing(struct net_device *dev)
3640 {
3641         struct adapter *adap;
3642
3643         adap = netdev2adap(dev);
3644         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3645 }
3646 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3647
3648 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3649 {
3650         struct adapter *adap;
3651         u32 offset, memtype, memaddr;
3652         u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
3653         u32 edc0_end, edc1_end, mc0_end, mc1_end;
3654         int ret;
3655
3656         adap = netdev2adap(dev);
3657
3658         offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3659
3660         /* Figure out where the offset lands in the Memory Type/Address scheme.
3661          * This code assumes that the memory is laid out starting at offset 0
3662          * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3663          * and EDC1.  Some cards will have neither MC0 nor MC1, most cards have
3664          * MC0, and some have both MC0 and MC1.
3665          */
3666         size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3667         edc0_size = EDRAM0_SIZE_G(size) << 20;
3668         size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3669         edc1_size = EDRAM1_SIZE_G(size) << 20;
3670         size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3671         mc0_size = EXT_MEM0_SIZE_G(size) << 20;
3672
3673         edc0_end = edc0_size;
3674         edc1_end = edc0_end + edc1_size;
3675         mc0_end = edc1_end + mc0_size;
3676
3677         if (offset < edc0_end) {
3678                 memtype = MEM_EDC0;
3679                 memaddr = offset;
3680         } else if (offset < edc1_end) {
3681                 memtype = MEM_EDC1;
3682                 memaddr = offset - edc0_end;
3683         } else {
3684                 if (offset < mc0_end) {
3685                         memtype = MEM_MC0;
3686                         memaddr = offset - edc1_end;
3687                 } else if (is_t4(adap->params.chip)) {
3688                         /* T4 only has a single memory channel */
3689                         goto err;
3690                 } else {
3691                         size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3692                         mc1_size = EXT_MEM1_SIZE_G(size) << 20;
3693                         mc1_end = mc0_end + mc1_size;
3694                         if (offset < mc1_end) {
3695                                 memtype = MEM_MC1;
3696                                 memaddr = offset - mc0_end;
3697                         } else {
3698                                 /* offset beyond the end of any memory */
3699                                 goto err;
3700                         }
3701                 }
3702         }
3703
3704         spin_lock(&adap->win0_lock);
3705         ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3706         spin_unlock(&adap->win0_lock);
3707         return ret;
3708
3709 err:
3710         dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3711                 stag, offset);
3712         return -EINVAL;
3713 }
3714 EXPORT_SYMBOL(cxgb4_read_tpte);
3715
3716 u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3717 {
3718         u32 hi, lo;
3719         struct adapter *adap;
3720
3721         adap = netdev2adap(dev);
3722         lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3723         hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
3724
3725         return ((u64)hi << 32) | (u64)lo;
3726 }
3727 EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3728
3729 int cxgb4_bar2_sge_qregs(struct net_device *dev,
3730                          unsigned int qid,
3731                          enum cxgb4_bar2_qtype qtype,
3732                          u64 *pbar2_qoffset,
3733                          unsigned int *pbar2_qid)
3734 {
3735         return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
3736                                  qid,
3737                                  (qtype == CXGB4_BAR2_QTYPE_EGRESS
3738                                   ? T4_BAR2_QTYPE_EGRESS
3739                                   : T4_BAR2_QTYPE_INGRESS),
3740                                  pbar2_qoffset,
3741                                  pbar2_qid);
3742 }
3743 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3744
3745 static struct pci_driver cxgb4_driver;
3746
3747 static void check_neigh_update(struct neighbour *neigh)
3748 {
3749         const struct device *parent;
3750         const struct net_device *netdev = neigh->dev;
3751
3752         if (netdev->priv_flags & IFF_802_1Q_VLAN)
3753                 netdev = vlan_dev_real_dev(netdev);
3754         parent = netdev->dev.parent;
3755         if (parent && parent->driver == &cxgb4_driver.driver)
3756                 t4_l2t_update(dev_get_drvdata(parent), neigh);
3757 }
3758
3759 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3760                        void *data)
3761 {
3762         switch (event) {
3763         case NETEVENT_NEIGH_UPDATE:
3764                 check_neigh_update(data);
3765                 break;
3766         case NETEVENT_REDIRECT:
3767         default:
3768                 break;
3769         }
3770         return 0;
3771 }
3772
3773 static bool netevent_registered;
3774 static struct notifier_block cxgb4_netevent_nb = {
3775         .notifier_call = netevent_cb
3776 };
3777
3778 static void drain_db_fifo(struct adapter *adap, int usecs)
3779 {
3780         u32 v1, v2, lp_count, hp_count;
3781
3782         do {
3783                 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3784                 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3785                 if (is_t4(adap->params.chip)) {
3786                         lp_count = LP_COUNT_G(v1);
3787                         hp_count = HP_COUNT_G(v1);
3788                 } else {
3789                         lp_count = LP_COUNT_T5_G(v1);
3790                         hp_count = HP_COUNT_T5_G(v2);
3791                 }
3792
3793                 if (lp_count == 0 && hp_count == 0)
3794                         break;
3795                 set_current_state(TASK_UNINTERRUPTIBLE);
3796                 schedule_timeout(usecs_to_jiffies(usecs));
3797         } while (1);
3798 }
3799
3800 static void disable_txq_db(struct sge_txq *q)
3801 {
3802         unsigned long flags;
3803
3804         spin_lock_irqsave(&q->db_lock, flags);
3805         q->db_disabled = 1;
3806         spin_unlock_irqrestore(&q->db_lock, flags);
3807 }
3808
3809 static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3810 {
3811         spin_lock_irq(&q->db_lock);
3812         if (q->db_pidx_inc) {
3813                 /* Make sure that all writes to the TX descriptors
3814                  * are committed before we tell HW about them.
3815                  */
3816                 wmb();
3817                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3818                              QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
3819                 q->db_pidx_inc = 0;
3820         }
3821         q->db_disabled = 0;
3822         spin_unlock_irq(&q->db_lock);
3823 }
3824
3825 static void disable_dbs(struct adapter *adap)
3826 {
3827         int i;
3828
3829         for_each_ethrxq(&adap->sge, i)
3830                 disable_txq_db(&adap->sge.ethtxq[i].q);
3831         for_each_ofldrxq(&adap->sge, i)
3832                 disable_txq_db(&adap->sge.ofldtxq[i].q);
3833         for_each_port(adap, i)
3834                 disable_txq_db(&adap->sge.ctrlq[i].q);
3835 }
3836
3837 static void enable_dbs(struct adapter *adap)
3838 {
3839         int i;
3840
3841         for_each_ethrxq(&adap->sge, i)
3842                 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3843         for_each_ofldrxq(&adap->sge, i)
3844                 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3845         for_each_port(adap, i)
3846                 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3847 }
3848
3849 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3850 {
3851         if (adap->uld_handle[CXGB4_ULD_RDMA])
3852                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3853                                 cmd);
3854 }
3855
3856 static void process_db_full(struct work_struct *work)
3857 {
3858         struct adapter *adap;
3859
3860         adap = container_of(work, struct adapter, db_full_task);
3861
3862         drain_db_fifo(adap, dbfifo_drain_delay);
3863         enable_dbs(adap);
3864         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3865         t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3866                          DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3867                          DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3868 }
3869
3870 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3871 {
3872         u16 hw_pidx, hw_cidx;
3873         int ret;
3874
3875         spin_lock_irq(&q->db_lock);
3876         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3877         if (ret)
3878                 goto out;
3879         if (q->db_pidx != hw_pidx) {
3880                 u16 delta;
3881                 u32 val;
3882
3883                 if (q->db_pidx >= hw_pidx)
3884                         delta = q->db_pidx - hw_pidx;
3885                 else
3886                         delta = q->size - hw_pidx + q->db_pidx;
3887
3888                 if (is_t4(adap->params.chip))
3889                         val = PIDX_V(delta);
3890                 else
3891                         val = PIDX_T5_V(delta);
3892                 wmb();
3893                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3894                              QID_V(q->cntxt_id) | val);
3895         }
3896 out:
3897         q->db_disabled = 0;
3898         q->db_pidx_inc = 0;
3899         spin_unlock_irq(&q->db_lock);
3900         if (ret)
3901                 CH_WARN(adap, "DB drop recovery failed.\n");
3902 }
3903 static void recover_all_queues(struct adapter *adap)
3904 {
3905         int i;
3906
3907         for_each_ethrxq(&adap->sge, i)
3908                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3909         for_each_ofldrxq(&adap->sge, i)
3910                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3911         for_each_port(adap, i)
3912                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3913 }
3914
3915 static void process_db_drop(struct work_struct *work)
3916 {
3917         struct adapter *adap;
3918
3919         adap = container_of(work, struct adapter, db_drop_task);
3920
3921         if (is_t4(adap->params.chip)) {
3922                 drain_db_fifo(adap, dbfifo_drain_delay);
3923                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3924                 drain_db_fifo(adap, dbfifo_drain_delay);
3925                 recover_all_queues(adap);
3926                 drain_db_fifo(adap, dbfifo_drain_delay);
3927                 enable_dbs(adap);
3928                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3929         } else {
3930                 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3931                 u16 qid = (dropped_db >> 15) & 0x1ffff;
3932                 u16 pidx_inc = dropped_db & 0x1fff;
3933                 u64 bar2_qoffset;
3934                 unsigned int bar2_qid;
3935                 int ret;
3936
3937                 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
3938                                         &bar2_qoffset, &bar2_qid);
3939                 if (ret)
3940                         dev_err(adap->pdev_dev, "doorbell drop recovery: "
3941                                 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3942                 else
3943                         writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
3944                                adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
3945
3946                 /* Re-enable BAR2 WC */
3947                 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3948         }
3949
3950         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
3951 }
3952
3953 void t4_db_full(struct adapter *adap)
3954 {
3955         if (is_t4(adap->params.chip)) {
3956                 disable_dbs(adap);
3957                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3958                 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3959                                  DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
3960                 queue_work(adap->workq, &adap->db_full_task);
3961         }
3962 }
3963
3964 void t4_db_dropped(struct adapter *adap)
3965 {
3966         if (is_t4(adap->params.chip)) {
3967                 disable_dbs(adap);
3968                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3969         }
3970         queue_work(adap->workq, &adap->db_drop_task);
3971 }
3972
3973 static void uld_attach(struct adapter *adap, unsigned int uld)
3974 {
3975         void *handle;
3976         struct cxgb4_lld_info lli;
3977         unsigned short i;
3978
3979         lli.pdev = adap->pdev;
3980         lli.pf = adap->fn;
3981         lli.l2t = adap->l2t;
3982         lli.tids = &adap->tids;
3983         lli.ports = adap->port;
3984         lli.vr = &adap->vres;
3985         lli.mtus = adap->params.mtus;
3986         if (uld == CXGB4_ULD_RDMA) {
3987                 lli.rxq_ids = adap->sge.rdma_rxq;
3988                 lli.ciq_ids = adap->sge.rdma_ciq;
3989                 lli.nrxq = adap->sge.rdmaqs;
3990                 lli.nciq = adap->sge.rdmaciqs;
3991         } else if (uld == CXGB4_ULD_ISCSI) {
3992                 lli.rxq_ids = adap->sge.ofld_rxq;
3993                 lli.nrxq = adap->sge.ofldqsets;
3994         }
3995         lli.ntxq = adap->sge.ofldqsets;
3996         lli.nchan = adap->params.nports;
3997         lli.nports = adap->params.nports;
3998         lli.wr_cred = adap->params.ofldq_wr_cred;
3999         lli.adapter_type = adap->params.chip;
4000         lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
4001         lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
4002         lli.udb_density = 1 << adap->params.sge.eq_qpp;
4003         lli.ucq_density = 1 << adap->params.sge.iq_qpp;
4004         lli.filt_mode = adap->params.tp.vlan_pri_map;
4005         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
4006         for (i = 0; i < NCHAN; i++)
4007                 lli.tx_modq[i] = i;
4008         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
4009         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
4010         lli.fw_vers = adap->params.fw_vers;
4011         lli.dbfifo_int_thresh = dbfifo_int_thresh;
4012         lli.sge_ingpadboundary = adap->sge.fl_align;
4013         lli.sge_egrstatuspagesize = adap->sge.stat_len;
4014         lli.sge_pktshift = adap->sge.pktshift;
4015         lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4016         lli.max_ordird_qp = adap->params.max_ordird_qp;
4017         lli.max_ird_adapter = adap->params.max_ird_adapter;
4018         lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
4019
4020         handle = ulds[uld].add(&lli);
4021         if (IS_ERR(handle)) {
4022                 dev_warn(adap->pdev_dev,
4023                          "could not attach to the %s driver, error %ld\n",
4024                          uld_str[uld], PTR_ERR(handle));
4025                 return;
4026         }
4027
4028         adap->uld_handle[uld] = handle;
4029
4030         if (!netevent_registered) {
4031                 register_netevent_notifier(&cxgb4_netevent_nb);
4032                 netevent_registered = true;
4033         }
4034
4035         if (adap->flags & FULL_INIT_DONE)
4036                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
4037 }
4038
4039 static void attach_ulds(struct adapter *adap)
4040 {
4041         unsigned int i;
4042
4043         spin_lock(&adap_rcu_lock);
4044         list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4045         spin_unlock(&adap_rcu_lock);
4046
4047         mutex_lock(&uld_mutex);
4048         list_add_tail(&adap->list_node, &adapter_list);
4049         for (i = 0; i < CXGB4_ULD_MAX; i++)
4050                 if (ulds[i].add)
4051                         uld_attach(adap, i);
4052         mutex_unlock(&uld_mutex);
4053 }
4054
4055 static void detach_ulds(struct adapter *adap)
4056 {
4057         unsigned int i;
4058
4059         mutex_lock(&uld_mutex);
4060         list_del(&adap->list_node);
4061         for (i = 0; i < CXGB4_ULD_MAX; i++)
4062                 if (adap->uld_handle[i]) {
4063                         ulds[i].state_change(adap->uld_handle[i],
4064                                              CXGB4_STATE_DETACH);
4065                         adap->uld_handle[i] = NULL;
4066                 }
4067         if (netevent_registered && list_empty(&adapter_list)) {
4068                 unregister_netevent_notifier(&cxgb4_netevent_nb);
4069                 netevent_registered = false;
4070         }
4071         mutex_unlock(&uld_mutex);
4072
4073         spin_lock(&adap_rcu_lock);
4074         list_del_rcu(&adap->rcu_node);
4075         spin_unlock(&adap_rcu_lock);
4076 }
4077
4078 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4079 {
4080         unsigned int i;
4081
4082         mutex_lock(&uld_mutex);
4083         for (i = 0; i < CXGB4_ULD_MAX; i++)
4084                 if (adap->uld_handle[i])
4085                         ulds[i].state_change(adap->uld_handle[i], new_state);
4086         mutex_unlock(&uld_mutex);
4087 }
4088
4089 /**
4090  *      cxgb4_register_uld - register an upper-layer driver
4091  *      @type: the ULD type
4092  *      @p: the ULD methods
4093  *
4094  *      Registers an upper-layer driver with this driver and notifies the ULD
4095  *      about any presently available devices that support its type.  Returns
4096  *      %-EBUSY if a ULD of the same type is already registered.
4097  */
4098 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4099 {
4100         int ret = 0;
4101         struct adapter *adap;
4102
4103         if (type >= CXGB4_ULD_MAX)
4104                 return -EINVAL;
4105         mutex_lock(&uld_mutex);
4106         if (ulds[type].add) {
4107                 ret = -EBUSY;
4108                 goto out;
4109         }
4110         ulds[type] = *p;
4111         list_for_each_entry(adap, &adapter_list, list_node)
4112                 uld_attach(adap, type);
4113 out:    mutex_unlock(&uld_mutex);
4114         return ret;
4115 }
4116 EXPORT_SYMBOL(cxgb4_register_uld);
4117
4118 /**
4119  *      cxgb4_unregister_uld - unregister an upper-layer driver
4120  *      @type: the ULD type
4121  *
4122  *      Unregisters an existing upper-layer driver.
4123  */
4124 int cxgb4_unregister_uld(enum cxgb4_uld type)
4125 {
4126         struct adapter *adap;
4127
4128         if (type >= CXGB4_ULD_MAX)
4129                 return -EINVAL;
4130         mutex_lock(&uld_mutex);
4131         list_for_each_entry(adap, &adapter_list, list_node)
4132                 adap->uld_handle[type] = NULL;
4133         ulds[type].add = NULL;
4134         mutex_unlock(&uld_mutex);
4135         return 0;
4136 }
4137 EXPORT_SYMBOL(cxgb4_unregister_uld);
4138
4139 #if IS_ENABLED(CONFIG_IPV6)
4140 static int cxgb4_inet6addr_handler(struct notifier_block *this,
4141                                    unsigned long event, void *data)
4142 {
4143         struct inet6_ifaddr *ifa = data;
4144         struct net_device *event_dev = ifa->idev->dev;
4145         const struct device *parent = NULL;
4146 #if IS_ENABLED(CONFIG_BONDING)
4147         struct adapter *adap;
4148 #endif
4149         if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4150                 event_dev = vlan_dev_real_dev(event_dev);
4151 #if IS_ENABLED(CONFIG_BONDING)
4152         if (event_dev->flags & IFF_MASTER) {
4153                 list_for_each_entry(adap, &adapter_list, list_node) {
4154                         switch (event) {
4155                         case NETDEV_UP:
4156                                 cxgb4_clip_get(adap->port[0],
4157                                                (const u32 *)ifa, 1);
4158                                 break;
4159                         case NETDEV_DOWN:
4160                                 cxgb4_clip_release(adap->port[0],
4161                                                    (const u32 *)ifa, 1);
4162                                 break;
4163                         default:
4164                                 break;
4165                         }
4166                 }
4167                 return NOTIFY_OK;
4168         }
4169 #endif
4170
4171         if (event_dev)
4172                 parent = event_dev->dev.parent;
4173
4174         if (parent && parent->driver == &cxgb4_driver.driver) {
4175                 switch (event) {
4176                 case NETDEV_UP:
4177                         cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
4178                         break;
4179                 case NETDEV_DOWN:
4180                         cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
4181                         break;
4182                 default:
4183                         break;
4184                 }
4185         }
4186         return NOTIFY_OK;
4187 }
4188
4189 static bool inet6addr_registered;
4190 static struct notifier_block cxgb4_inet6addr_notifier = {
4191         .notifier_call = cxgb4_inet6addr_handler
4192 };
4193
4194 static void update_clip(const struct adapter *adap)
4195 {
4196         int i;
4197         struct net_device *dev;
4198         int ret;
4199
4200         rcu_read_lock();
4201
4202         for (i = 0; i < MAX_NPORTS; i++) {
4203                 dev = adap->port[i];
4204                 ret = 0;
4205
4206                 if (dev)
4207                         ret = cxgb4_update_root_dev_clip(dev);
4208
4209                 if (ret < 0)
4210                         break;
4211         }
4212         rcu_read_unlock();
4213 }
4214 #endif /* IS_ENABLED(CONFIG_IPV6) */
4215
4216 /**
4217  *      cxgb_up - enable the adapter
4218  *      @adap: adapter being enabled
4219  *
4220  *      Called when the first port is enabled, this function performs the
4221  *      actions necessary to make an adapter operational, such as completing
4222  *      the initialization of HW modules, and enabling interrupts.
4223  *
4224  *      Must be called with the rtnl lock held.
4225  */
4226 static int cxgb_up(struct adapter *adap)
4227 {
4228         int err;
4229
4230         err = setup_sge_queues(adap);
4231         if (err)
4232                 goto out;
4233         err = setup_rss(adap);
4234         if (err)
4235                 goto freeq;
4236
4237         if (adap->flags & USING_MSIX) {
4238                 name_msix_vecs(adap);
4239                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4240                                   adap->msix_info[0].desc, adap);
4241                 if (err)
4242                         goto irq_err;
4243
4244                 err = request_msix_queue_irqs(adap);
4245                 if (err) {
4246                         free_irq(adap->msix_info[0].vec, adap);
4247                         goto irq_err;
4248                 }
4249         } else {
4250                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4251                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4252                                   adap->port[0]->name, adap);
4253                 if (err)
4254                         goto irq_err;
4255         }
4256         enable_rx(adap);
4257         t4_sge_start(adap);
4258         t4_intr_enable(adap);
4259         adap->flags |= FULL_INIT_DONE;
4260         notify_ulds(adap, CXGB4_STATE_UP);
4261 #if IS_ENABLED(CONFIG_IPV6)
4262         update_clip(adap);
4263 #endif
4264  out:
4265         return err;
4266  irq_err:
4267         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4268  freeq:
4269         t4_free_sge_resources(adap);
4270         goto out;
4271 }
4272
4273 static void cxgb_down(struct adapter *adapter)
4274 {
4275         cancel_work_sync(&adapter->tid_release_task);
4276         cancel_work_sync(&adapter->db_full_task);
4277         cancel_work_sync(&adapter->db_drop_task);
4278         adapter->tid_release_task_busy = false;
4279         adapter->tid_release_head = NULL;
4280
4281         t4_sge_stop(adapter);
4282         t4_free_sge_resources(adapter);
4283         adapter->flags &= ~FULL_INIT_DONE;
4284 }
4285
4286 /*
4287  * net_device operations
4288  */
4289 static int cxgb_open(struct net_device *dev)
4290 {
4291         int err;
4292         struct port_info *pi = netdev_priv(dev);
4293         struct adapter *adapter = pi->adapter;
4294
4295         netif_carrier_off(dev);
4296
4297         if (!(adapter->flags & FULL_INIT_DONE)) {
4298                 err = cxgb_up(adapter);
4299                 if (err < 0)
4300                         return err;
4301         }
4302
4303         err = link_start(dev);
4304         if (!err)
4305                 netif_tx_start_all_queues(dev);
4306         return err;
4307 }
4308
4309 static int cxgb_close(struct net_device *dev)
4310 {
4311         struct port_info *pi = netdev_priv(dev);
4312         struct adapter *adapter = pi->adapter;
4313
4314         netif_tx_stop_all_queues(dev);
4315         netif_carrier_off(dev);
4316         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4317 }
4318
4319 /* Return an error number if the indicated filter isn't writable ...
4320  */
4321 static int writable_filter(struct filter_entry *f)
4322 {
4323         if (f->locked)
4324                 return -EPERM;
4325         if (f->pending)
4326                 return -EBUSY;
4327
4328         return 0;
4329 }
4330
4331 /* Delete the filter at the specified index (if valid).  The checks for all
4332  * the common problems with doing this like the filter being locked, currently
4333  * pending in another operation, etc.
4334  */
4335 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4336 {
4337         struct filter_entry *f;
4338         int ret;
4339
4340         if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4341                 return -EINVAL;
4342
4343         f = &adapter->tids.ftid_tab[fidx];
4344         ret = writable_filter(f);
4345         if (ret)
4346                 return ret;
4347         if (f->valid)
4348                 return del_filter_wr(adapter, fidx);
4349
4350         return 0;
4351 }
4352
4353 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4354                 __be32 sip, __be16 sport, __be16 vlan,
4355                 unsigned int queue, unsigned char port, unsigned char mask)
4356 {
4357         int ret;
4358         struct filter_entry *f;
4359         struct adapter *adap;
4360         int i;
4361         u8 *val;
4362
4363         adap = netdev2adap(dev);
4364
4365         /* Adjust stid to correct filter index */
4366         stid -= adap->tids.sftid_base;
4367         stid += adap->tids.nftids;
4368
4369         /* Check to make sure the filter requested is writable ...
4370          */
4371         f = &adap->tids.ftid_tab[stid];
4372         ret = writable_filter(f);
4373         if (ret)
4374                 return ret;
4375
4376         /* Clear out any old resources being used by the filter before
4377          * we start constructing the new filter.
4378          */
4379         if (f->valid)
4380                 clear_filter(adap, f);
4381
4382         /* Clear out filter specifications */
4383         memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4384         f->fs.val.lport = cpu_to_be16(sport);
4385         f->fs.mask.lport  = ~0;
4386         val = (u8 *)&sip;
4387         if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4388                 for (i = 0; i < 4; i++) {
4389                         f->fs.val.lip[i] = val[i];
4390                         f->fs.mask.lip[i] = ~0;
4391                 }
4392                 if (adap->params.tp.vlan_pri_map & PORT_F) {
4393                         f->fs.val.iport = port;
4394                         f->fs.mask.iport = mask;
4395                 }
4396         }
4397
4398         if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
4399                 f->fs.val.proto = IPPROTO_TCP;
4400                 f->fs.mask.proto = ~0;
4401         }
4402
4403         f->fs.dirsteer = 1;
4404         f->fs.iq = queue;
4405         /* Mark filter as locked */
4406         f->locked = 1;
4407         f->fs.rpttid = 1;
4408
4409         ret = set_filter_wr(adap, stid);
4410         if (ret) {
4411                 clear_filter(adap, f);
4412                 return ret;
4413         }
4414
4415         return 0;
4416 }
4417 EXPORT_SYMBOL(cxgb4_create_server_filter);
4418
4419 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4420                 unsigned int queue, bool ipv6)
4421 {
4422         int ret;
4423         struct filter_entry *f;
4424         struct adapter *adap;
4425
4426         adap = netdev2adap(dev);
4427
4428         /* Adjust stid to correct filter index */
4429         stid -= adap->tids.sftid_base;
4430         stid += adap->tids.nftids;
4431
4432         f = &adap->tids.ftid_tab[stid];
4433         /* Unlock the filter */
4434         f->locked = 0;
4435
4436         ret = delete_filter(adap, stid);
4437         if (ret)
4438                 return ret;
4439
4440         return 0;
4441 }
4442 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4443
4444 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4445                                                 struct rtnl_link_stats64 *ns)
4446 {
4447         struct port_stats stats;
4448         struct port_info *p = netdev_priv(dev);
4449         struct adapter *adapter = p->adapter;
4450
4451         /* Block retrieving statistics during EEH error
4452          * recovery. Otherwise, the recovery might fail
4453          * and the PCI device will be removed permanently
4454          */
4455         spin_lock(&adapter->stats_lock);
4456         if (!netif_device_present(dev)) {
4457                 spin_unlock(&adapter->stats_lock);
4458                 return ns;
4459         }
4460         t4_get_port_stats(adapter, p->tx_chan, &stats);
4461         spin_unlock(&adapter->stats_lock);
4462
4463         ns->tx_bytes   = stats.tx_octets;
4464         ns->tx_packets = stats.tx_frames;
4465         ns->rx_bytes   = stats.rx_octets;
4466         ns->rx_packets = stats.rx_frames;
4467         ns->multicast  = stats.rx_mcast_frames;
4468
4469         /* detailed rx_errors */
4470         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4471                                stats.rx_runt;
4472         ns->rx_over_errors   = 0;
4473         ns->rx_crc_errors    = stats.rx_fcs_err;
4474         ns->rx_frame_errors  = stats.rx_symbol_err;
4475         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
4476                                stats.rx_ovflow2 + stats.rx_ovflow3 +
4477                                stats.rx_trunc0 + stats.rx_trunc1 +
4478                                stats.rx_trunc2 + stats.rx_trunc3;
4479         ns->rx_missed_errors = 0;
4480
4481         /* detailed tx_errors */
4482         ns->tx_aborted_errors   = 0;
4483         ns->tx_carrier_errors   = 0;
4484         ns->tx_fifo_errors      = 0;
4485         ns->tx_heartbeat_errors = 0;
4486         ns->tx_window_errors    = 0;
4487
4488         ns->tx_errors = stats.tx_error_frames;
4489         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4490                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4491         return ns;
4492 }
4493
4494 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4495 {
4496         unsigned int mbox;
4497         int ret = 0, prtad, devad;
4498         struct port_info *pi = netdev_priv(dev);
4499         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4500
4501         switch (cmd) {
4502         case SIOCGMIIPHY:
4503                 if (pi->mdio_addr < 0)
4504                         return -EOPNOTSUPP;
4505                 data->phy_id = pi->mdio_addr;
4506                 break;
4507         case SIOCGMIIREG:
4508         case SIOCSMIIREG:
4509                 if (mdio_phy_id_is_c45(data->phy_id)) {
4510                         prtad = mdio_phy_id_prtad(data->phy_id);
4511                         devad = mdio_phy_id_devad(data->phy_id);
4512                 } else if (data->phy_id < 32) {
4513                         prtad = data->phy_id;
4514                         devad = 0;
4515                         data->reg_num &= 0x1f;
4516                 } else
4517                         return -EINVAL;
4518
4519                 mbox = pi->adapter->fn;
4520                 if (cmd == SIOCGMIIREG)
4521                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4522                                          data->reg_num, &data->val_out);
4523                 else
4524                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4525                                          data->reg_num, data->val_in);
4526                 break;
4527         default:
4528                 return -EOPNOTSUPP;
4529         }
4530         return ret;
4531 }
4532
4533 static void cxgb_set_rxmode(struct net_device *dev)
4534 {
4535         /* unfortunately we can't return errors to the stack */
4536         set_rxmode(dev, -1, false);
4537 }
4538
4539 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4540 {
4541         int ret;
4542         struct port_info *pi = netdev_priv(dev);
4543
4544         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
4545                 return -EINVAL;
4546         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4547                             -1, -1, -1, true);
4548         if (!ret)
4549                 dev->mtu = new_mtu;
4550         return ret;
4551 }
4552
4553 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4554 {
4555         int ret;
4556         struct sockaddr *addr = p;
4557         struct port_info *pi = netdev_priv(dev);
4558
4559         if (!is_valid_ether_addr(addr->sa_data))
4560                 return -EADDRNOTAVAIL;
4561
4562         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4563                             pi->xact_addr_filt, addr->sa_data, true, true);
4564         if (ret < 0)
4565                 return ret;
4566
4567         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4568         pi->xact_addr_filt = ret;
4569         return 0;
4570 }
4571
4572 #ifdef CONFIG_NET_POLL_CONTROLLER
4573 static void cxgb_netpoll(struct net_device *dev)
4574 {
4575         struct port_info *pi = netdev_priv(dev);
4576         struct adapter *adap = pi->adapter;
4577
4578         if (adap->flags & USING_MSIX) {
4579                 int i;
4580                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4581
4582                 for (i = pi->nqsets; i; i--, rx++)
4583                         t4_sge_intr_msix(0, &rx->rspq);
4584         } else
4585                 t4_intr_handler(adap)(0, adap);
4586 }
4587 #endif
4588
4589 static const struct net_device_ops cxgb4_netdev_ops = {
4590         .ndo_open             = cxgb_open,
4591         .ndo_stop             = cxgb_close,
4592         .ndo_start_xmit       = t4_eth_xmit,
4593         .ndo_select_queue     = cxgb_select_queue,
4594         .ndo_get_stats64      = cxgb_get_stats,
4595         .ndo_set_rx_mode      = cxgb_set_rxmode,
4596         .ndo_set_mac_address  = cxgb_set_mac_addr,
4597         .ndo_set_features     = cxgb_set_features,
4598         .ndo_validate_addr    = eth_validate_addr,
4599         .ndo_do_ioctl         = cxgb_ioctl,
4600         .ndo_change_mtu       = cxgb_change_mtu,
4601 #ifdef CONFIG_NET_POLL_CONTROLLER
4602         .ndo_poll_controller  = cxgb_netpoll,
4603 #endif
4604 #ifdef CONFIG_NET_RX_BUSY_POLL
4605         .ndo_busy_poll        = cxgb_busy_poll,
4606 #endif
4607
4608 };
4609
4610 void t4_fatal_err(struct adapter *adap)
4611 {
4612         t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
4613         t4_intr_disable(adap);
4614         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4615 }
4616
4617 /* Return the specified PCI-E Configuration Space register from our Physical
4618  * Function.  We try first via a Firmware LDST Command since we prefer to let
4619  * the firmware own all of these registers, but if that fails we go for it
4620  * directly ourselves.
4621  */
4622 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4623 {
4624         struct fw_ldst_cmd ldst_cmd;
4625         u32 val;
4626         int ret;
4627
4628         /* Construct and send the Firmware LDST Command to retrieve the
4629          * specified PCI-E Configuration Space register.
4630          */
4631         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4632         ldst_cmd.op_to_addrspace =
4633                 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4634                       FW_CMD_REQUEST_F |
4635                       FW_CMD_READ_F |
4636                       FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
4637         ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
4638         ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
4639         ldst_cmd.u.pcie.ctrl_to_fn =
4640                 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
4641         ldst_cmd.u.pcie.r = reg;
4642         ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4643                          &ldst_cmd);
4644
4645         /* If the LDST Command suucceeded, exctract the returned register
4646          * value.  Otherwise read it directly ourself.
4647          */
4648         if (ret == 0)
4649                 val = ntohl(ldst_cmd.u.pcie.data[0]);
4650         else
4651                 t4_hw_pci_read_cfg4(adap, reg, &val);
4652
4653         return val;
4654 }
4655
4656 static void setup_memwin(struct adapter *adap)
4657 {
4658         u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
4659
4660         if (is_t4(adap->params.chip)) {
4661                 u32 bar0;
4662
4663                 /* Truncation intentional: we only read the bottom 32-bits of
4664                  * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
4665                  * mechanism to read BAR0 instead of using
4666                  * pci_resource_start() because we could be operating from
4667                  * within a Virtual Machine which is trapping our accesses to
4668                  * our Configuration Space and we need to set up the PCI-E
4669                  * Memory Window decoders with the actual addresses which will
4670                  * be coming across the PCI-E link.
4671                  */
4672                 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4673                 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4674                 adap->t4_bar0 = bar0;
4675
4676                 mem_win0_base = bar0 + MEMWIN0_BASE;
4677                 mem_win1_base = bar0 + MEMWIN1_BASE;
4678                 mem_win2_base = bar0 + MEMWIN2_BASE;
4679                 mem_win2_aperture = MEMWIN2_APERTURE;
4680         } else {
4681                 /* For T5, only relative offset inside the PCIe BAR is passed */
4682                 mem_win0_base = MEMWIN0_BASE;
4683                 mem_win1_base = MEMWIN1_BASE;
4684                 mem_win2_base = MEMWIN2_BASE_T5;
4685                 mem_win2_aperture = MEMWIN2_APERTURE_T5;
4686         }
4687         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4688                      mem_win0_base | BIR_V(0) |
4689                      WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4690         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4691                      mem_win1_base | BIR_V(0) |
4692                      WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4693         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4694                      mem_win2_base | BIR_V(0) |
4695                      WINDOW_V(ilog2(mem_win2_aperture) - 10));
4696         t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
4697 }
4698
4699 static void setup_memwin_rdma(struct adapter *adap)
4700 {
4701         if (adap->vres.ocq.size) {
4702                 u32 start;
4703                 unsigned int sz_kb;
4704
4705                 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4706                 start &= PCI_BASE_ADDRESS_MEM_MASK;
4707                 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4708                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4709                 t4_write_reg(adap,
4710                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4711                              start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
4712                 t4_write_reg(adap,
4713                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
4714                              adap->vres.ocq.start);
4715                 t4_read_reg(adap,
4716                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
4717         }
4718 }
4719
4720 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4721 {
4722         u32 v;
4723         int ret;
4724
4725         /* get device capabilities */
4726         memset(c, 0, sizeof(*c));
4727         c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4728                                FW_CMD_REQUEST_F | FW_CMD_READ_F);
4729         c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4730         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4731         if (ret < 0)
4732                 return ret;
4733
4734         /* select capabilities we'll be using */
4735         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4736                 if (!vf_acls)
4737                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4738                 else
4739                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4740         } else if (vf_acls) {
4741                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4742                 return ret;
4743         }
4744         c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4745                                FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
4746         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4747         if (ret < 0)
4748                 return ret;
4749
4750         ret = t4_config_glbl_rss(adap, adap->fn,
4751                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4752                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4753                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
4754         if (ret < 0)
4755                 return ret;
4756
4757         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, adap->sge.egr_sz, 64,
4758                           MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF,
4759                           FW_CMD_CAP_PF);
4760         if (ret < 0)
4761                 return ret;
4762
4763         t4_sge_init(adap);
4764
4765         /* tweak some settings */
4766         t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
4767         t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
4768         t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4769         v = t4_read_reg(adap, TP_PIO_DATA_A);
4770         t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
4771
4772         /* first 4 Tx modulation queues point to consecutive Tx channels */
4773         adap->params.tp.tx_modq_map = 0xE4;
4774         t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4775                      TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
4776
4777         /* associate each Tx modulation queue with consecutive Tx channels */
4778         v = 0x84218421;
4779         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4780                           &v, 1, TP_TX_SCHED_HDR_A);
4781         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4782                           &v, 1, TP_TX_SCHED_FIFO_A);
4783         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4784                           &v, 1, TP_TX_SCHED_PCMD_A);
4785
4786 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4787         if (is_offload(adap)) {
4788                 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4789                              TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4790                              TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4791                              TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4792                              TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4793                 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4794                              TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4795                              TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4796                              TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4797                              TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4798         }
4799
4800         /* get basic stuff going */
4801         return t4_early_init(adap, adap->fn);
4802 }
4803
4804 /*
4805  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
4806  */
4807 #define MAX_ATIDS 8192U
4808
4809 /*
4810  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4811  *
4812  * If the firmware we're dealing with has Configuration File support, then
4813  * we use that to perform all configuration
4814  */
4815
4816 /*
4817  * Tweak configuration based on module parameters, etc.  Most of these have
4818  * defaults assigned to them by Firmware Configuration Files (if we're using
4819  * them) but need to be explicitly set if we're using hard-coded
4820  * initialization.  But even in the case of using Firmware Configuration
4821  * Files, we'd like to expose the ability to change these via module
4822  * parameters so these are essentially common tweaks/settings for
4823  * Configuration Files and hard-coded initialization ...
4824  */
4825 static int adap_init0_tweaks(struct adapter *adapter)
4826 {
4827         /*
4828          * Fix up various Host-Dependent Parameters like Page Size, Cache
4829          * Line Size, etc.  The firmware default is for a 4KB Page Size and
4830          * 64B Cache Line Size ...
4831          */
4832         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4833
4834         /*
4835          * Process module parameters which affect early initialization.
4836          */
4837         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4838                 dev_err(&adapter->pdev->dev,
4839                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
4840                         rx_dma_offset);
4841                 rx_dma_offset = 2;
4842         }
4843         t4_set_reg_field(adapter, SGE_CONTROL_A,
4844                          PKTSHIFT_V(PKTSHIFT_M),
4845                          PKTSHIFT_V(rx_dma_offset));
4846
4847         /*
4848          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4849          * adds the pseudo header itself.
4850          */
4851         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4852                                CSUM_HAS_PSEUDO_HDR_F, 0);
4853
4854         return 0;
4855 }
4856
4857 /*
4858  * Attempt to initialize the adapter via a Firmware Configuration File.
4859  */
4860 static int adap_init0_config(struct adapter *adapter, int reset)
4861 {
4862         struct fw_caps_config_cmd caps_cmd;
4863         const struct firmware *cf;
4864         unsigned long mtype = 0, maddr = 0;
4865         u32 finiver, finicsum, cfcsum;
4866         int ret;
4867         int config_issued = 0;
4868         char *fw_config_file, fw_config_file_path[256];
4869         char *config_name = NULL;
4870
4871         /*
4872          * Reset device if necessary.
4873          */
4874         if (reset) {
4875                 ret = t4_fw_reset(adapter, adapter->mbox,
4876                                   PIORSTMODE_F | PIORST_F);
4877                 if (ret < 0)
4878                         goto bye;
4879         }
4880
4881         /*
4882          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4883          * then use that.  Otherwise, use the configuration file stored
4884          * in the adapter flash ...
4885          */
4886         switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
4887         case CHELSIO_T4:
4888                 fw_config_file = FW4_CFNAME;
4889                 break;
4890         case CHELSIO_T5:
4891                 fw_config_file = FW5_CFNAME;
4892                 break;
4893         default:
4894                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4895                        adapter->pdev->device);
4896                 ret = -EINVAL;
4897                 goto bye;
4898         }
4899
4900         ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4901         if (ret < 0) {
4902                 config_name = "On FLASH";
4903                 mtype = FW_MEMTYPE_CF_FLASH;
4904                 maddr = t4_flash_cfg_addr(adapter);
4905         } else {
4906                 u32 params[7], val[7];
4907
4908                 sprintf(fw_config_file_path,
4909                         "/lib/firmware/%s", fw_config_file);
4910                 config_name = fw_config_file_path;
4911
4912                 if (cf->size >= FLASH_CFG_MAX_SIZE)
4913                         ret = -ENOMEM;
4914                 else {
4915                         params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4916                              FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
4917                         ret = t4_query_params(adapter, adapter->mbox,
4918                                               adapter->fn, 0, 1, params, val);
4919                         if (ret == 0) {
4920                                 /*
4921                                  * For t4_memory_rw() below addresses and
4922                                  * sizes have to be in terms of multiples of 4
4923                                  * bytes.  So, if the Configuration File isn't
4924                                  * a multiple of 4 bytes in length we'll have
4925                                  * to write that out separately since we can't
4926                                  * guarantee that the bytes following the
4927                                  * residual byte in the buffer returned by
4928                                  * request_firmware() are zeroed out ...
4929                                  */
4930                                 size_t resid = cf->size & 0x3;
4931                                 size_t size = cf->size & ~0x3;
4932                                 __be32 *data = (__be32 *)cf->data;
4933
4934                                 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4935                                 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
4936
4937                                 spin_lock(&adapter->win0_lock);
4938                                 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4939                                                    size, data, T4_MEMORY_WRITE);
4940                                 if (ret == 0 && resid != 0) {
4941                                         union {
4942                                                 __be32 word;
4943                                                 char buf[4];
4944                                         } last;
4945                                         int i;
4946
4947                                         last.word = data[size >> 2];
4948                                         for (i = resid; i < 4; i++)
4949                                                 last.buf[i] = 0;
4950                                         ret = t4_memory_rw(adapter, 0, mtype,
4951                                                            maddr + size,
4952                                                            4, &last.word,
4953                                                            T4_MEMORY_WRITE);
4954                                 }
4955                                 spin_unlock(&adapter->win0_lock);
4956                         }
4957                 }
4958
4959                 release_firmware(cf);
4960                 if (ret)
4961                         goto bye;
4962         }
4963
4964         /*
4965          * Issue a Capability Configuration command to the firmware to get it
4966          * to parse the Configuration File.  We don't use t4_fw_config_file()
4967          * because we want the ability to modify various features after we've
4968          * processed the configuration file ...
4969          */
4970         memset(&caps_cmd, 0, sizeof(caps_cmd));
4971         caps_cmd.op_to_write =
4972                 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4973                       FW_CMD_REQUEST_F |
4974                       FW_CMD_READ_F);
4975         caps_cmd.cfvalid_to_len16 =
4976                 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4977                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4978                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
4979                       FW_LEN16(caps_cmd));
4980         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4981                          &caps_cmd);
4982
4983         /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4984          * Configuration File in FLASH), our last gasp effort is to use the
4985          * Firmware Configuration File which is embedded in the firmware.  A
4986          * very few early versions of the firmware didn't have one embedded
4987          * but we can ignore those.
4988          */
4989         if (ret == -ENOENT) {
4990                 memset(&caps_cmd, 0, sizeof(caps_cmd));
4991                 caps_cmd.op_to_write =
4992                         htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4993                                         FW_CMD_REQUEST_F |
4994                                         FW_CMD_READ_F);
4995                 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4996                 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4997                                 sizeof(caps_cmd), &caps_cmd);
4998                 config_name = "Firmware Default";
4999         }
5000
5001         config_issued = 1;
5002         if (ret < 0)
5003                 goto bye;
5004
5005         finiver = ntohl(caps_cmd.finiver);
5006         finicsum = ntohl(caps_cmd.finicsum);
5007         cfcsum = ntohl(caps_cmd.cfcsum);
5008         if (finicsum != cfcsum)
5009                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
5010                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
5011                          finicsum, cfcsum);
5012
5013         /*
5014          * And now tell the firmware to use the configuration we just loaded.
5015          */
5016         caps_cmd.op_to_write =
5017                 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5018                       FW_CMD_REQUEST_F |
5019                       FW_CMD_WRITE_F);
5020         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5021         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5022                          NULL);
5023         if (ret < 0)
5024                 goto bye;
5025
5026         /*
5027          * Tweak configuration based on system architecture, module
5028          * parameters, etc.
5029          */
5030         ret = adap_init0_tweaks(adapter);
5031         if (ret < 0)
5032                 goto bye;
5033
5034         /*
5035          * And finally tell the firmware to initialize itself using the
5036          * parameters from the Configuration File.
5037          */
5038         ret = t4_fw_initialize(adapter, adapter->mbox);
5039         if (ret < 0)
5040                 goto bye;
5041
5042         /* Emit Firmware Configuration File information and return
5043          * successfully.
5044          */
5045         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
5046                  "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5047                  config_name, finiver, cfcsum);
5048         return 0;
5049
5050         /*
5051          * Something bad happened.  Return the error ...  (If the "error"
5052          * is that there's no Configuration File on the adapter we don't
5053          * want to issue a warning since this is fairly common.)
5054          */
5055 bye:
5056         if (config_issued && ret != -ENOENT)
5057                 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5058                          config_name, -ret);
5059         return ret;
5060 }
5061
5062 static struct fw_info fw_info_array[] = {
5063         {
5064                 .chip = CHELSIO_T4,
5065                 .fs_name = FW4_CFNAME,
5066                 .fw_mod_name = FW4_FNAME,
5067                 .fw_hdr = {
5068                         .chip = FW_HDR_CHIP_T4,
5069                         .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5070                         .intfver_nic = FW_INTFVER(T4, NIC),
5071                         .intfver_vnic = FW_INTFVER(T4, VNIC),
5072                         .intfver_ri = FW_INTFVER(T4, RI),
5073                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5074                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
5075                 },
5076         }, {
5077                 .chip = CHELSIO_T5,
5078                 .fs_name = FW5_CFNAME,
5079                 .fw_mod_name = FW5_FNAME,
5080                 .fw_hdr = {
5081                         .chip = FW_HDR_CHIP_T5,
5082                         .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5083                         .intfver_nic = FW_INTFVER(T5, NIC),
5084                         .intfver_vnic = FW_INTFVER(T5, VNIC),
5085                         .intfver_ri = FW_INTFVER(T5, RI),
5086                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5087                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
5088                 },
5089         }
5090 };
5091
5092 static struct fw_info *find_fw_info(int chip)
5093 {
5094         int i;
5095
5096         for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5097                 if (fw_info_array[i].chip == chip)
5098                         return &fw_info_array[i];
5099         }
5100         return NULL;
5101 }
5102
5103 /*
5104  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5105  */
5106 static int adap_init0(struct adapter *adap)
5107 {
5108         int ret;
5109         u32 v, port_vec;
5110         enum dev_state state;
5111         u32 params[7], val[7];
5112         struct fw_caps_config_cmd caps_cmd;
5113         int reset = 1;
5114
5115         /* Grab Firmware Device Log parameters as early as possible so we have
5116          * access to it for debugging, etc.
5117          */
5118         ret = t4_init_devlog_params(adap);
5119         if (ret < 0)
5120                 return ret;
5121
5122         /* Contact FW, advertising Master capability */
5123         ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
5124         if (ret < 0) {
5125                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5126                         ret);
5127                 return ret;
5128         }
5129         if (ret == adap->mbox)
5130                 adap->flags |= MASTER_PF;
5131
5132         /*
5133          * If we're the Master PF Driver and the device is uninitialized,
5134          * then let's consider upgrading the firmware ...  (We always want
5135          * to check the firmware version number in order to A. get it for
5136          * later reporting and B. to warn if the currently loaded firmware
5137          * is excessively mismatched relative to the driver.)
5138          */
5139         t4_get_fw_version(adap, &adap->params.fw_vers);
5140         t4_get_tp_version(adap, &adap->params.tp_vers);
5141         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5142                 struct fw_info *fw_info;
5143                 struct fw_hdr *card_fw;
5144                 const struct firmware *fw;
5145                 const u8 *fw_data = NULL;
5146                 unsigned int fw_size = 0;
5147
5148                 /* This is the firmware whose headers the driver was compiled
5149                  * against
5150                  */
5151                 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5152                 if (fw_info == NULL) {
5153                         dev_err(adap->pdev_dev,
5154                                 "unable to get firmware info for chip %d.\n",
5155                                 CHELSIO_CHIP_VERSION(adap->params.chip));
5156                         return -EINVAL;
5157                 }
5158
5159                 /* allocate memory to read the header of the firmware on the
5160                  * card
5161                  */
5162                 card_fw = t4_alloc_mem(sizeof(*card_fw));
5163
5164                 /* Get FW from from /lib/firmware/ */
5165                 ret = request_firmware(&fw, fw_info->fw_mod_name,
5166                                        adap->pdev_dev);
5167                 if (ret < 0) {
5168                         dev_err(adap->pdev_dev,
5169                                 "unable to load firmware image %s, error %d\n",
5170                                 fw_info->fw_mod_name, ret);
5171                 } else {
5172                         fw_data = fw->data;
5173                         fw_size = fw->size;
5174                 }
5175
5176                 /* upgrade FW logic */
5177                 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5178                                  state, &reset);
5179
5180                 /* Cleaning up */
5181                 release_firmware(fw);
5182                 t4_free_mem(card_fw);
5183
5184                 if (ret < 0)
5185                         goto bye;
5186         }
5187
5188         /*
5189          * Grab VPD parameters.  This should be done after we establish a
5190          * connection to the firmware since some of the VPD parameters
5191          * (notably the Core Clock frequency) are retrieved via requests to
5192          * the firmware.  On the other hand, we need these fairly early on
5193          * so we do this right after getting ahold of the firmware.
5194          */
5195         ret = get_vpd_params(adap, &adap->params.vpd);
5196         if (ret < 0)
5197                 goto bye;
5198
5199         /*
5200          * Find out what ports are available to us.  Note that we need to do
5201          * this before calling adap_init0_no_config() since it needs nports
5202          * and portvec ...
5203          */
5204         v =
5205             FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5206             FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
5207         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5208         if (ret < 0)
5209                 goto bye;
5210
5211         adap->params.nports = hweight32(port_vec);
5212         adap->params.portvec = port_vec;
5213
5214         /* If the firmware is initialized already, emit a simply note to that
5215          * effect. Otherwise, it's time to try initializing the adapter.
5216          */
5217         if (state == DEV_STATE_INIT) {
5218                 dev_info(adap->pdev_dev, "Coming up as %s: "\
5219                          "Adapter already initialized\n",
5220                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5221         } else {
5222                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5223                          "Initializing adapter\n");
5224
5225                 /* Find out whether we're dealing with a version of the
5226                  * firmware which has configuration file support.
5227                  */
5228                 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5229                              FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5230                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5231                                       params, val);
5232
5233                 /* If the firmware doesn't support Configuration Files,
5234                  * return an error.
5235                  */
5236                 if (ret < 0) {
5237                         dev_err(adap->pdev_dev, "firmware doesn't support "
5238                                 "Firmware Configuration Files\n");
5239                         goto bye;
5240                 }
5241
5242                 /* The firmware provides us with a memory buffer where we can
5243                  * load a Configuration File from the host if we want to
5244                  * override the Configuration File in flash.
5245                  */
5246                 ret = adap_init0_config(adap, reset);
5247                 if (ret == -ENOENT) {
5248                         dev_err(adap->pdev_dev, "no Configuration File "
5249                                 "present on adapter.\n");
5250                         goto bye;
5251                 }
5252                 if (ret < 0) {
5253                         dev_err(adap->pdev_dev, "could not initialize "
5254                                 "adapter, error %d\n", -ret);
5255                         goto bye;
5256                 }
5257         }
5258
5259         /* Give the SGE code a chance to pull in anything that it needs ...
5260          * Note that this must be called after we retrieve our VPD parameters
5261          * in order to know how to convert core ticks to seconds, etc.
5262          */
5263         ret = t4_sge_init(adap);
5264         if (ret < 0)
5265                 goto bye;
5266
5267         if (is_bypass_device(adap->pdev->device))
5268                 adap->params.bypass = 1;
5269
5270         /*
5271          * Grab some of our basic fundamental operating parameters.
5272          */
5273 #define FW_PARAM_DEV(param) \
5274         (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5275         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
5276
5277 #define FW_PARAM_PFVF(param) \
5278         FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5279         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)|  \
5280         FW_PARAMS_PARAM_Y_V(0) | \
5281         FW_PARAMS_PARAM_Z_V(0)
5282
5283         params[0] = FW_PARAM_PFVF(EQ_START);
5284         params[1] = FW_PARAM_PFVF(L2T_START);
5285         params[2] = FW_PARAM_PFVF(L2T_END);
5286         params[3] = FW_PARAM_PFVF(FILTER_START);
5287         params[4] = FW_PARAM_PFVF(FILTER_END);
5288         params[5] = FW_PARAM_PFVF(IQFLINT_START);
5289         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5290         if (ret < 0)
5291                 goto bye;
5292         adap->sge.egr_start = val[0];
5293         adap->l2t_start = val[1];
5294         adap->l2t_end = val[2];
5295         adap->tids.ftid_base = val[3];
5296         adap->tids.nftids = val[4] - val[3] + 1;
5297         adap->sge.ingr_start = val[5];
5298
5299         /* qids (ingress/egress) returned from firmware can be anywhere
5300          * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END.
5301          * Hence driver needs to allocate memory for this range to
5302          * store the queue info. Get the highest IQFLINT/EQ index returned
5303          * in FW_EQ_*_CMD.alloc command.
5304          */
5305         params[0] = FW_PARAM_PFVF(EQ_END);
5306         params[1] = FW_PARAM_PFVF(IQFLINT_END);
5307         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5308         if (ret < 0)
5309                 goto bye;
5310         adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1;
5311         adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1;
5312
5313         adap->sge.egr_map = kcalloc(adap->sge.egr_sz,
5314                                     sizeof(*adap->sge.egr_map), GFP_KERNEL);
5315         if (!adap->sge.egr_map) {
5316                 ret = -ENOMEM;
5317                 goto bye;
5318         }
5319
5320         adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz,
5321                                      sizeof(*adap->sge.ingr_map), GFP_KERNEL);
5322         if (!adap->sge.ingr_map) {
5323                 ret = -ENOMEM;
5324                 goto bye;
5325         }
5326
5327         /* Allocate the memory for the vaious egress queue bitmaps
5328          * ie starving_fl and txq_maperr.
5329          */
5330         adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
5331                                         sizeof(long), GFP_KERNEL);
5332         if (!adap->sge.starving_fl) {
5333                 ret = -ENOMEM;
5334                 goto bye;
5335         }
5336
5337         adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
5338                                        sizeof(long), GFP_KERNEL);
5339         if (!adap->sge.txq_maperr) {
5340                 ret = -ENOMEM;
5341                 goto bye;
5342         }
5343
5344         params[0] = FW_PARAM_PFVF(CLIP_START);
5345         params[1] = FW_PARAM_PFVF(CLIP_END);
5346         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5347         if (ret < 0)
5348                 goto bye;
5349         adap->clipt_start = val[0];
5350         adap->clipt_end = val[1];
5351
5352         /* query params related to active filter region */
5353         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5354         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5355         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5356         /* If Active filter size is set we enable establishing
5357          * offload connection through firmware work request
5358          */
5359         if ((val[0] != val[1]) && (ret >= 0)) {
5360                 adap->flags |= FW_OFLD_CONN;
5361                 adap->tids.aftid_base = val[0];
5362                 adap->tids.aftid_end = val[1];
5363         }
5364
5365         /* If we're running on newer firmware, let it know that we're
5366          * prepared to deal with encapsulated CPL messages.  Older
5367          * firmware won't understand this and we'll just get
5368          * unencapsulated messages ...
5369          */
5370         params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5371         val[0] = 1;
5372         (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5373
5374         /*
5375          * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5376          * capability.  Earlier versions of the firmware didn't have the
5377          * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5378          * permission to use ULPTX MEMWRITE DSGL.
5379          */
5380         if (is_t4(adap->params.chip)) {
5381                 adap->params.ulptx_memwrite_dsgl = false;
5382         } else {
5383                 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5384                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5385                                       1, params, val);
5386                 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5387         }
5388
5389         /*
5390          * Get device capabilities so we can determine what resources we need
5391          * to manage.
5392          */
5393         memset(&caps_cmd, 0, sizeof(caps_cmd));
5394         caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5395                                      FW_CMD_REQUEST_F | FW_CMD_READ_F);
5396         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5397         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5398                          &caps_cmd);
5399         if (ret < 0)
5400                 goto bye;
5401
5402         if (caps_cmd.ofldcaps) {
5403                 /* query offload-related parameters */
5404                 params[0] = FW_PARAM_DEV(NTID);
5405                 params[1] = FW_PARAM_PFVF(SERVER_START);
5406                 params[2] = FW_PARAM_PFVF(SERVER_END);
5407                 params[3] = FW_PARAM_PFVF(TDDP_START);
5408                 params[4] = FW_PARAM_PFVF(TDDP_END);
5409                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5410                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5411                                       params, val);
5412                 if (ret < 0)
5413                         goto bye;
5414                 adap->tids.ntids = val[0];
5415                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5416                 adap->tids.stid_base = val[1];
5417                 adap->tids.nstids = val[2] - val[1] + 1;
5418                 /*
5419                  * Setup server filter region. Divide the availble filter
5420                  * region into two parts. Regular filters get 1/3rd and server
5421                  * filters get 2/3rd part. This is only enabled if workarond
5422                  * path is enabled.
5423                  * 1. For regular filters.
5424                  * 2. Server filter: This are special filters which are used
5425                  * to redirect SYN packets to offload queue.
5426                  */
5427                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5428                         adap->tids.sftid_base = adap->tids.ftid_base +
5429                                         DIV_ROUND_UP(adap->tids.nftids, 3);
5430                         adap->tids.nsftids = adap->tids.nftids -
5431                                          DIV_ROUND_UP(adap->tids.nftids, 3);
5432                         adap->tids.nftids = adap->tids.sftid_base -
5433                                                 adap->tids.ftid_base;
5434                 }
5435                 adap->vres.ddp.start = val[3];
5436                 adap->vres.ddp.size = val[4] - val[3] + 1;
5437                 adap->params.ofldq_wr_cred = val[5];
5438
5439                 adap->params.offload = 1;
5440         }
5441         if (caps_cmd.rdmacaps) {
5442                 params[0] = FW_PARAM_PFVF(STAG_START);
5443                 params[1] = FW_PARAM_PFVF(STAG_END);
5444                 params[2] = FW_PARAM_PFVF(RQ_START);
5445                 params[3] = FW_PARAM_PFVF(RQ_END);
5446                 params[4] = FW_PARAM_PFVF(PBL_START);
5447                 params[5] = FW_PARAM_PFVF(PBL_END);
5448                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5449                                       params, val);
5450                 if (ret < 0)
5451                         goto bye;
5452                 adap->vres.stag.start = val[0];
5453                 adap->vres.stag.size = val[1] - val[0] + 1;
5454                 adap->vres.rq.start = val[2];
5455                 adap->vres.rq.size = val[3] - val[2] + 1;
5456                 adap->vres.pbl.start = val[4];
5457                 adap->vres.pbl.size = val[5] - val[4] + 1;
5458
5459                 params[0] = FW_PARAM_PFVF(SQRQ_START);
5460                 params[1] = FW_PARAM_PFVF(SQRQ_END);
5461                 params[2] = FW_PARAM_PFVF(CQ_START);
5462                 params[3] = FW_PARAM_PFVF(CQ_END);
5463                 params[4] = FW_PARAM_PFVF(OCQ_START);
5464                 params[5] = FW_PARAM_PFVF(OCQ_END);
5465                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5466                                       val);
5467                 if (ret < 0)
5468                         goto bye;
5469                 adap->vres.qp.start = val[0];
5470                 adap->vres.qp.size = val[1] - val[0] + 1;
5471                 adap->vres.cq.start = val[2];
5472                 adap->vres.cq.size = val[3] - val[2] + 1;
5473                 adap->vres.ocq.start = val[4];
5474                 adap->vres.ocq.size = val[5] - val[4] + 1;
5475
5476                 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5477                 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5478                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5479                                       val);
5480                 if (ret < 0) {
5481                         adap->params.max_ordird_qp = 8;
5482                         adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5483                         ret = 0;
5484                 } else {
5485                         adap->params.max_ordird_qp = val[0];
5486                         adap->params.max_ird_adapter = val[1];
5487                 }
5488                 dev_info(adap->pdev_dev,
5489                          "max_ordird_qp %d max_ird_adapter %d\n",
5490                          adap->params.max_ordird_qp,
5491                          adap->params.max_ird_adapter);
5492         }
5493         if (caps_cmd.iscsicaps) {
5494                 params[0] = FW_PARAM_PFVF(ISCSI_START);
5495                 params[1] = FW_PARAM_PFVF(ISCSI_END);
5496                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5497                                       params, val);
5498                 if (ret < 0)
5499                         goto bye;
5500                 adap->vres.iscsi.start = val[0];
5501                 adap->vres.iscsi.size = val[1] - val[0] + 1;
5502         }
5503 #undef FW_PARAM_PFVF
5504 #undef FW_PARAM_DEV
5505
5506         /* The MTU/MSS Table is initialized by now, so load their values.  If
5507          * we're initializing the adapter, then we'll make any modifications
5508          * we want to the MTU/MSS Table and also initialize the congestion
5509          * parameters.
5510          */
5511         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5512         if (state != DEV_STATE_INIT) {
5513                 int i;
5514
5515                 /* The default MTU Table contains values 1492 and 1500.
5516                  * However, for TCP, it's better to have two values which are
5517                  * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5518                  * This allows us to have a TCP Data Payload which is a
5519                  * multiple of 8 regardless of what combination of TCP Options
5520                  * are in use (always a multiple of 4 bytes) which is
5521                  * important for performance reasons.  For instance, if no
5522                  * options are in use, then we have a 20-byte IP header and a
5523                  * 20-byte TCP header.  In this case, a 1500-byte MSS would
5524                  * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5525                  * which is not a multiple of 8.  So using an MSS of 1488 in
5526                  * this case results in a TCP Data Payload of 1448 bytes which
5527                  * is a multiple of 8.  On the other hand, if 12-byte TCP Time
5528                  * Stamps have been negotiated, then an MTU of 1500 bytes
5529                  * results in a TCP Data Payload of 1448 bytes which, as
5530                  * above, is a multiple of 8 bytes ...
5531                  */
5532                 for (i = 0; i < NMTUS; i++)
5533                         if (adap->params.mtus[i] == 1492) {
5534                                 adap->params.mtus[i] = 1488;
5535                                 break;
5536                         }
5537
5538                 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5539                              adap->params.b_wnd);
5540         }
5541         t4_init_sge_params(adap);
5542         t4_init_tp_params(adap);
5543         adap->flags |= FW_OK;
5544         return 0;
5545
5546         /*
5547          * Something bad happened.  If a command timed out or failed with EIO
5548          * FW does not operate within its spec or something catastrophic
5549          * happened to HW/FW, stop issuing commands.
5550          */
5551 bye:
5552         kfree(adap->sge.egr_map);
5553         kfree(adap->sge.ingr_map);
5554         kfree(adap->sge.starving_fl);
5555         kfree(adap->sge.txq_maperr);
5556         if (ret != -ETIMEDOUT && ret != -EIO)
5557                 t4_fw_bye(adap, adap->mbox);
5558         return ret;
5559 }
5560
5561 /* EEH callbacks */
5562
5563 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5564                                          pci_channel_state_t state)
5565 {
5566         int i;
5567         struct adapter *adap = pci_get_drvdata(pdev);
5568
5569         if (!adap)
5570                 goto out;
5571
5572         rtnl_lock();
5573         adap->flags &= ~FW_OK;
5574         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5575         spin_lock(&adap->stats_lock);
5576         for_each_port(adap, i) {
5577                 struct net_device *dev = adap->port[i];
5578
5579                 netif_device_detach(dev);
5580                 netif_carrier_off(dev);
5581         }
5582         spin_unlock(&adap->stats_lock);
5583         disable_interrupts(adap);
5584         if (adap->flags & FULL_INIT_DONE)
5585                 cxgb_down(adap);
5586         rtnl_unlock();
5587         if ((adap->flags & DEV_ENABLED)) {
5588                 pci_disable_device(pdev);
5589                 adap->flags &= ~DEV_ENABLED;
5590         }
5591 out:    return state == pci_channel_io_perm_failure ?
5592                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5593 }
5594
5595 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5596 {
5597         int i, ret;
5598         struct fw_caps_config_cmd c;
5599         struct adapter *adap = pci_get_drvdata(pdev);
5600
5601         if (!adap) {
5602                 pci_restore_state(pdev);
5603                 pci_save_state(pdev);
5604                 return PCI_ERS_RESULT_RECOVERED;
5605         }
5606
5607         if (!(adap->flags & DEV_ENABLED)) {
5608                 if (pci_enable_device(pdev)) {
5609                         dev_err(&pdev->dev, "Cannot reenable PCI "
5610                                             "device after reset\n");
5611                         return PCI_ERS_RESULT_DISCONNECT;
5612                 }
5613                 adap->flags |= DEV_ENABLED;
5614         }
5615
5616         pci_set_master(pdev);
5617         pci_restore_state(pdev);
5618         pci_save_state(pdev);
5619         pci_cleanup_aer_uncorrect_error_status(pdev);
5620
5621         if (t4_wait_dev_ready(adap->regs) < 0)
5622                 return PCI_ERS_RESULT_DISCONNECT;
5623         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5624                 return PCI_ERS_RESULT_DISCONNECT;
5625         adap->flags |= FW_OK;
5626         if (adap_init1(adap, &c))
5627                 return PCI_ERS_RESULT_DISCONNECT;
5628
5629         for_each_port(adap, i) {
5630                 struct port_info *p = adap2pinfo(adap, i);
5631
5632                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5633                                   NULL, NULL);
5634                 if (ret < 0)
5635                         return PCI_ERS_RESULT_DISCONNECT;
5636                 p->viid = ret;
5637                 p->xact_addr_filt = -1;
5638         }
5639
5640         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5641                      adap->params.b_wnd);
5642         setup_memwin(adap);
5643         if (cxgb_up(adap))
5644                 return PCI_ERS_RESULT_DISCONNECT;
5645         return PCI_ERS_RESULT_RECOVERED;
5646 }
5647
5648 static void eeh_resume(struct pci_dev *pdev)
5649 {
5650         int i;
5651         struct adapter *adap = pci_get_drvdata(pdev);
5652
5653         if (!adap)
5654                 return;
5655
5656         rtnl_lock();
5657         for_each_port(adap, i) {
5658                 struct net_device *dev = adap->port[i];
5659
5660                 if (netif_running(dev)) {
5661                         link_start(dev);
5662                         cxgb_set_rxmode(dev);
5663                 }
5664                 netif_device_attach(dev);
5665         }
5666         rtnl_unlock();
5667 }
5668
5669 static const struct pci_error_handlers cxgb4_eeh = {
5670         .error_detected = eeh_err_detected,
5671         .slot_reset     = eeh_slot_reset,
5672         .resume         = eeh_resume,
5673 };
5674
5675 static inline bool is_x_10g_port(const struct link_config *lc)
5676 {
5677         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5678                (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
5679 }
5680
5681 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5682                              unsigned int us, unsigned int cnt,
5683                              unsigned int size, unsigned int iqe_size)
5684 {
5685         q->adap = adap;
5686         set_rspq_intr_params(q, us, cnt);
5687         q->iqe_len = iqe_size;
5688         q->size = size;
5689 }
5690
5691 /*
5692  * Perform default configuration of DMA queues depending on the number and type
5693  * of ports we found and the number of available CPUs.  Most settings can be
5694  * modified by the admin prior to actual use.
5695  */
5696 static void cfg_queues(struct adapter *adap)
5697 {
5698         struct sge *s = &adap->sge;
5699         int i, n10g = 0, qidx = 0;
5700 #ifndef CONFIG_CHELSIO_T4_DCB
5701         int q10g = 0;
5702 #endif
5703         int ciq_size;
5704
5705         for_each_port(adap, i)
5706                 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
5707 #ifdef CONFIG_CHELSIO_T4_DCB
5708         /* For Data Center Bridging support we need to be able to support up
5709          * to 8 Traffic Priorities; each of which will be assigned to its
5710          * own TX Queue in order to prevent Head-Of-Line Blocking.
5711          */
5712         if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5713                 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5714                         MAX_ETH_QSETS, adap->params.nports * 8);
5715                 BUG_ON(1);
5716         }
5717
5718         for_each_port(adap, i) {
5719                 struct port_info *pi = adap2pinfo(adap, i);
5720
5721                 pi->first_qset = qidx;
5722                 pi->nqsets = 8;
5723                 qidx += pi->nqsets;
5724         }
5725 #else /* !CONFIG_CHELSIO_T4_DCB */
5726         /*
5727          * We default to 1 queue per non-10G port and up to # of cores queues
5728          * per 10G port.
5729          */
5730         if (n10g)
5731                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5732         if (q10g > netif_get_num_default_rss_queues())
5733                 q10g = netif_get_num_default_rss_queues();
5734
5735         for_each_port(adap, i) {
5736                 struct port_info *pi = adap2pinfo(adap, i);
5737
5738                 pi->first_qset = qidx;
5739                 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
5740                 qidx += pi->nqsets;
5741         }
5742 #endif /* !CONFIG_CHELSIO_T4_DCB */
5743
5744         s->ethqsets = qidx;
5745         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
5746
5747         if (is_offload(adap)) {
5748                 /*
5749                  * For offload we use 1 queue/channel if all ports are up to 1G,
5750                  * otherwise we divide all available queues amongst the channels
5751                  * capped by the number of available cores.
5752                  */
5753                 if (n10g) {
5754                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5755                                   num_online_cpus());
5756                         s->ofldqsets = roundup(i, adap->params.nports);
5757                 } else
5758                         s->ofldqsets = adap->params.nports;
5759                 /* For RDMA one Rx queue per channel suffices */
5760                 s->rdmaqs = adap->params.nports;
5761                 s->rdmaciqs = adap->params.nports;
5762         }
5763
5764         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5765                 struct sge_eth_rxq *r = &s->ethrxq[i];
5766
5767                 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
5768                 r->fl.size = 72;
5769         }
5770
5771         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5772                 s->ethtxq[i].q.size = 1024;
5773
5774         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5775                 s->ctrlq[i].q.size = 512;
5776
5777         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5778                 s->ofldtxq[i].q.size = 1024;
5779
5780         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5781                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5782
5783                 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
5784                 r->rspq.uld = CXGB4_ULD_ISCSI;
5785                 r->fl.size = 72;
5786         }
5787
5788         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5789                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5790
5791                 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
5792                 r->rspq.uld = CXGB4_ULD_RDMA;
5793                 r->fl.size = 72;
5794         }
5795
5796         ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5797         if (ciq_size > SGE_MAX_IQ_SIZE) {
5798                 CH_WARN(adap, "CIQ size too small for available IQs\n");
5799                 ciq_size = SGE_MAX_IQ_SIZE;
5800         }
5801
5802         for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5803                 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5804
5805                 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
5806                 r->rspq.uld = CXGB4_ULD_RDMA;
5807         }
5808
5809         init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5810         init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
5811 }
5812
5813 /*
5814  * Reduce the number of Ethernet queues across all ports to at most n.
5815  * n provides at least one queue per port.
5816  */
5817 static void reduce_ethqs(struct adapter *adap, int n)
5818 {
5819         int i;
5820         struct port_info *pi;
5821
5822         while (n < adap->sge.ethqsets)
5823                 for_each_port(adap, i) {
5824                         pi = adap2pinfo(adap, i);
5825                         if (pi->nqsets > 1) {
5826                                 pi->nqsets--;
5827                                 adap->sge.ethqsets--;
5828                                 if (adap->sge.ethqsets <= n)
5829                                         break;
5830                         }
5831                 }
5832
5833         n = 0;
5834         for_each_port(adap, i) {
5835                 pi = adap2pinfo(adap, i);
5836                 pi->first_qset = n;
5837                 n += pi->nqsets;
5838         }
5839 }
5840
5841 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5842 #define EXTRA_VECS 2
5843
5844 static int enable_msix(struct adapter *adap)
5845 {
5846         int ofld_need = 0;
5847         int i, want, need;
5848         struct sge *s = &adap->sge;
5849         unsigned int nchan = adap->params.nports;
5850         struct msix_entry entries[MAX_INGQ + 1];
5851
5852         for (i = 0; i < ARRAY_SIZE(entries); ++i)
5853                 entries[i].entry = i;
5854
5855         want = s->max_ethqsets + EXTRA_VECS;
5856         if (is_offload(adap)) {
5857                 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
5858                 /* need nchan for each possible ULD */
5859                 ofld_need = 3 * nchan;
5860         }
5861 #ifdef CONFIG_CHELSIO_T4_DCB
5862         /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5863          * each port.
5864          */
5865         need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5866 #else
5867         need = adap->params.nports + EXTRA_VECS + ofld_need;
5868 #endif
5869         want = pci_enable_msix_range(adap->pdev, entries, need, want);
5870         if (want < 0)
5871                 return want;
5872
5873         /*
5874          * Distribute available vectors to the various queue groups.
5875          * Every group gets its minimum requirement and NIC gets top
5876          * priority for leftovers.
5877          */
5878         i = want - EXTRA_VECS - ofld_need;
5879         if (i < s->max_ethqsets) {
5880                 s->max_ethqsets = i;
5881                 if (i < s->ethqsets)
5882                         reduce_ethqs(adap, i);
5883         }
5884         if (is_offload(adap)) {
5885                 i = want - EXTRA_VECS - s->max_ethqsets;
5886                 i -= ofld_need - nchan;
5887                 s->ofldqsets = (i / nchan) * nchan;  /* round down */
5888         }
5889         for (i = 0; i < want; ++i)
5890                 adap->msix_info[i].vec = entries[i].vector;
5891
5892         return 0;
5893 }
5894
5895 #undef EXTRA_VECS
5896
5897 static int init_rss(struct adapter *adap)
5898 {
5899         unsigned int i, j;
5900
5901         for_each_port(adap, i) {
5902                 struct port_info *pi = adap2pinfo(adap, i);
5903
5904                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5905                 if (!pi->rss)
5906                         return -ENOMEM;
5907                 for (j = 0; j < pi->rss_size; j++)
5908                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5909         }
5910         return 0;
5911 }
5912
5913 static void print_port_info(const struct net_device *dev)
5914 {
5915         char buf[80];
5916         char *bufp = buf;
5917         const char *spd = "";
5918         const struct port_info *pi = netdev_priv(dev);
5919         const struct adapter *adap = pi->adapter;
5920
5921         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5922                 spd = " 2.5 GT/s";
5923         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5924                 spd = " 5 GT/s";
5925         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5926                 spd = " 8 GT/s";
5927
5928         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5929                 bufp += sprintf(bufp, "100/");
5930         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5931                 bufp += sprintf(bufp, "1000/");
5932         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5933                 bufp += sprintf(bufp, "10G/");
5934         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5935                 bufp += sprintf(bufp, "40G/");
5936         if (bufp != buf)
5937                 --bufp;
5938         sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5939
5940         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5941                     adap->params.vpd.id,
5942                     CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
5943                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5944                     (adap->flags & USING_MSIX) ? " MSI-X" :
5945                     (adap->flags & USING_MSI) ? " MSI" : "");
5946         netdev_info(dev, "S/N: %s, P/N: %s\n",
5947                     adap->params.vpd.sn, adap->params.vpd.pn);
5948 }
5949
5950 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5951 {
5952         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5953 }
5954
5955 /*
5956  * Free the following resources:
5957  * - memory used for tables
5958  * - MSI/MSI-X
5959  * - net devices
5960  * - resources FW is holding for us
5961  */
5962 static void free_some_resources(struct adapter *adapter)
5963 {
5964         unsigned int i;
5965
5966         t4_free_mem(adapter->l2t);
5967         t4_free_mem(adapter->tids.tid_tab);
5968         kfree(adapter->sge.egr_map);
5969         kfree(adapter->sge.ingr_map);
5970         kfree(adapter->sge.starving_fl);
5971         kfree(adapter->sge.txq_maperr);
5972         disable_msi(adapter);
5973
5974         for_each_port(adapter, i)
5975                 if (adapter->port[i]) {
5976                         kfree(adap2pinfo(adapter, i)->rss);
5977                         free_netdev(adapter->port[i]);
5978                 }
5979         if (adapter->flags & FW_OK)
5980                 t4_fw_bye(adapter, adapter->fn);
5981 }
5982
5983 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5984 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5985                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5986 #define SEGMENT_SIZE 128
5987
5988 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5989 {
5990         int func, i, err, s_qpp, qpp, num_seg;
5991         struct port_info *pi;
5992         bool highdma = false;
5993         struct adapter *adapter = NULL;
5994         void __iomem *regs;
5995
5996         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5997
5998         err = pci_request_regions(pdev, KBUILD_MODNAME);
5999         if (err) {
6000                 /* Just info, some other driver may have claimed the device. */
6001                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
6002                 return err;
6003         }
6004
6005         err = pci_enable_device(pdev);
6006         if (err) {
6007                 dev_err(&pdev->dev, "cannot enable PCI device\n");
6008                 goto out_release_regions;
6009         }
6010
6011         regs = pci_ioremap_bar(pdev, 0);
6012         if (!regs) {
6013                 dev_err(&pdev->dev, "cannot map device registers\n");
6014                 err = -ENOMEM;
6015                 goto out_disable_device;
6016         }
6017
6018         err = t4_wait_dev_ready(regs);
6019         if (err < 0)
6020                 goto out_unmap_bar0;
6021
6022         /* We control everything through one PF */
6023         func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
6024         if (func != ent->driver_data) {
6025                 iounmap(regs);
6026                 pci_disable_device(pdev);
6027                 pci_save_state(pdev);        /* to restore SR-IOV later */
6028                 goto sriov;
6029         }
6030
6031         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
6032                 highdma = true;
6033                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
6034                 if (err) {
6035                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6036                                 "coherent allocations\n");
6037                         goto out_unmap_bar0;
6038                 }
6039         } else {
6040                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6041                 if (err) {
6042                         dev_err(&pdev->dev, "no usable DMA configuration\n");
6043                         goto out_unmap_bar0;
6044                 }
6045         }
6046
6047         pci_enable_pcie_error_reporting(pdev);
6048         enable_pcie_relaxed_ordering(pdev);
6049         pci_set_master(pdev);
6050         pci_save_state(pdev);
6051
6052         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6053         if (!adapter) {
6054                 err = -ENOMEM;
6055                 goto out_unmap_bar0;
6056         }
6057
6058         adapter->workq = create_singlethread_workqueue("cxgb4");
6059         if (!adapter->workq) {
6060                 err = -ENOMEM;
6061                 goto out_free_adapter;
6062         }
6063
6064         /* PCI device has been enabled */
6065         adapter->flags |= DEV_ENABLED;
6066
6067         adapter->regs = regs;
6068         adapter->pdev = pdev;
6069         adapter->pdev_dev = &pdev->dev;
6070         adapter->mbox = func;
6071         adapter->fn = func;
6072         adapter->msg_enable = dflt_msg_enable;
6073         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6074
6075         spin_lock_init(&adapter->stats_lock);
6076         spin_lock_init(&adapter->tid_release_lock);
6077         spin_lock_init(&adapter->win0_lock);
6078
6079         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6080         INIT_WORK(&adapter->db_full_task, process_db_full);
6081         INIT_WORK(&adapter->db_drop_task, process_db_drop);
6082
6083         err = t4_prep_adapter(adapter);
6084         if (err)
6085                 goto out_free_adapter;
6086
6087
6088         if (!is_t4(adapter->params.chip)) {
6089                 s_qpp = (QUEUESPERPAGEPF0_S +
6090                         (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6091                         adapter->fn);
6092                 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6093                       SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
6094                 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6095
6096                 /* Each segment size is 128B. Write coalescing is enabled only
6097                  * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6098                  * queue is less no of segments that can be accommodated in
6099                  * a page size.
6100                  */
6101                 if (qpp > num_seg) {
6102                         dev_err(&pdev->dev,
6103                                 "Incorrect number of egress queues per page\n");
6104                         err = -EINVAL;
6105                         goto out_free_adapter;
6106                 }
6107                 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6108                 pci_resource_len(pdev, 2));
6109                 if (!adapter->bar2) {
6110                         dev_err(&pdev->dev, "cannot map device bar2 region\n");
6111                         err = -ENOMEM;
6112                         goto out_free_adapter;
6113                 }
6114         }
6115
6116         setup_memwin(adapter);
6117         err = adap_init0(adapter);
6118         setup_memwin_rdma(adapter);
6119         if (err)
6120                 goto out_unmap_bar;
6121
6122         for_each_port(adapter, i) {
6123                 struct net_device *netdev;
6124
6125                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6126                                            MAX_ETH_QSETS);
6127                 if (!netdev) {
6128                         err = -ENOMEM;
6129                         goto out_free_dev;
6130                 }
6131
6132                 SET_NETDEV_DEV(netdev, &pdev->dev);
6133
6134                 adapter->port[i] = netdev;
6135                 pi = netdev_priv(netdev);
6136                 pi->adapter = adapter;
6137                 pi->xact_addr_filt = -1;
6138                 pi->port_id = i;
6139                 netdev->irq = pdev->irq;
6140
6141                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6142                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6143                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
6144                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6145                 if (highdma)
6146                         netdev->hw_features |= NETIF_F_HIGHDMA;
6147                 netdev->features |= netdev->hw_features;
6148                 netdev->vlan_features = netdev->features & VLAN_FEAT;
6149
6150                 netdev->priv_flags |= IFF_UNICAST_FLT;
6151
6152                 netdev->netdev_ops = &cxgb4_netdev_ops;
6153 #ifdef CONFIG_CHELSIO_T4_DCB
6154                 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6155                 cxgb4_dcb_state_init(netdev);
6156 #endif
6157                 netdev->ethtool_ops = &cxgb_ethtool_ops;
6158         }
6159
6160         pci_set_drvdata(pdev, adapter);
6161
6162         if (adapter->flags & FW_OK) {
6163                 err = t4_port_init(adapter, func, func, 0);
6164                 if (err)
6165                         goto out_free_dev;
6166         }
6167
6168         /*
6169          * Configure queues and allocate tables now, they can be needed as
6170          * soon as the first register_netdev completes.
6171          */
6172         cfg_queues(adapter);
6173
6174         adapter->l2t = t4_init_l2t();
6175         if (!adapter->l2t) {
6176                 /* We tolerate a lack of L2T, giving up some functionality */
6177                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6178                 adapter->params.offload = 0;
6179         }
6180
6181 #if IS_ENABLED(CONFIG_IPV6)
6182         adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6183                                           adapter->clipt_end);
6184         if (!adapter->clipt) {
6185                 /* We tolerate a lack of clip_table, giving up
6186                  * some functionality
6187                  */
6188                 dev_warn(&pdev->dev,
6189                          "could not allocate Clip table, continuing\n");
6190                 adapter->params.offload = 0;
6191         }
6192 #endif
6193         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6194                 dev_warn(&pdev->dev, "could not allocate TID table, "
6195                          "continuing\n");
6196                 adapter->params.offload = 0;
6197         }
6198
6199         /* See what interrupts we'll be using */
6200         if (msi > 1 && enable_msix(adapter) == 0)
6201                 adapter->flags |= USING_MSIX;
6202         else if (msi > 0 && pci_enable_msi(pdev) == 0)
6203                 adapter->flags |= USING_MSI;
6204
6205         err = init_rss(adapter);
6206         if (err)
6207                 goto out_free_dev;
6208
6209         /*
6210          * The card is now ready to go.  If any errors occur during device
6211          * registration we do not fail the whole card but rather proceed only
6212          * with the ports we manage to register successfully.  However we must
6213          * register at least one net device.
6214          */
6215         for_each_port(adapter, i) {
6216                 pi = adap2pinfo(adapter, i);
6217                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6218                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6219
6220                 err = register_netdev(adapter->port[i]);
6221                 if (err)
6222                         break;
6223                 adapter->chan_map[pi->tx_chan] = i;
6224                 print_port_info(adapter->port[i]);
6225         }
6226         if (i == 0) {
6227                 dev_err(&pdev->dev, "could not register any net devices\n");
6228                 goto out_free_dev;
6229         }
6230         if (err) {
6231                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6232                 err = 0;
6233         }
6234
6235         if (cxgb4_debugfs_root) {
6236                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6237                                                            cxgb4_debugfs_root);
6238                 setup_debugfs(adapter);
6239         }
6240
6241         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6242         pdev->needs_freset = 1;
6243
6244         if (is_offload(adapter))
6245                 attach_ulds(adapter);
6246
6247 sriov:
6248 #ifdef CONFIG_PCI_IOV
6249         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6250                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6251                         dev_info(&pdev->dev,
6252                                  "instantiated %u virtual functions\n",
6253                                  num_vf[func]);
6254 #endif
6255         return 0;
6256
6257  out_free_dev:
6258         free_some_resources(adapter);
6259  out_unmap_bar:
6260         if (!is_t4(adapter->params.chip))
6261                 iounmap(adapter->bar2);
6262  out_free_adapter:
6263         if (adapter->workq)
6264                 destroy_workqueue(adapter->workq);
6265
6266         kfree(adapter);
6267  out_unmap_bar0:
6268         iounmap(regs);
6269  out_disable_device:
6270         pci_disable_pcie_error_reporting(pdev);
6271         pci_disable_device(pdev);
6272  out_release_regions:
6273         pci_release_regions(pdev);
6274         return err;
6275 }
6276
6277 static void remove_one(struct pci_dev *pdev)
6278 {
6279         struct adapter *adapter = pci_get_drvdata(pdev);
6280
6281 #ifdef CONFIG_PCI_IOV
6282         pci_disable_sriov(pdev);
6283
6284 #endif
6285
6286         if (adapter) {
6287                 int i;
6288
6289                 /* Tear down per-adapter Work Queue first since it can contain
6290                  * references to our adapter data structure.
6291                  */
6292                 destroy_workqueue(adapter->workq);
6293
6294                 if (is_offload(adapter))
6295                         detach_ulds(adapter);
6296
6297                 disable_interrupts(adapter);
6298
6299                 for_each_port(adapter, i)
6300                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6301                                 unregister_netdev(adapter->port[i]);
6302
6303                 debugfs_remove_recursive(adapter->debugfs_root);
6304
6305                 /* If we allocated filters, free up state associated with any
6306                  * valid filters ...
6307                  */
6308                 if (adapter->tids.ftid_tab) {
6309                         struct filter_entry *f = &adapter->tids.ftid_tab[0];
6310                         for (i = 0; i < (adapter->tids.nftids +
6311                                         adapter->tids.nsftids); i++, f++)
6312                                 if (f->valid)
6313                                         clear_filter(adapter, f);
6314                 }
6315
6316                 if (adapter->flags & FULL_INIT_DONE)
6317                         cxgb_down(adapter);
6318
6319                 free_some_resources(adapter);
6320 #if IS_ENABLED(CONFIG_IPV6)
6321                 t4_cleanup_clip_tbl(adapter);
6322 #endif
6323                 iounmap(adapter->regs);
6324                 if (!is_t4(adapter->params.chip))
6325                         iounmap(adapter->bar2);
6326                 pci_disable_pcie_error_reporting(pdev);
6327                 if ((adapter->flags & DEV_ENABLED)) {
6328                         pci_disable_device(pdev);
6329                         adapter->flags &= ~DEV_ENABLED;
6330                 }
6331                 pci_release_regions(pdev);
6332                 synchronize_rcu();
6333                 kfree(adapter);
6334         } else
6335                 pci_release_regions(pdev);
6336 }
6337
6338 static struct pci_driver cxgb4_driver = {
6339         .name     = KBUILD_MODNAME,
6340         .id_table = cxgb4_pci_tbl,
6341         .probe    = init_one,
6342         .remove   = remove_one,
6343         .shutdown = remove_one,
6344         .err_handler = &cxgb4_eeh,
6345 };
6346
6347 static int __init cxgb4_init_module(void)
6348 {
6349         int ret;
6350
6351         /* Debugfs support is optional, just warn if this fails */
6352         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6353         if (!cxgb4_debugfs_root)
6354                 pr_warn("could not create debugfs entry, continuing\n");
6355
6356         ret = pci_register_driver(&cxgb4_driver);
6357         if (ret < 0)
6358                 debugfs_remove(cxgb4_debugfs_root);
6359
6360 #if IS_ENABLED(CONFIG_IPV6)
6361         if (!inet6addr_registered) {
6362                 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6363                 inet6addr_registered = true;
6364         }
6365 #endif
6366
6367         return ret;
6368 }
6369
6370 static void __exit cxgb4_cleanup_module(void)
6371 {
6372 #if IS_ENABLED(CONFIG_IPV6)
6373         if (inet6addr_registered) {
6374                 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6375                 inet6addr_registered = false;
6376         }
6377 #endif
6378         pci_unregister_driver(&cxgb4_driver);
6379         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
6380 }
6381
6382 module_init(cxgb4_init_module);
6383 module_exit(cxgb4_cleanup_module);