drivers/rapidio/devices/tsi721.c

   1 /*
   2  * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
   3  *
   4  * Copyright 2011 Integrated Device Technology, Inc.
   5  * Alexandre Bounine <alexandre.bounine@idt.com>
   6  * Chul Kim <chul.kim@idt.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along with
  19  * this program; if not, write to the Free Software Foundation, Inc., 59
  20  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  21  */
  22
  23 #include <linux/io.h>
  24 #include <linux/errno.h>
  25 #include <linux/init.h>
  26 #include <linux/ioport.h>
  27 #include <linux/kernel.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/rio.h>
  31 #include <linux/rio_drv.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/delay.h>
  36
  37 #include "tsi721.h"
  38
  39 #define DEBUG_PW        /* Inbound Port-Write debugging */
  40
  41 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
  42 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
  43
  44 /**
  45  * tsi721_lcread - read from local SREP config space
  46  * @mport: RapidIO master port info
  47  * @index: ID of RapdiIO interface
  48  * @offset: Offset into configuration space
  49  * @len: Length (in bytes) of the maintenance transaction
  50  * @data: Value to be read into
  51  *
  52  * Generates a local SREP space read. Returns %0 on
  53  * success or %-EINVAL on failure.
  54  */
  55 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
  56                          int len, u32 *data)
  57 {
  58         struct tsi721_device *priv = mport->priv;
  59
  60         if (len != sizeof(u32))
  61                 return -EINVAL; /* only 32-bit access is supported */
  62
  63         *data = ioread32(priv->regs + offset);
  64
  65         return 0;
  66 }
  67
  68 /**
  69  * tsi721_lcwrite - write into local SREP config space
  70  * @mport: RapidIO master port info
  71  * @index: ID of RapdiIO interface
  72  * @offset: Offset into configuration space
  73  * @len: Length (in bytes) of the maintenance transaction
  74  * @data: Value to be written
  75  *
  76  * Generates a local write into SREP configuration space. Returns %0 on
  77  * success or %-EINVAL on failure.
  78  */
  79 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
  80                           int len, u32 data)
  81 {
  82         struct tsi721_device *priv = mport->priv;
  83
  84         if (len != sizeof(u32))
  85                 return -EINVAL; /* only 32-bit access is supported */
  86
  87         iowrite32(data, priv->regs + offset);
  88
  89         return 0;
  90 }
  91
  92 /**
  93  * tsi721_maint_dma - Helper function to generate RapidIO maintenance
  94  *                    transactions using designated Tsi721 DMA channel.
  95  * @priv: pointer to tsi721 private data
  96  * @sys_size: RapdiIO transport system size
  97  * @destid: Destination ID of transaction
  98  * @hopcount: Number of hops to target device
  99  * @offset: Offset into configuration space
 100  * @len: Length (in bytes) of the maintenance transaction
 101  * @data: Location to be read from or write into
 102  * @do_wr: Operation flag (1 == MAINT_WR)
 103  *
 104  * Generates a RapidIO maintenance transaction (Read or Write).
 105  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 106  */
 107 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
 108                         u16 destid, u8 hopcount, u32 offset, int len,
 109                         u32 *data, int do_wr)
 110 {
 111         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
 112         struct tsi721_dma_desc *bd_ptr;
 113         u32 rd_count, swr_ptr, ch_stat;
 114         int i, err = 0;
 115         u32 op = do_wr ? MAINT_WR : MAINT_RD;
 116
 117         if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
 118                 return -EINVAL;
 119
 120         bd_ptr = priv->mdma.bd_base;
 121
 122         rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
 123
 124         /* Initialize DMA descriptor */
 125         bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
 126         bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
 127         bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
 128         bd_ptr[0].raddr_hi = 0;
 129         if (do_wr)
 130                 bd_ptr[0].data[0] = cpu_to_be32p(data);
 131         else
 132                 bd_ptr[0].data[0] = 0xffffffff;
 133
 134         mb();
 135
 136         /* Start DMA operation */
 137         iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
 138         ioread32(regs + TSI721_DMAC_DWRCNT);
 139         i = 0;
 140
 141         /* Wait until DMA transfer is finished */
 142         while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
 143                                                         & TSI721_DMAC_STS_RUN) {
 144                 udelay(1);
 145                 if (++i >= 5000000) {
 146                         dev_dbg(&priv->pdev->dev,
 147                                 "%s : DMA[%d] read timeout ch_status=%x\n",
 148                                 __func__, priv->mdma.ch_id, ch_stat);
 149                         if (!do_wr)
 150                                 *data = 0xffffffff;
 151                         err = -EIO;
 152                         goto err_out;
 153                 }
 154         }
 155
 156         if (ch_stat & TSI721_DMAC_STS_ABORT) {
 157                 /* If DMA operation aborted due to error,
 158                  * reinitialize DMA channel
 159                  */
 160                 dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",
 161                         __func__, ch_stat);
 162                 dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",
 163                         do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset);
 164                 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
 165                 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
 166                 udelay(10);
 167                 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
 168                 udelay(1);
 169                 if (!do_wr)
 170                         *data = 0xffffffff;
 171                 err = -EIO;
 172                 goto err_out;
 173         }
 174
 175         if (!do_wr)
 176                 *data = be32_to_cpu(bd_ptr[0].data[0]);
 177
 178         /*
 179          * Update descriptor status FIFO RD pointer.
 180          * NOTE: Skipping check and clear FIFO entries because we are waiting
 181          * for transfer to be completed.
 182          */
 183         swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
 184         iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
 185 err_out:
 186
 187         return err;
 188 }
 189
 190 /**
 191  * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
 192  *                    using Tsi721 BDMA engine.
 193  * @mport: RapidIO master port control structure
 194  * @index: ID of RapdiIO interface
 195  * @destid: Destination ID of transaction
 196  * @hopcount: Number of hops to target device
 197  * @offset: Offset into configuration space
 198  * @len: Length (in bytes) of the maintenance transaction
 199  * @val: Location to be read into
 200  *
 201  * Generates a RapidIO maintenance read transaction.
 202  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 203  */
 204 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
 205                         u8 hopcount, u32 offset, int len, u32 *data)
 206 {
 207         struct tsi721_device *priv = mport->priv;
 208
 209         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 210                                 offset, len, data, 0);
 211 }
 212
 213 /**
 214  * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
 215  *                     using Tsi721 BDMA engine
 216  * @mport: RapidIO master port control structure
 217  * @index: ID of RapdiIO interface
 218  * @destid: Destination ID of transaction
 219  * @hopcount: Number of hops to target device
 220  * @offset: Offset into configuration space
 221  * @len: Length (in bytes) of the maintenance transaction
 222  * @val: Value to be written
 223  *
 224  * Generates a RapidIO maintenance write transaction.
 225  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 226  */
 227 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
 228                          u8 hopcount, u32 offset, int len, u32 data)
 229 {
 230         struct tsi721_device *priv = mport->priv;
 231         u32 temp = data;
 232
 233         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 234                                 offset, len, &temp, 1);
 235 }
 236
 237 /**
 238  * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
 239  * @mport: RapidIO master port structure
 240  *
 241  * Handles inbound port-write interrupts. Copies PW message from an internal
 242  * buffer into PW message FIFO and schedules deferred routine to process
 243  * queued messages.
 244  */
 245 static int
 246 tsi721_pw_handler(struct rio_mport *mport)
 247 {
 248         struct tsi721_device *priv = mport->priv;
 249         u32 pw_stat;
 250         u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
 251
 252
 253         pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
 254
 255         if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
 256                 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
 257                 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
 258                 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
 259                 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
 260
 261                 /* Queue PW message (if there is room in FIFO),
 262                  * otherwise discard it.
 263                  */
 264                 spin_lock(&priv->pw_fifo_lock);
 265                 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
 266                         kfifo_in(&priv->pw_fifo, pw_buf,
 267                                                 TSI721_RIO_PW_MSG_SIZE);
 268                 else
 269                         priv->pw_discard_count++;
 270                 spin_unlock(&priv->pw_fifo_lock);
 271         }
 272
 273         /* Clear pending PW interrupts */
 274         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 275                   priv->regs + TSI721_RIO_PW_RX_STAT);
 276
 277         schedule_work(&priv->pw_work);
 278
 279         return 0;
 280 }
 281
 282 static void tsi721_pw_dpc(struct work_struct *work)
 283 {
 284         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 285                                                     pw_work);
 286         u32 msg_buffer[RIO_PW_MSG_SIZE/sizeof(u32)]; /* Use full size PW message
 287                                                         buffer for RIO layer */
 288
 289         /*
 290          * Process port-write messages
 291          */
 292         while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)msg_buffer,
 293                          TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
 294                 /* Process one message */
 295 #ifdef DEBUG_PW
 296                 {
 297                 u32 i;
 298                 pr_debug("%s : Port-Write Message:", __func__);
 299                 for (i = 0; i < RIO_PW_MSG_SIZE/sizeof(u32); ) {
 300                         pr_debug("0x%02x: %08x %08x %08x %08x", i*4,
 301                                 msg_buffer[i], msg_buffer[i + 1],
 302                                 msg_buffer[i + 2], msg_buffer[i + 3]);
 303                         i += 4;
 304                 }
 305                 pr_debug("\n");
 306                 }
 307 #endif
 308                 /* Pass the port-write message to RIO core for processing */
 309                 rio_inb_pwrite_handler((union rio_pw_msg *)msg_buffer);
 310         }
 311 }
 312
 313 /**
 314  * tsi721_pw_enable - enable/disable port-write interface init
 315  * @mport: Master port implementing the port write unit
 316  * @enable:    1=enable; 0=disable port-write message handling
 317  */
 318 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
 319 {
 320         struct tsi721_device *priv = mport->priv;
 321         u32 rval;
 322
 323         rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
 324
 325         if (enable)
 326                 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
 327         else
 328                 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
 329
 330         /* Clear pending PW interrupts */
 331         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 332                   priv->regs + TSI721_RIO_PW_RX_STAT);
 333         /* Update enable bits */
 334         iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
 335
 336         return 0;
 337 }
 338
 339 /**
 340  * tsi721_dsend - Send a RapidIO doorbell
 341  * @mport: RapidIO master port info
 342  * @index: ID of RapidIO interface
 343  * @destid: Destination ID of target device
 344  * @data: 16-bit info field of RapidIO doorbell
 345  *
 346  * Sends a RapidIO doorbell message. Always returns %0.
 347  */
 348 static int tsi721_dsend(struct rio_mport *mport, int index,
 349                         u16 destid, u16 data)
 350 {
 351         struct tsi721_device *priv = mport->priv;
 352         u32 offset;
 353
 354         offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
 355                  (destid << 2);
 356
 357         dev_dbg(&priv->pdev->dev,
 358                 "Send Doorbell 0x%04x to destID 0x%x\n", data, destid);
 359         iowrite16be(data, priv->odb_base + offset);
 360
 361         return 0;
 362 }
 363
 364 /**
 365  * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
 366  * @mport: RapidIO master port structure
 367  *
 368  * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
 369  * buffer into DB message FIFO and schedules deferred  routine to process
 370  * queued DBs.
 371  */
 372 static int
 373 tsi721_dbell_handler(struct rio_mport *mport)
 374 {
 375         struct tsi721_device *priv = mport->priv;
 376         u32 regval;
 377
 378         /* Disable IDB interrupts */
 379         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 380         regval &= ~TSI721_SR_CHINT_IDBQRCV;
 381         iowrite32(regval,
 382                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 383
 384         schedule_work(&priv->idb_work);
 385
 386         return 0;
 387 }
 388
 389 static void tsi721_db_dpc(struct work_struct *work)
 390 {
 391         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 392                                                     idb_work);
 393         struct rio_mport *mport;
 394         struct rio_dbell *dbell;
 395         int found = 0;
 396         u32 wr_ptr, rd_ptr;
 397         u64 *idb_entry;
 398         u32 regval;
 399         union {
 400                 u64 msg;
 401                 u8  bytes[8];
 402         } idb;
 403
 404         /*
 405          * Process queued inbound doorbells
 406          */
 407         mport = priv->mport;
 408
 409         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 410         rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
 411
 412         while (wr_ptr != rd_ptr) {
 413                 idb_entry = (u64 *)(priv->idb_base +
 414                                         (TSI721_IDB_ENTRY_SIZE * rd_ptr));
 415                 rd_ptr++;
 416                 rd_ptr %= IDB_QSIZE;
 417                 idb.msg = *idb_entry;
 418                 *idb_entry = 0;
 419
 420                 /* Process one doorbell */
 421                 list_for_each_entry(dbell, &mport->dbells, node) {
 422                         if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
 423                             (dbell->res->end >= DBELL_INF(idb.bytes))) {
 424                                 found = 1;
 425                                 break;
 426                         }
 427                 }
 428
 429                 if (found) {
 430                         dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
 431                                     DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
 432                 } else {
 433                         dev_dbg(&priv->pdev->dev,
 434                                 "spurious inb doorbell, sid %2.2x tid %2.2x"
 435                                 " info %4.4x\n", DBELL_SID(idb.bytes),
 436                                 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
 437                 }
 438         }
 439
 440         iowrite32(rd_ptr & (IDB_QSIZE - 1),
 441                 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
 442
 443         /* Re-enable IDB interrupts */
 444         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 445         regval |= TSI721_SR_CHINT_IDBQRCV;
 446         iowrite32(regval,
 447                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 448 }
 449
 450 /**
 451  * tsi721_irqhandler - Tsi721 interrupt handler
 452  * @irq: Linux interrupt number
 453  * @ptr: Pointer to interrupt-specific data (mport structure)
 454  *
 455  * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
 456  * interrupt events and calls an event-specific handler(s).
 457  */
 458 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
 459 {
 460         struct rio_mport *mport = (struct rio_mport *)ptr;
 461         struct tsi721_device *priv = mport->priv;
 462         u32 dev_int;
 463         u32 dev_ch_int;
 464         u32 intval;
 465         u32 ch_inte;
 466
 467         dev_int = ioread32(priv->regs + TSI721_DEV_INT);
 468         if (!dev_int)
 469                 return IRQ_NONE;
 470
 471         dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
 472
 473         if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
 474                 /* Service SR2PC Channel interrupts */
 475                 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
 476                         /* Service Inbound Doorbell interrupt */
 477                         intval = ioread32(priv->regs +
 478                                                 TSI721_SR_CHINT(IDB_QUEUE));
 479                         if (intval & TSI721_SR_CHINT_IDBQRCV)
 480                                 tsi721_dbell_handler(mport);
 481                         else
 482                                 dev_info(&priv->pdev->dev,
 483                                         "Unsupported SR_CH_INT %x\n", intval);
 484
 485                         /* Clear interrupts */
 486                         iowrite32(intval,
 487                                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 488                         ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 489                 }
 490         }
 491
 492         if (dev_int & TSI721_DEV_INT_SMSG_CH) {
 493                 int ch;
 494
 495                 /*
 496                  * Service channel interrupts from Messaging Engine
 497                  */
 498
 499                 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
 500                         /* Disable signaled OB MSG Channel interrupts */
 501                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 502                         ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
 503                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 504
 505                         /*
 506                          * Process Inbound Message interrupt for each MBOX
 507                          */
 508                         for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
 509                                 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
 510                                         continue;
 511                                 tsi721_imsg_handler(priv, ch);
 512                         }
 513                 }
 514
 515                 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
 516                         /* Disable signaled OB MSG Channel interrupts */
 517                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 518                         ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
 519                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 520
 521                         /*
 522                          * Process Outbound Message interrupts for each MBOX
 523                          */
 524
 525                         for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
 526                                 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
 527                                         continue;
 528                                 tsi721_omsg_handler(priv, ch);
 529                         }
 530                 }
 531         }
 532
 533         if (dev_int & TSI721_DEV_INT_SRIO) {
 534                 /* Service SRIO MAC interrupts */
 535                 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 536                 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
 537                         tsi721_pw_handler(mport);
 538         }
 539
 540 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 541         if (dev_int & TSI721_DEV_INT_BDMA_CH) {
 542                 int ch;
 543
 544                 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
 545                         dev_dbg(&priv->pdev->dev,
 546                                 "IRQ from DMA channel 0x%08x\n", dev_ch_int);
 547
 548                         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
 549                                 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
 550                                         continue;
 551                                 tsi721_bdma_handler(&priv->bdma[ch]);
 552                         }
 553                 }
 554         }
 555 #endif
 556         return IRQ_HANDLED;
 557 }
 558
 559 static void tsi721_interrupts_init(struct tsi721_device *priv)
 560 {
 561         u32 intr;
 562
 563         /* Enable IDB interrupts */
 564         iowrite32(TSI721_SR_CHINT_ALL,
 565                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 566         iowrite32(TSI721_SR_CHINT_IDBQRCV,
 567                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 568
 569         /* Enable SRIO MAC interrupts */
 570         iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
 571                 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
 572
 573         /* Enable interrupts from channels in use */
 574 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 575         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
 576                 (TSI721_INT_BDMA_CHAN_M &
 577                  ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
 578 #else
 579         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
 580 #endif
 581         iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
 582
 583         if (priv->flags & TSI721_USING_MSIX)
 584                 intr = TSI721_DEV_INT_SRIO;
 585         else
 586                 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
 587                         TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
 588
 589         iowrite32(intr, priv->regs + TSI721_DEV_INTE);
 590         ioread32(priv->regs + TSI721_DEV_INTE);
 591 }
 592
 593 #ifdef CONFIG_PCI_MSI
 594 /**
 595  * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
 596  * @irq: Linux interrupt number
 597  * @ptr: Pointer to interrupt-specific data (mport structure)
 598  *
 599  * Handles outbound messaging interrupts signaled using MSI-X.
 600  */
 601 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
 602 {
 603         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 604         int mbox;
 605
 606         mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
 607         tsi721_omsg_handler(priv, mbox);
 608         return IRQ_HANDLED;
 609 }
 610
 611 /**
 612  * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
 613  * @irq: Linux interrupt number
 614  * @ptr: Pointer to interrupt-specific data (mport structure)
 615  *
 616  * Handles inbound messaging interrupts signaled using MSI-X.
 617  */
 618 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
 619 {
 620         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 621         int mbox;
 622
 623         mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
 624         tsi721_imsg_handler(priv, mbox + 4);
 625         return IRQ_HANDLED;
 626 }
 627
 628 /**
 629  * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
 630  * @irq: Linux interrupt number
 631  * @ptr: Pointer to interrupt-specific data (mport structure)
 632  *
 633  * Handles Tsi721 interrupts from SRIO MAC.
 634  */
 635 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
 636 {
 637         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 638         u32 srio_int;
 639
 640         /* Service SRIO MAC interrupts */
 641         srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 642         if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
 643                 tsi721_pw_handler((struct rio_mport *)ptr);
 644
 645         return IRQ_HANDLED;
 646 }
 647
 648 /**
 649  * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
 650  * @irq: Linux interrupt number
 651  * @ptr: Pointer to interrupt-specific data (mport structure)
 652  *
 653  * Handles Tsi721 interrupts from SR2PC Channel.
 654  * NOTE: At this moment services only one SR2PC channel associated with inbound
 655  * doorbells.
 656  */
 657 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
 658 {
 659         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 660         u32 sr_ch_int;
 661
 662         /* Service Inbound DB interrupt from SR2PC channel */
 663         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 664         if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
 665                 tsi721_dbell_handler((struct rio_mport *)ptr);
 666
 667         /* Clear interrupts */
 668         iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 669         /* Read back to ensure that interrupt was cleared */
 670         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 671
 672         return IRQ_HANDLED;
 673 }
 674
 675 /**
 676  * tsi721_request_msix - register interrupt service for MSI-X mode.
 677  * @mport: RapidIO master port structure
 678  *
 679  * Registers MSI-X interrupt service routines for interrupts that are active
 680  * immediately after mport initialization. Messaging interrupt service routines
 681  * should be registered during corresponding open requests.
 682  */
 683 static int tsi721_request_msix(struct rio_mport *mport)
 684 {
 685         struct tsi721_device *priv = mport->priv;
 686         int err = 0;
 687
 688         err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
 689                         tsi721_sr2pc_ch_msix, 0,
 690                         priv->msix[TSI721_VECT_IDB].irq_name, (void *)mport);
 691         if (err)
 692                 goto out;
 693
 694         err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
 695                         tsi721_srio_msix, 0,
 696                         priv->msix[TSI721_VECT_PWRX].irq_name, (void *)mport);
 697         if (err)
 698                 free_irq(
 699                         priv->msix[TSI721_VECT_IDB].vector,
 700                         (void *)mport);
 701 out:
 702         return err;
 703 }
 704
 705 /**
 706  * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
 707  * @priv: pointer to tsi721 private data
 708  *
 709  * Configures MSI-X support for Tsi721. Supports only an exact number
 710  * of requested vectors.
 711  */
 712 static int tsi721_enable_msix(struct tsi721_device *priv)
 713 {
 714         struct msix_entry entries[TSI721_VECT_MAX];
 715         int err;
 716         int i;
 717
 718         entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
 719         entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
 720
 721         /*
 722          * Initialize MSI-X entries for Messaging Engine:
 723          * this driver supports four RIO mailboxes (inbound and outbound)
 724          * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
 725          * offset +4 is added to IB MBOX number.
 726          */
 727         for (i = 0; i < RIO_MAX_MBOX; i++) {
 728                 entries[TSI721_VECT_IMB0_RCV + i].entry =
 729                                         TSI721_MSIX_IMSG_DQ_RCV(i + 4);
 730                 entries[TSI721_VECT_IMB0_INT + i].entry =
 731                                         TSI721_MSIX_IMSG_INT(i + 4);
 732                 entries[TSI721_VECT_OMB0_DONE + i].entry =
 733                                         TSI721_MSIX_OMSG_DONE(i);
 734                 entries[TSI721_VECT_OMB0_INT + i].entry =
 735                                         TSI721_MSIX_OMSG_INT(i);
 736         }
 737
 738 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 739         /*
 740          * Initialize MSI-X entries for Block DMA Engine:
 741          * this driver supports XXX DMA channels
 742          * (one is reserved for SRIO maintenance transactions)
 743          */
 744         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 745                 entries[TSI721_VECT_DMA0_DONE + i].entry =
 746                                         TSI721_MSIX_DMACH_DONE(i);
 747                 entries[TSI721_VECT_DMA0_INT + i].entry =
 748                                         TSI721_MSIX_DMACH_INT(i);
 749         }
 750 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 751
 752         err = pci_enable_msix(priv->pdev, entries, ARRAY_SIZE(entries));
 753         if (err) {
 754                 if (err > 0)
 755                         dev_info(&priv->pdev->dev,
 756                                  "Only %d MSI-X vectors available, "
 757                                  "not using MSI-X\n", err);
 758                 else
 759                         dev_err(&priv->pdev->dev,
 760                                 "Failed to enable MSI-X (err=%d)\n", err);
 761                 return err;
 762         }
 763
 764         /*
 765          * Copy MSI-X vector information into tsi721 private structure
 766          */
 767         priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
 768         snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
 769                  DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
 770         priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
 771         snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
 772                  DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
 773
 774         for (i = 0; i < RIO_MAX_MBOX; i++) {
 775                 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
 776                                 entries[TSI721_VECT_IMB0_RCV + i].vector;
 777                 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
 778                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
 779                          i, pci_name(priv->pdev));
 780
 781                 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
 782                                 entries[TSI721_VECT_IMB0_INT + i].vector;
 783                 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
 784                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
 785                          i, pci_name(priv->pdev));
 786
 787                 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
 788                                 entries[TSI721_VECT_OMB0_DONE + i].vector;
 789                 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
 790                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
 791                          i, pci_name(priv->pdev));
 792
 793                 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
 794                                 entries[TSI721_VECT_OMB0_INT + i].vector;
 795                 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
 796                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
 797                          i, pci_name(priv->pdev));
 798         }
 799
 800 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 801         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 802                 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
 803                                 entries[TSI721_VECT_DMA0_DONE + i].vector;
 804                 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
 805                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
 806                          i, pci_name(priv->pdev));
 807
 808                 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
 809                                 entries[TSI721_VECT_DMA0_INT + i].vector;
 810                 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
 811                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
 812                          i, pci_name(priv->pdev));
 813         }
 814 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 815
 816         return 0;
 817 }
 818 #endif /* CONFIG_PCI_MSI */
 819
 820 static int tsi721_request_irq(struct rio_mport *mport)
 821 {
 822         struct tsi721_device *priv = mport->priv;
 823         int err;
 824
 825 #ifdef CONFIG_PCI_MSI
 826         if (priv->flags & TSI721_USING_MSIX)
 827                 err = tsi721_request_msix(mport);
 828         else
 829 #endif
 830                 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
 831                           (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
 832                           DRV_NAME, (void *)mport);
 833
 834         if (err)
 835                 dev_err(&priv->pdev->dev,
 836                         "Unable to allocate interrupt, Error: %d\n", err);
 837
 838         return err;
 839 }
 840
 841 /**
 842  * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
 843  * translation regions.
 844  * @priv: pointer to tsi721 private data
 845  *
 846  * Disables SREP translation regions.
 847  */
 848 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
 849 {
 850         int i;
 851
 852         /* Disable all PC2SR translation windows */
 853         for (i = 0; i < TSI721_OBWIN_NUM; i++)
 854                 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
 855 }
 856
 857 /**
 858  * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
 859  * translation regions.
 860  * @priv: pointer to tsi721 private data
 861  *
 862  * Disables inbound windows.
 863  */
 864 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
 865 {
 866         int i;
 867
 868         /* Disable all SR2PC inbound windows */
 869         for (i = 0; i < TSI721_IBWIN_NUM; i++)
 870                 iowrite32(0, priv->regs + TSI721_IBWINLB(i));
 871 }
 872
 873 /**
 874  * tsi721_port_write_init - Inbound port write interface init
 875  * @priv: pointer to tsi721 private data
 876  *
 877  * Initializes inbound port write handler.
 878  * Returns %0 on success or %-ENOMEM on failure.
 879  */
 880 static int tsi721_port_write_init(struct tsi721_device *priv)
 881 {
 882         priv->pw_discard_count = 0;
 883         INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
 884         spin_lock_init(&priv->pw_fifo_lock);
 885         if (kfifo_alloc(&priv->pw_fifo,
 886                         TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
 887                 dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");
 888                 return -ENOMEM;
 889         }
 890
 891         /* Use reliable port-write capture mode */
 892         iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
 893         return 0;
 894 }
 895
 896 static int tsi721_doorbell_init(struct tsi721_device *priv)
 897 {
 898         /* Outbound Doorbells do not require any setup.
 899          * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
 900          * That BAR1 was mapped during the probe routine.
 901          */
 902
 903         /* Initialize Inbound Doorbell processing DPC and queue */
 904         priv->db_discard_count = 0;
 905         INIT_WORK(&priv->idb_work, tsi721_db_dpc);
 906
 907         /* Allocate buffer for inbound doorbells queue */
 908         priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
 909                                 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
 910                                 &priv->idb_dma, GFP_KERNEL);
 911         if (!priv->idb_base)
 912                 return -ENOMEM;
 913
 914         dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",
 915                 priv->idb_base, (unsigned long long)priv->idb_dma);
 916
 917         iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
 918                 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
 919         iowrite32(((u64)priv->idb_dma >> 32),
 920                 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
 921         iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
 922                 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
 923         /* Enable accepting all inbound doorbells */
 924         iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
 925
 926         iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
 927
 928         iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
 929
 930         return 0;
 931 }
 932
 933 static void tsi721_doorbell_free(struct tsi721_device *priv)
 934 {
 935         if (priv->idb_base == NULL)
 936                 return;
 937
 938         /* Free buffer allocated for inbound doorbell queue */
 939         dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
 940                           priv->idb_base, priv->idb_dma);
 941         priv->idb_base = NULL;
 942 }
 943
 944 /**
 945  * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
 946  * @priv: pointer to tsi721 private data
 947  *
 948  * Initialize BDMA channel allocated for RapidIO maintenance read/write
 949  * request generation
 950  * Returns %0 on success or %-ENOMEM on failure.
 951  */
 952 static int tsi721_bdma_maint_init(struct tsi721_device *priv)
 953 {
 954         struct tsi721_dma_desc *bd_ptr;
 955         u64             *sts_ptr;
 956         dma_addr_t      bd_phys, sts_phys;
 957         int             sts_size;
 958         int             bd_num = 2;
 959         void __iomem    *regs;
 960
 961         dev_dbg(&priv->pdev->dev,
 962                 "Init Block DMA Engine for Maintenance requests, CH%d\n",
 963                 TSI721_DMACH_MAINT);
 964
 965         /*
 966          * Initialize DMA channel for maintenance requests
 967          */
 968
 969         priv->mdma.ch_id = TSI721_DMACH_MAINT;
 970         regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
 971
 972         /* Allocate space for DMA descriptors */
 973         bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
 974                                         bd_num * sizeof(struct tsi721_dma_desc),
 975                                         &bd_phys, GFP_KERNEL);
 976         if (!bd_ptr)
 977                 return -ENOMEM;
 978
 979         priv->mdma.bd_num = bd_num;
 980         priv->mdma.bd_phys = bd_phys;
 981         priv->mdma.bd_base = bd_ptr;
 982
 983         dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",
 984                 bd_ptr, (unsigned long long)bd_phys);
 985
 986         /* Allocate space for descriptor status FIFO */
 987         sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
 988                                         bd_num : TSI721_DMA_MINSTSSZ;
 989         sts_size = roundup_pow_of_two(sts_size);
 990         sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
 991                                      sts_size * sizeof(struct tsi721_dma_sts),
 992                                      &sts_phys, GFP_KERNEL);
 993         if (!sts_ptr) {
 994                 /* Free space allocated for DMA descriptors */
 995                 dma_free_coherent(&priv->pdev->dev,
 996                                   bd_num * sizeof(struct tsi721_dma_desc),
 997                                   bd_ptr, bd_phys);
 998                 priv->mdma.bd_base = NULL;
 999                 return -ENOMEM;
1000         }
1001
1002         priv->mdma.sts_phys = sts_phys;
1003         priv->mdma.sts_base = sts_ptr;
1004         priv->mdma.sts_size = sts_size;
1005
1006         dev_dbg(&priv->pdev->dev,
1007                 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
1008                 sts_ptr, (unsigned long long)sts_phys, sts_size);
1009
1010         /* Initialize DMA descriptors ring */
1011         bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1012         bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1013                                                  TSI721_DMAC_DPTRL_MASK);
1014         bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1015
1016         /* Setup DMA descriptor pointers */
1017         iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
1018         iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
1019                 regs + TSI721_DMAC_DPTRL);
1020
1021         /* Setup descriptor status FIFO */
1022         iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
1023         iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
1024                 regs + TSI721_DMAC_DSBL);
1025         iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
1026                 regs + TSI721_DMAC_DSSZ);
1027
1028         /* Clear interrupt bits */
1029         iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
1030
1031         ioread32(regs + TSI721_DMAC_INT);
1032
1033         /* Toggle DMA channel initialization */
1034         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1035         ioread32(regs + TSI721_DMAC_CTL);
1036         udelay(10);
1037
1038         return 0;
1039 }
1040
1041 static int tsi721_bdma_maint_free(struct tsi721_device *priv)
1042 {
1043         u32 ch_stat;
1044         struct tsi721_bdma_maint *mdma = &priv->mdma;
1045         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
1046
1047         if (mdma->bd_base == NULL)
1048                 return 0;
1049
1050         /* Check if DMA channel still running */
1051         ch_stat = ioread32(regs + TSI721_DMAC_STS);
1052         if (ch_stat & TSI721_DMAC_STS_RUN)
1053                 return -EFAULT;
1054
1055         /* Put DMA channel into init state */
1056         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1057
1058         /* Free space allocated for DMA descriptors */
1059         dma_free_coherent(&priv->pdev->dev,
1060                 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1061                 mdma->bd_base, mdma->bd_phys);
1062         mdma->bd_base = NULL;
1063
1064         /* Free space allocated for status FIFO */
1065         dma_free_coherent(&priv->pdev->dev,
1066                 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1067                 mdma->sts_base, mdma->sts_phys);
1068         mdma->sts_base = NULL;
1069         return 0;
1070 }
1071
1072 /* Enable Inbound Messaging Interrupts */
1073 static void
1074 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1075                                   u32 inte_mask)
1076 {
1077         u32 rval;
1078
1079         if (!inte_mask)
1080                 return;
1081
1082         /* Clear pending Inbound Messaging interrupts */
1083         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1084
1085         /* Enable Inbound Messaging interrupts */
1086         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1087         iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1088
1089         if (priv->flags & TSI721_USING_MSIX)
1090                 return; /* Finished if we are in MSI-X mode */
1091
1092         /*
1093          * For MSI and INTA interrupt signalling we need to enable next levels
1094          */
1095
1096         /* Enable Device Channel Interrupt */
1097         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1098         iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1099                   priv->regs + TSI721_DEV_CHAN_INTE);
1100 }
1101
1102 /* Disable Inbound Messaging Interrupts */
1103 static void
1104 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1105                                    u32 inte_mask)
1106 {
1107         u32 rval;
1108
1109         if (!inte_mask)
1110                 return;
1111
1112         /* Clear pending Inbound Messaging interrupts */
1113         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1114
1115         /* Disable Inbound Messaging interrupts */
1116         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1117         rval &= ~inte_mask;
1118         iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1119
1120         if (priv->flags & TSI721_USING_MSIX)
1121                 return; /* Finished if we are in MSI-X mode */
1122
1123         /*
1124          * For MSI and INTA interrupt signalling we need to disable next levels
1125          */
1126
1127         /* Disable Device Channel Interrupt */
1128         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1129         rval &= ~TSI721_INT_IMSG_CHAN(ch);
1130         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1131 }
1132
1133 /* Enable Outbound Messaging interrupts */
1134 static void
1135 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1136                                   u32 inte_mask)
1137 {
1138         u32 rval;
1139
1140         if (!inte_mask)
1141                 return;
1142
1143         /* Clear pending Outbound Messaging interrupts */
1144         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1145
1146         /* Enable Outbound Messaging channel interrupts */
1147         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1148         iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1149
1150         if (priv->flags & TSI721_USING_MSIX)
1151                 return; /* Finished if we are in MSI-X mode */
1152
1153         /*
1154          * For MSI and INTA interrupt signalling we need to enable next levels
1155          */
1156
1157         /* Enable Device Channel Interrupt */
1158         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1159         iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1160                   priv->regs + TSI721_DEV_CHAN_INTE);
1161 }
1162
1163 /* Disable Outbound Messaging interrupts */
1164 static void
1165 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1166                                    u32 inte_mask)
1167 {
1168         u32 rval;
1169
1170         if (!inte_mask)
1171                 return;
1172
1173         /* Clear pending Outbound Messaging interrupts */
1174         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1175
1176         /* Disable Outbound Messaging interrupts */
1177         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1178         rval &= ~inte_mask;
1179         iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1180
1181         if (priv->flags & TSI721_USING_MSIX)
1182                 return; /* Finished if we are in MSI-X mode */
1183
1184         /*
1185          * For MSI and INTA interrupt signalling we need to disable next levels
1186          */
1187
1188         /* Disable Device Channel Interrupt */
1189         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1190         rval &= ~TSI721_INT_OMSG_CHAN(ch);
1191         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1192 }
1193
1194 /**
1195  * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1196  * @mport: Master port with outbound message queue
1197  * @rdev: Target of outbound message
1198  * @mbox: Outbound mailbox
1199  * @buffer: Message to add to outbound queue
1200  * @len: Length of message
1201  */
1202 static int
1203 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1204                         void *buffer, size_t len)
1205 {
1206         struct tsi721_device *priv = mport->priv;
1207         struct tsi721_omsg_desc *desc;
1208         u32 tx_slot;
1209
1210         if (!priv->omsg_init[mbox] ||
1211             len > TSI721_MSG_MAX_SIZE || len < 8)
1212                 return -EINVAL;
1213
1214         tx_slot = priv->omsg_ring[mbox].tx_slot;
1215
1216         /* Copy copy message into transfer buffer */
1217         memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1218
1219         if (len & 0x7)
1220                 len += 8;
1221
1222         /* Build descriptor associated with buffer */
1223         desc = priv->omsg_ring[mbox].omd_base;
1224         desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1225         if (tx_slot % 4 == 0)
1226                 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1227
1228         desc[tx_slot].msg_info =
1229                 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1230                             (0xe << 12) | (len & 0xff8));
1231         desc[tx_slot].bufptr_lo =
1232                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1233                             0xffffffff);
1234         desc[tx_slot].bufptr_hi =
1235                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1236
1237         priv->omsg_ring[mbox].wr_count++;
1238
1239         /* Go to next descriptor */
1240         if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1241                 priv->omsg_ring[mbox].tx_slot = 0;
1242                 /* Move through the ring link descriptor at the end */
1243                 priv->omsg_ring[mbox].wr_count++;
1244         }
1245
1246         mb();
1247
1248         /* Set new write count value */
1249         iowrite32(priv->omsg_ring[mbox].wr_count,
1250                 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1251         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1252
1253         return 0;
1254 }
1255
1256 /**
1257  * tsi721_omsg_handler - Outbound Message Interrupt Handler
1258  * @priv: pointer to tsi721 private data
1259  * @ch:   number of OB MSG channel to service
1260  *
1261  * Services channel interrupts from outbound messaging engine.
1262  */
1263 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1264 {
1265         u32 omsg_int;
1266
1267         spin_lock(&priv->omsg_ring[ch].lock);
1268
1269         omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1270
1271         if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1272                 dev_info(&priv->pdev->dev,
1273                         "OB MBOX%d: Status FIFO is full\n", ch);
1274
1275         if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1276                 u32 srd_ptr;
1277                 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1278                 int i, j;
1279                 u32 tx_slot;
1280
1281                 /*
1282                  * Find last successfully processed descriptor
1283                  */
1284
1285                 /* Check and clear descriptor status FIFO entries */
1286                 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1287                 sts_ptr = priv->omsg_ring[ch].sts_base;
1288                 j = srd_ptr * 8;
1289                 while (sts_ptr[j]) {
1290                         for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1291                                 prev_ptr = last_ptr;
1292                                 last_ptr = le64_to_cpu(sts_ptr[j]);
1293                                 sts_ptr[j] = 0;
1294                         }
1295
1296                         ++srd_ptr;
1297                         srd_ptr %= priv->omsg_ring[ch].sts_size;
1298                         j = srd_ptr * 8;
1299                 }
1300
1301                 if (last_ptr == 0)
1302                         goto no_sts_update;
1303
1304                 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1305                 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1306
1307                 if (!priv->mport->outb_msg[ch].mcback)
1308                         goto no_sts_update;
1309
1310                 /* Inform upper layer about transfer completion */
1311
1312                 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1313                                                 sizeof(struct tsi721_omsg_desc);
1314
1315                 /*
1316                  * Check if this is a Link Descriptor (LD).
1317                  * If yes, ignore LD and use descriptor processed
1318                  * before LD.
1319                  */
1320                 if (tx_slot == priv->omsg_ring[ch].size) {
1321                         if (prev_ptr)
1322                                 tx_slot = (prev_ptr -
1323                                         (u64)priv->omsg_ring[ch].omd_phys)/
1324                                                 sizeof(struct tsi721_omsg_desc);
1325                         else
1326                                 goto no_sts_update;
1327                 }
1328
1329                 /* Move slot index to the next message to be sent */
1330                 ++tx_slot;
1331                 if (tx_slot == priv->omsg_ring[ch].size)
1332                         tx_slot = 0;
1333                 BUG_ON(tx_slot >= priv->omsg_ring[ch].size);
1334                 priv->mport->outb_msg[ch].mcback(priv->mport,
1335                                 priv->omsg_ring[ch].dev_id, ch,
1336                                 tx_slot);
1337         }
1338
1339 no_sts_update:
1340
1341         if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1342                 /*
1343                 * Outbound message operation aborted due to error,
1344                 * reinitialize OB MSG channel
1345                 */
1346
1347                 dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",
1348                         ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1349
1350                 iowrite32(TSI721_OBDMAC_INT_ERROR,
1351                                 priv->regs + TSI721_OBDMAC_INT(ch));
1352                 iowrite32(TSI721_OBDMAC_CTL_INIT,
1353                                 priv->regs + TSI721_OBDMAC_CTL(ch));
1354                 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1355
1356                 /* Inform upper level to clear all pending tx slots */
1357                 if (priv->mport->outb_msg[ch].mcback)
1358                         priv->mport->outb_msg[ch].mcback(priv->mport,
1359                                         priv->omsg_ring[ch].dev_id, ch,
1360                                         priv->omsg_ring[ch].tx_slot);
1361                 /* Synch tx_slot tracking */
1362                 iowrite32(priv->omsg_ring[ch].tx_slot,
1363                         priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1364                 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1365                 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1366                 priv->omsg_ring[ch].sts_rdptr = 0;
1367         }
1368
1369         /* Clear channel interrupts */
1370         iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1371
1372         if (!(priv->flags & TSI721_USING_MSIX)) {
1373                 u32 ch_inte;
1374
1375                 /* Re-enable channel interrupts */
1376                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1377                 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1378                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1379         }
1380
1381         spin_unlock(&priv->omsg_ring[ch].lock);
1382 }
1383
1384 /**
1385  * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1386  * @mport: Master port implementing Outbound Messaging Engine
1387  * @dev_id: Device specific pointer to pass on event
1388  * @mbox: Mailbox to open
1389  * @entries: Number of entries in the outbound mailbox ring
1390  */
1391 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1392                                  int mbox, int entries)
1393 {
1394         struct tsi721_device *priv = mport->priv;
1395         struct tsi721_omsg_desc *bd_ptr;
1396         int i, rc = 0;
1397
1398         if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1399             (entries > (TSI721_OMSGD_RING_SIZE)) ||
1400             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1401                 rc = -EINVAL;
1402                 goto out;
1403         }
1404
1405         priv->omsg_ring[mbox].dev_id = dev_id;
1406         priv->omsg_ring[mbox].size = entries;
1407         priv->omsg_ring[mbox].sts_rdptr = 0;
1408         spin_lock_init(&priv->omsg_ring[mbox].lock);
1409
1410         /* Outbound Msg Buffer allocation based on
1411            the number of maximum descriptor entries */
1412         for (i = 0; i < entries; i++) {
1413                 priv->omsg_ring[mbox].omq_base[i] =
1414                         dma_alloc_coherent(
1415                                 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1416                                 &priv->omsg_ring[mbox].omq_phys[i],
1417                                 GFP_KERNEL);
1418                 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1419                         dev_dbg(&priv->pdev->dev,
1420                                 "Unable to allocate OB MSG data buffer for"
1421                                 " MBOX%d\n", mbox);
1422                         rc = -ENOMEM;
1423                         goto out_buf;
1424                 }
1425         }
1426
1427         /* Outbound message descriptor allocation */
1428         priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1429                                 &priv->pdev->dev,
1430                                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1431                                 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1432         if (priv->omsg_ring[mbox].omd_base == NULL) {
1433                 dev_dbg(&priv->pdev->dev,
1434                         "Unable to allocate OB MSG descriptor memory "
1435                         "for MBOX%d\n", mbox);
1436                 rc = -ENOMEM;
1437                 goto out_buf;
1438         }
1439
1440         priv->omsg_ring[mbox].tx_slot = 0;
1441
1442         /* Outbound message descriptor status FIFO allocation */
1443         priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1444         priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1445                         priv->omsg_ring[mbox].sts_size *
1446                                                 sizeof(struct tsi721_dma_sts),
1447                         &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1448         if (priv->omsg_ring[mbox].sts_base == NULL) {
1449                 dev_dbg(&priv->pdev->dev,
1450                         "Unable to allocate OB MSG descriptor status FIFO "
1451                         "for MBOX%d\n", mbox);
1452                 rc = -ENOMEM;
1453                 goto out_desc;
1454         }
1455
1456         /*
1457          * Configure Outbound Messaging Engine
1458          */
1459
1460         /* Setup Outbound Message descriptor pointer */
1461         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1462                         priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1463         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1464                                         TSI721_OBDMAC_DPTRL_MASK),
1465                         priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1466
1467         /* Setup Outbound Message descriptor status FIFO */
1468         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1469                         priv->regs + TSI721_OBDMAC_DSBH(mbox));
1470         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1471                                         TSI721_OBDMAC_DSBL_MASK),
1472                         priv->regs + TSI721_OBDMAC_DSBL(mbox));
1473         iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1474                 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1475
1476         /* Enable interrupts */
1477
1478 #ifdef CONFIG_PCI_MSI
1479         if (priv->flags & TSI721_USING_MSIX) {
1480                 /* Request interrupt service if we are in MSI-X mode */
1481                 rc = request_irq(
1482                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1483                         tsi721_omsg_msix, 0,
1484                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].irq_name,
1485                         (void *)mport);
1486
1487                 if (rc) {
1488                         dev_dbg(&priv->pdev->dev,
1489                                 "Unable to allocate MSI-X interrupt for "
1490                                 "OBOX%d-DONE\n", mbox);
1491                         goto out_stat;
1492                 }
1493
1494                 rc = request_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1495                         tsi721_omsg_msix, 0,
1496                         priv->msix[TSI721_VECT_OMB0_INT + mbox].irq_name,
1497                         (void *)mport);
1498
1499                 if (rc) {
1500                         dev_dbg(&priv->pdev->dev,
1501                                 "Unable to allocate MSI-X interrupt for "
1502                                 "MBOX%d-INT\n", mbox);
1503                         free_irq(
1504                                 priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1505                                 (void *)mport);
1506                         goto out_stat;
1507                 }
1508         }
1509 #endif /* CONFIG_PCI_MSI */
1510
1511         tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1512
1513         /* Initialize Outbound Message descriptors ring */
1514         bd_ptr = priv->omsg_ring[mbox].omd_base;
1515         bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1516         bd_ptr[entries].msg_info = 0;
1517         bd_ptr[entries].next_lo =
1518                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1519                 TSI721_OBDMAC_DPTRL_MASK);
1520         bd_ptr[entries].next_hi =
1521                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1522         priv->omsg_ring[mbox].wr_count = 0;
1523         mb();
1524
1525         /* Initialize Outbound Message engine */
1526         iowrite32(TSI721_OBDMAC_CTL_INIT, priv->regs + TSI721_OBDMAC_CTL(mbox));
1527         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1528         udelay(10);
1529
1530         priv->omsg_init[mbox] = 1;
1531
1532         return 0;
1533
1534 #ifdef CONFIG_PCI_MSI
1535 out_stat:
1536         dma_free_coherent(&priv->pdev->dev,
1537                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1538                 priv->omsg_ring[mbox].sts_base,
1539                 priv->omsg_ring[mbox].sts_phys);
1540
1541         priv->omsg_ring[mbox].sts_base = NULL;
1542 #endif /* CONFIG_PCI_MSI */
1543
1544 out_desc:
1545         dma_free_coherent(&priv->pdev->dev,
1546                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1547                 priv->omsg_ring[mbox].omd_base,
1548                 priv->omsg_ring[mbox].omd_phys);
1549
1550         priv->omsg_ring[mbox].omd_base = NULL;
1551
1552 out_buf:
1553         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1554                 if (priv->omsg_ring[mbox].omq_base[i]) {
1555                         dma_free_coherent(&priv->pdev->dev,
1556                                 TSI721_MSG_BUFFER_SIZE,
1557                                 priv->omsg_ring[mbox].omq_base[i],
1558                                 priv->omsg_ring[mbox].omq_phys[i]);
1559
1560                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1561                 }
1562         }
1563
1564 out:
1565         return rc;
1566 }
1567
1568 /**
1569  * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
1570  * @mport: Master port implementing the outbound message unit
1571  * @mbox: Mailbox to close
1572  */
1573 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
1574 {
1575         struct tsi721_device *priv = mport->priv;
1576         u32 i;
1577
1578         if (!priv->omsg_init[mbox])
1579                 return;
1580         priv->omsg_init[mbox] = 0;
1581
1582         /* Disable Interrupts */
1583
1584         tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1585
1586 #ifdef CONFIG_PCI_MSI
1587         if (priv->flags & TSI721_USING_MSIX) {
1588                 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1589                          (void *)mport);
1590                 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1591                          (void *)mport);
1592         }
1593 #endif /* CONFIG_PCI_MSI */
1594
1595         /* Free OMSG Descriptor Status FIFO */
1596         dma_free_coherent(&priv->pdev->dev,
1597                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1598                 priv->omsg_ring[mbox].sts_base,
1599                 priv->omsg_ring[mbox].sts_phys);
1600
1601         priv->omsg_ring[mbox].sts_base = NULL;
1602
1603         /* Free OMSG descriptors */
1604         dma_free_coherent(&priv->pdev->dev,
1605                 (priv->omsg_ring[mbox].size + 1) *
1606                         sizeof(struct tsi721_omsg_desc),
1607                 priv->omsg_ring[mbox].omd_base,
1608                 priv->omsg_ring[mbox].omd_phys);
1609
1610         priv->omsg_ring[mbox].omd_base = NULL;
1611
1612         /* Free message buffers */
1613         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1614                 if (priv->omsg_ring[mbox].omq_base[i]) {
1615                         dma_free_coherent(&priv->pdev->dev,
1616                                 TSI721_MSG_BUFFER_SIZE,
1617                                 priv->omsg_ring[mbox].omq_base[i],
1618                                 priv->omsg_ring[mbox].omq_phys[i]);
1619
1620                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1621                 }
1622         }
1623 }
1624
1625 /**
1626  * tsi721_imsg_handler - Inbound Message Interrupt Handler
1627  * @priv: pointer to tsi721 private data
1628  * @ch: inbound message channel number to service
1629  *
1630  * Services channel interrupts from inbound messaging engine.
1631  */
1632 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
1633 {
1634         u32 mbox = ch - 4;
1635         u32 imsg_int;
1636
1637         spin_lock(&priv->imsg_ring[mbox].lock);
1638
1639         imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
1640
1641         if (imsg_int & TSI721_IBDMAC_INT_SRTO)
1642                 dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",
1643                         mbox);
1644
1645         if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
1646                 dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",
1647                         mbox);
1648
1649         if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
1650                 dev_info(&priv->pdev->dev,
1651                         "IB MBOX%d IB free queue low\n", mbox);
1652
1653         /* Clear IB channel interrupts */
1654         iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
1655
1656         /* If an IB Msg is received notify the upper layer */
1657         if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
1658                 priv->mport->inb_msg[mbox].mcback)
1659                 priv->mport->inb_msg[mbox].mcback(priv->mport,
1660                                 priv->imsg_ring[mbox].dev_id, mbox, -1);
1661
1662         if (!(priv->flags & TSI721_USING_MSIX)) {
1663                 u32 ch_inte;
1664
1665                 /* Re-enable channel interrupts */
1666                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1667                 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
1668                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1669         }
1670
1671         spin_unlock(&priv->imsg_ring[mbox].lock);
1672 }
1673
1674 /**
1675  * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
1676  * @mport: Master port implementing the Inbound Messaging Engine
1677  * @dev_id: Device specific pointer to pass on event
1678  * @mbox: Mailbox to open
1679  * @entries: Number of entries in the inbound mailbox ring
1680  */
1681 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
1682                                 int mbox, int entries)
1683 {
1684         struct tsi721_device *priv = mport->priv;
1685         int ch = mbox + 4;
1686         int i;
1687         u64 *free_ptr;
1688         int rc = 0;
1689
1690         if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
1691             (entries > TSI721_IMSGD_RING_SIZE) ||
1692             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1693                 rc = -EINVAL;
1694                 goto out;
1695         }
1696
1697         /* Initialize IB Messaging Ring */
1698         priv->imsg_ring[mbox].dev_id = dev_id;
1699         priv->imsg_ring[mbox].size = entries;
1700         priv->imsg_ring[mbox].rx_slot = 0;
1701         priv->imsg_ring[mbox].desc_rdptr = 0;
1702         priv->imsg_ring[mbox].fq_wrptr = 0;
1703         for (i = 0; i < priv->imsg_ring[mbox].size; i++)
1704                 priv->imsg_ring[mbox].imq_base[i] = NULL;
1705         spin_lock_init(&priv->imsg_ring[mbox].lock);
1706
1707         /* Allocate buffers for incoming messages */
1708         priv->imsg_ring[mbox].buf_base =
1709                 dma_alloc_coherent(&priv->pdev->dev,
1710                                    entries * TSI721_MSG_BUFFER_SIZE,
1711                                    &priv->imsg_ring[mbox].buf_phys,
1712                                    GFP_KERNEL);
1713
1714         if (priv->imsg_ring[mbox].buf_base == NULL) {
1715                 dev_err(&priv->pdev->dev,
1716                         "Failed to allocate buffers for IB MBOX%d\n", mbox);
1717                 rc = -ENOMEM;
1718                 goto out;
1719         }
1720
1721         /* Allocate memory for circular free list */
1722         priv->imsg_ring[mbox].imfq_base =
1723                 dma_alloc_coherent(&priv->pdev->dev,
1724                                    entries * 8,
1725                                    &priv->imsg_ring[mbox].imfq_phys,
1726                                    GFP_KERNEL);
1727
1728         if (priv->imsg_ring[mbox].imfq_base == NULL) {
1729                 dev_err(&priv->pdev->dev,
1730                         "Failed to allocate free queue for IB MBOX%d\n", mbox);
1731                 rc = -ENOMEM;
1732                 goto out_buf;
1733         }
1734
1735         /* Allocate memory for Inbound message descriptors */
1736         priv->imsg_ring[mbox].imd_base =
1737                 dma_alloc_coherent(&priv->pdev->dev,
1738                                    entries * sizeof(struct tsi721_imsg_desc),
1739                                    &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
1740
1741         if (priv->imsg_ring[mbox].imd_base == NULL) {
1742                 dev_err(&priv->pdev->dev,
1743                         "Failed to allocate descriptor memory for IB MBOX%d\n",
1744                         mbox);
1745                 rc = -ENOMEM;
1746                 goto out_dma;
1747         }
1748
1749         /* Fill free buffer pointer list */
1750         free_ptr = priv->imsg_ring[mbox].imfq_base;
1751         for (i = 0; i < entries; i++)
1752                 free_ptr[i] = cpu_to_le64(
1753                                 (u64)(priv->imsg_ring[mbox].buf_phys) +
1754                                 i * 0x1000);
1755
1756         mb();
1757
1758         /*
1759          * For mapping of inbound SRIO Messages into appropriate queues we need
1760          * to set Inbound Device ID register in the messaging engine. We do it
1761          * once when first inbound mailbox is requested.
1762          */
1763         if (!(priv->flags & TSI721_IMSGID_SET)) {
1764                 iowrite32((u32)priv->mport->host_deviceid,
1765                         priv->regs + TSI721_IB_DEVID);
1766                 priv->flags |= TSI721_IMSGID_SET;
1767         }
1768
1769         /*
1770          * Configure Inbound Messaging channel (ch = mbox + 4)
1771          */
1772
1773         /* Setup Inbound Message free queue */
1774         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
1775                 priv->regs + TSI721_IBDMAC_FQBH(ch));
1776         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
1777                         TSI721_IBDMAC_FQBL_MASK),
1778                 priv->regs+TSI721_IBDMAC_FQBL(ch));
1779         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1780                 priv->regs + TSI721_IBDMAC_FQSZ(ch));
1781
1782         /* Setup Inbound Message descriptor queue */
1783         iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
1784                 priv->regs + TSI721_IBDMAC_DQBH(ch));
1785         iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
1786                    (u32)TSI721_IBDMAC_DQBL_MASK),
1787                 priv->regs+TSI721_IBDMAC_DQBL(ch));
1788         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1789                 priv->regs + TSI721_IBDMAC_DQSZ(ch));
1790
1791         /* Enable interrupts */
1792
1793 #ifdef CONFIG_PCI_MSI
1794         if (priv->flags & TSI721_USING_MSIX) {
1795                 /* Request interrupt service if we are in MSI-X mode */
1796                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1797                         tsi721_imsg_msix, 0,
1798                         priv->msix[TSI721_VECT_IMB0_RCV + mbox].irq_name,
1799                         (void *)mport);
1800
1801                 if (rc) {
1802                         dev_dbg(&priv->pdev->dev,
1803                                 "Unable to allocate MSI-X interrupt for "
1804                                 "IBOX%d-DONE\n", mbox);
1805                         goto out_desc;
1806                 }
1807
1808                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1809                         tsi721_imsg_msix, 0,
1810                         priv->msix[TSI721_VECT_IMB0_INT + mbox].irq_name,
1811                         (void *)mport);
1812
1813                 if (rc) {
1814                         dev_dbg(&priv->pdev->dev,
1815                                 "Unable to allocate MSI-X interrupt for "
1816                                 "IBOX%d-INT\n", mbox);
1817                         free_irq(
1818                                 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1819                                 (void *)mport);
1820                         goto out_desc;
1821                 }
1822         }
1823 #endif /* CONFIG_PCI_MSI */
1824
1825         tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
1826
1827         /* Initialize Inbound Message Engine */
1828         iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
1829         ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
1830         udelay(10);
1831         priv->imsg_ring[mbox].fq_wrptr = entries - 1;
1832         iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
1833
1834         priv->imsg_init[mbox] = 1;
1835         return 0;
1836
1837 #ifdef CONFIG_PCI_MSI
1838 out_desc:
1839         dma_free_coherent(&priv->pdev->dev,
1840                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1841                 priv->imsg_ring[mbox].imd_base,
1842                 priv->imsg_ring[mbox].imd_phys);
1843
1844         priv->imsg_ring[mbox].imd_base = NULL;
1845 #endif /* CONFIG_PCI_MSI */
1846
1847 out_dma:
1848         dma_free_coherent(&priv->pdev->dev,
1849                 priv->imsg_ring[mbox].size * 8,
1850                 priv->imsg_ring[mbox].imfq_base,
1851                 priv->imsg_ring[mbox].imfq_phys);
1852
1853         priv->imsg_ring[mbox].imfq_base = NULL;
1854
1855 out_buf:
1856         dma_free_coherent(&priv->pdev->dev,
1857                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1858                 priv->imsg_ring[mbox].buf_base,
1859                 priv->imsg_ring[mbox].buf_phys);
1860
1861         priv->imsg_ring[mbox].buf_base = NULL;
1862
1863 out:
1864         return rc;
1865 }
1866
1867 /**
1868  * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
1869  * @mport: Master port implementing the Inbound Messaging Engine
1870  * @mbox: Mailbox to close
1871  */
1872 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
1873 {
1874         struct tsi721_device *priv = mport->priv;
1875         u32 rx_slot;
1876         int ch = mbox + 4;
1877
1878         if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
1879                 return;
1880         priv->imsg_init[mbox] = 0;
1881
1882         /* Disable Inbound Messaging Engine */
1883
1884         /* Disable Interrupts */
1885         tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
1886
1887 #ifdef CONFIG_PCI_MSI
1888         if (priv->flags & TSI721_USING_MSIX) {
1889                 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1890                                 (void *)mport);
1891                 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1892                                 (void *)mport);
1893         }
1894 #endif /* CONFIG_PCI_MSI */
1895
1896         /* Clear Inbound Buffer Queue */
1897         for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
1898                 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
1899
1900         /* Free memory allocated for message buffers */
1901         dma_free_coherent(&priv->pdev->dev,
1902                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1903                 priv->imsg_ring[mbox].buf_base,
1904                 priv->imsg_ring[mbox].buf_phys);
1905
1906         priv->imsg_ring[mbox].buf_base = NULL;
1907
1908         /* Free memory allocated for free pointr list */
1909         dma_free_coherent(&priv->pdev->dev,
1910                 priv->imsg_ring[mbox].size * 8,
1911                 priv->imsg_ring[mbox].imfq_base,
1912                 priv->imsg_ring[mbox].imfq_phys);
1913
1914         priv->imsg_ring[mbox].imfq_base = NULL;
1915
1916         /* Free memory allocated for RX descriptors */
1917         dma_free_coherent(&priv->pdev->dev,
1918                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1919                 priv->imsg_ring[mbox].imd_base,
1920                 priv->imsg_ring[mbox].imd_phys);
1921
1922         priv->imsg_ring[mbox].imd_base = NULL;
1923 }
1924
1925 /**
1926  * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
1927  * @mport: Master port implementing the Inbound Messaging Engine
1928  * @mbox: Inbound mailbox number
1929  * @buf: Buffer to add to inbound queue
1930  */
1931 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
1932 {
1933         struct tsi721_device *priv = mport->priv;
1934         u32 rx_slot;
1935         int rc = 0;
1936
1937         rx_slot = priv->imsg_ring[mbox].rx_slot;
1938         if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
1939                 dev_err(&priv->pdev->dev,
1940                         "Error adding inbound buffer %d, buffer exists\n",
1941                         rx_slot);
1942                 rc = -EINVAL;
1943                 goto out;
1944         }
1945
1946         priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
1947
1948         if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
1949                 priv->imsg_ring[mbox].rx_slot = 0;
1950
1951 out:
1952         return rc;
1953 }
1954
1955 /**
1956  * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
1957  * @mport: Master port implementing the Inbound Messaging Engine
1958  * @mbox: Inbound mailbox number
1959  *
1960  * Returns pointer to the message on success or NULL on failure.
1961  */
1962 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
1963 {
1964         struct tsi721_device *priv = mport->priv;
1965         struct tsi721_imsg_desc *desc;
1966         u32 rx_slot;
1967         void *rx_virt = NULL;
1968         u64 rx_phys;
1969         void *buf = NULL;
1970         u64 *free_ptr;
1971         int ch = mbox + 4;
1972         int msg_size;
1973
1974         if (!priv->imsg_init[mbox])
1975                 return NULL;
1976
1977         desc = priv->imsg_ring[mbox].imd_base;
1978         desc += priv->imsg_ring[mbox].desc_rdptr;
1979
1980         if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
1981                 goto out;
1982
1983         rx_slot = priv->imsg_ring[mbox].rx_slot;
1984         while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
1985                 if (++rx_slot == priv->imsg_ring[mbox].size)
1986                         rx_slot = 0;
1987         }
1988
1989         rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
1990                         le32_to_cpu(desc->bufptr_lo);
1991
1992         rx_virt = priv->imsg_ring[mbox].buf_base +
1993                   (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
1994
1995         buf = priv->imsg_ring[mbox].imq_base[rx_slot];
1996         msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
1997         if (msg_size == 0)
1998                 msg_size = RIO_MAX_MSG_SIZE;
1999
2000         memcpy(buf, rx_virt, msg_size);
2001         priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2002
2003         desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2004         if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2005                 priv->imsg_ring[mbox].desc_rdptr = 0;
2006
2007         iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2008                 priv->regs + TSI721_IBDMAC_DQRP(ch));
2009
2010         /* Return free buffer into the pointer list */
2011         free_ptr = priv->imsg_ring[mbox].imfq_base;
2012         free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2013
2014         if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2015                 priv->imsg_ring[mbox].fq_wrptr = 0;
2016
2017         iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2018                 priv->regs + TSI721_IBDMAC_FQWP(ch));
2019 out:
2020         return buf;
2021 }
2022
2023 /**
2024  * tsi721_messages_init - Initialization of Messaging Engine
2025  * @priv: pointer to tsi721 private data
2026  *
2027  * Configures Tsi721 messaging engine.
2028  */
2029 static int tsi721_messages_init(struct tsi721_device *priv)
2030 {
2031         int     ch;
2032
2033         iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2034         iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2035         iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2036
2037         /* Set SRIO Message Request/Response Timeout */
2038         iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2039
2040         /* Initialize Inbound Messaging Engine Registers */
2041         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2042                 /* Clear interrupt bits */
2043                 iowrite32(TSI721_IBDMAC_INT_MASK,
2044                         priv->regs + TSI721_IBDMAC_INT(ch));
2045                 /* Clear Status */
2046                 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2047
2048                 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2049                                 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2050                 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2051                                 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2052         }
2053
2054         return 0;
2055 }
2056
2057 /**
2058  * tsi721_disable_ints - disables all device interrupts
2059  * @priv: pointer to tsi721 private data
2060  */
2061 static void tsi721_disable_ints(struct tsi721_device *priv)
2062 {
2063         int ch;
2064
2065         /* Disable all device level interrupts */
2066         iowrite32(0, priv->regs + TSI721_DEV_INTE);
2067
2068         /* Disable all Device Channel interrupts */
2069         iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2070
2071         /* Disable all Inbound Msg Channel interrupts */
2072         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2073                 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2074
2075         /* Disable all Outbound Msg Channel interrupts */
2076         for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2077                 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2078
2079         /* Disable all general messaging interrupts */
2080         iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2081
2082         /* Disable all BDMA Channel interrupts */
2083         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2084                 iowrite32(0,
2085                         priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
2086
2087         /* Disable all general BDMA interrupts */
2088         iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2089
2090         /* Disable all SRIO Channel interrupts */
2091         for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2092                 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2093
2094         /* Disable all general SR2PC interrupts */
2095         iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2096
2097         /* Disable all PC2SR interrupts */
2098         iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2099
2100         /* Disable all I2C interrupts */
2101         iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2102
2103         /* Disable SRIO MAC interrupts */
2104         iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2105         iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2106 }
2107
2108 /**
2109  * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2110  * @priv: pointer to tsi721 private data
2111  *
2112  * Configures Tsi721 as RapidIO master port.
2113  */
2114 static int __devinit tsi721_setup_mport(struct tsi721_device *priv)
2115 {
2116         struct pci_dev *pdev = priv->pdev;
2117         int err = 0;
2118         struct rio_ops *ops;
2119
2120         struct rio_mport *mport;
2121
2122         ops = kzalloc(sizeof(struct rio_ops), GFP_KERNEL);
2123         if (!ops) {
2124                 dev_dbg(&pdev->dev, "Unable to allocate memory for rio_ops\n");
2125                 return -ENOMEM;
2126         }
2127
2128         ops->lcread = tsi721_lcread;
2129         ops->lcwrite = tsi721_lcwrite;
2130         ops->cread = tsi721_cread_dma;
2131         ops->cwrite = tsi721_cwrite_dma;
2132         ops->dsend = tsi721_dsend;
2133         ops->open_inb_mbox = tsi721_open_inb_mbox;
2134         ops->close_inb_mbox = tsi721_close_inb_mbox;
2135         ops->open_outb_mbox = tsi721_open_outb_mbox;
2136         ops->close_outb_mbox = tsi721_close_outb_mbox;
2137         ops->add_outb_message = tsi721_add_outb_message;
2138         ops->add_inb_buffer = tsi721_add_inb_buffer;
2139         ops->get_inb_message = tsi721_get_inb_message;
2140
2141         mport = kzalloc(sizeof(struct rio_mport), GFP_KERNEL);
2142         if (!mport) {
2143                 kfree(ops);
2144                 dev_dbg(&pdev->dev, "Unable to allocate memory for mport\n");
2145                 return -ENOMEM;
2146         }
2147
2148         mport->ops = ops;
2149         mport->index = 0;
2150         mport->sys_size = 0; /* small system */
2151         mport->phy_type = RIO_PHY_SERIAL;
2152         mport->priv = (void *)priv;
2153         mport->phys_efptr = 0x100;
2154         priv->mport = mport;
2155
2156         INIT_LIST_HEAD(&mport->dbells);
2157
2158         rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2159         rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2160         rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2161         strcpy(mport->name, "Tsi721 mport");
2162
2163         /* Hook up interrupt handler */
2164
2165 #ifdef CONFIG_PCI_MSI
2166         if (!tsi721_enable_msix(priv))
2167                 priv->flags |= TSI721_USING_MSIX;
2168         else if (!pci_enable_msi(pdev))
2169                 priv->flags |= TSI721_USING_MSI;
2170         else
2171                 dev_info(&pdev->dev,
2172                          "MSI/MSI-X is not available. Using legacy INTx.\n");
2173 #endif /* CONFIG_PCI_MSI */
2174
2175         err = tsi721_request_irq(mport);
2176
2177         if (!err) {
2178                 tsi721_interrupts_init(priv);
2179                 ops->pwenable = tsi721_pw_enable;
2180         } else {
2181                 dev_err(&pdev->dev, "Unable to get assigned PCI IRQ "
2182                         "vector %02X err=0x%x\n", pdev->irq, err);
2183                 goto err_exit;
2184         }
2185
2186 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2187         tsi721_register_dma(priv);
2188 #endif
2189         /* Enable SRIO link */
2190         iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2191                   TSI721_DEVCTL_SRBOOT_CMPL,
2192                   priv->regs + TSI721_DEVCTL);
2193
2194         rio_register_mport(mport);
2195
2196         if (mport->host_deviceid >= 0)
2197                 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2198                           RIO_PORT_GEN_DISCOVERED,
2199                           priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2200         else
2201                 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2202
2203         return 0;
2204
2205 err_exit:
2206         kfree(mport);
2207         kfree(ops);
2208         return err;
2209 }
2210
2211 static int __devinit tsi721_probe(struct pci_dev *pdev,
2212                                   const struct pci_device_id *id)
2213 {
2214         struct tsi721_device *priv;
2215         int i, cap;
2216         int err;
2217         u32 regval;
2218
2219         priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2220         if (priv == NULL) {
2221                 dev_err(&pdev->dev, "Failed to allocate memory for device\n");
2222                 err = -ENOMEM;
2223                 goto err_exit;
2224         }
2225
2226         err = pci_enable_device(pdev);
2227         if (err) {
2228                 dev_err(&pdev->dev, "Failed to enable PCI device\n");
2229                 goto err_clean;
2230         }
2231
2232         priv->pdev = pdev;
2233
2234 #ifdef DEBUG
2235         for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2236                 dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n",
2237                         i, (unsigned long long)pci_resource_start(pdev, i),
2238                         (unsigned long)pci_resource_len(pdev, i),
2239                         pci_resource_flags(pdev, i));
2240         }
2241 #endif
2242         /*
2243          * Verify BAR configuration
2244          */
2245
2246         /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2247         if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2248             pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2249             pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2250                 dev_err(&pdev->dev,
2251                         "Missing or misconfigured CSR BAR0, aborting.\n");
2252                 err = -ENODEV;
2253                 goto err_disable_pdev;
2254         }
2255
2256         /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2257         if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2258             pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2259             pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2260                 dev_err(&pdev->dev,
2261                         "Missing or misconfigured Doorbell BAR1, aborting.\n");
2262                 err = -ENODEV;
2263                 goto err_disable_pdev;
2264         }
2265
2266         /*
2267          * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2268          * space.
2269          * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2270          * It may be a good idea to keep them disabled using HW configuration
2271          * to save PCI memory space.
2272          */
2273         if ((pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM) &&
2274             (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64)) {
2275                 dev_info(&pdev->dev, "Outbound BAR2 is not used but enabled.\n");
2276         }
2277
2278         if ((pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM) &&
2279             (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64)) {
2280                 dev_info(&pdev->dev, "Outbound BAR4 is not used but enabled.\n");
2281         }
2282
2283         err = pci_request_regions(pdev, DRV_NAME);
2284         if (err) {
2285                 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
2286                         "aborting.\n");
2287                 goto err_disable_pdev;
2288         }
2289
2290         pci_set_master(pdev);
2291
2292         priv->regs = pci_ioremap_bar(pdev, BAR_0);
2293         if (!priv->regs) {
2294                 dev_err(&pdev->dev,
2295                         "Unable to map device registers space, aborting\n");
2296                 err = -ENOMEM;
2297                 goto err_free_res;
2298         }
2299
2300         priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2301         if (!priv->odb_base) {
2302                 dev_err(&pdev->dev,
2303                         "Unable to map outbound doorbells space, aborting\n");
2304                 err = -ENOMEM;
2305                 goto err_unmap_bars;
2306         }
2307
2308         /* Configure DMA attributes. */
2309         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2310                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
2311                         dev_info(&pdev->dev, "Unable to set DMA mask\n");
2312                         goto err_unmap_bars;
2313                 }
2314
2315                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2316                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2317         } else {
2318                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2319                 if (err)
2320                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2321         }
2322
2323         cap = pci_pcie_cap(pdev);
2324         BUG_ON(cap == 0);
2325
2326         /* Clear "no snoop" and "relaxed ordering" bits, use default MRRS. */
2327         pci_read_config_dword(pdev, cap + PCI_EXP_DEVCTL, &regval);
2328         regval &= ~(PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_RELAX_EN |
2329                     PCI_EXP_DEVCTL_NOSNOOP_EN);
2330         regval |= 0x2 << MAX_READ_REQUEST_SZ_SHIFT;
2331         pci_write_config_dword(pdev, cap + PCI_EXP_DEVCTL, regval);
2332
2333         /* Adjust PCIe completion timeout. */
2334         pci_read_config_dword(pdev, cap + PCI_EXP_DEVCTL2, &regval);
2335         regval &= ~(0x0f);
2336         pci_write_config_dword(pdev, cap + PCI_EXP_DEVCTL2, regval | 0x2);
2337
2338         /*
2339          * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2340          */
2341         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2342         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2343                                                 TSI721_MSIXTBL_OFFSET);
2344         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2345                                                 TSI721_MSIXPBA_OFFSET);
2346         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2347         /* End of FIXUP */
2348
2349         tsi721_disable_ints(priv);
2350
2351         tsi721_init_pc2sr_mapping(priv);
2352         tsi721_init_sr2pc_mapping(priv);
2353
2354         if (tsi721_bdma_maint_init(priv)) {
2355                 dev_err(&pdev->dev, "BDMA initialization failed, aborting\n");
2356                 err = -ENOMEM;
2357                 goto err_unmap_bars;
2358         }
2359
2360         err = tsi721_doorbell_init(priv);
2361         if (err)
2362                 goto err_free_bdma;
2363
2364         tsi721_port_write_init(priv);
2365
2366         err = tsi721_messages_init(priv);
2367         if (err)
2368                 goto err_free_consistent;
2369
2370         err = tsi721_setup_mport(priv);
2371         if (err)
2372                 goto err_free_consistent;
2373
2374         return 0;
2375
2376 err_free_consistent:
2377         tsi721_doorbell_free(priv);
2378 err_free_bdma:
2379         tsi721_bdma_maint_free(priv);
2380 err_unmap_bars:
2381         if (priv->regs)
2382                 iounmap(priv->regs);
2383         if (priv->odb_base)
2384                 iounmap(priv->odb_base);
2385 err_free_res:
2386         pci_release_regions(pdev);
2387         pci_clear_master(pdev);
2388 err_disable_pdev:
2389         pci_disable_device(pdev);
2390 err_clean:
2391         kfree(priv);
2392 err_exit:
2393         return err;
2394 }
2395
2396 static DEFINE_PCI_DEVICE_TABLE(tsi721_pci_tbl) = {
2397         { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2398         { 0, }  /* terminate list */
2399 };
2400
2401 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2402
2403 static struct pci_driver tsi721_driver = {
2404         .name           = "tsi721",
2405         .id_table       = tsi721_pci_tbl,
2406         .probe          = tsi721_probe,
2407 };
2408
2409 static int __init tsi721_init(void)
2410 {
2411         return pci_register_driver(&tsi721_driver);
2412 }
2413
2414 static void __exit tsi721_exit(void)
2415 {
2416         pci_unregister_driver(&tsi721_driver);
2417 }
2418
2419 device_initcall(tsi721_init);