drivers/dma/edma.c

   1 /*
   2  * TI EDMA DMA engine driver
   3  *
   4  * Copyright 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/err.h>
  19 #include <linux/init.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/list.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/slab.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/of.h>
  27
  28 #include <linux/platform_data/edma.h>
  29
  30 #include "dmaengine.h"
  31 #include "virt-dma.h"
  32
  33 /*
  34  * This will go away when the private EDMA API is folded
  35  * into this driver and the platform device(s) are
  36  * instantiated in the arch code. We can only get away
  37  * with this simplification because DA8XX may not be built
  38  * in the same kernel image with other DaVinci parts. This
  39  * avoids having to sprinkle dmaengine driver platform devices
  40  * and data throughout all the existing board files.
  41  */
  42 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
  43 #define EDMA_CTLRS      2
  44 #define EDMA_CHANS      32
  45 #else
  46 #define EDMA_CTLRS      1
  47 #define EDMA_CHANS      64
  48 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
  49
  50 /*
  51  * Max of 20 segments per channel to conserve PaRAM slots
  52  * Also note that MAX_NR_SG should be atleast the no.of periods
  53  * that are required for ASoC, otherwise DMA prep calls will
  54  * fail. Today davinci-pcm is the only user of this driver and
  55  * requires atleast 17 slots, so we setup the default to 20.
  56  */
  57 #define MAX_NR_SG               20
  58 #define EDMA_MAX_SLOTS          MAX_NR_SG
  59 #define EDMA_DESCRIPTORS        16
  60
  61 struct edma_pset {
  62         u32                             len;
  63         dma_addr_t                      addr;
  64         struct edmacc_param             param;
  65 };
  66
  67 struct edma_desc {
  68         struct virt_dma_desc            vdesc;
  69         struct list_head                node;
  70         enum dma_transfer_direction     direction;
  71         int                             cyclic;
  72         int                             absync;
  73         int                             pset_nr;
  74         struct edma_chan                *echan;
  75         int                             processed;
  76
  77         /*
  78          * The following 4 elements are used for residue accounting.
  79          *
  80          * - processed_stat: the number of SG elements we have traversed
  81          * so far to cover accounting. This is updated directly to processed
  82          * during edma_callback and is always <= processed, because processed
  83          * refers to the number of pending transfer (programmed to EDMA
  84          * controller), where as processed_stat tracks number of transfers
  85          * accounted for so far.
  86          *
  87          * - residue: The amount of bytes we have left to transfer for this desc
  88          *
  89          * - residue_stat: The residue in bytes of data we have covered
  90          * so far for accounting. This is updated directly to residue
  91          * during callbacks to keep it current.
  92          *
  93          * - sg_len: Tracks the length of the current intermediate transfer,
  94          * this is required to update the residue during intermediate transfer
  95          * completion callback.
  96          */
  97         int                             processed_stat;
  98         u32                             sg_len;
  99         u32                             residue;
 100         u32                             residue_stat;
 101
 102         struct edma_pset                pset[0];
 103 };
 104
 105 struct edma_cc;
 106
 107 struct edma_chan {
 108         struct virt_dma_chan            vchan;
 109         struct list_head                node;
 110         struct edma_desc                *edesc;
 111         struct edma_cc                  *ecc;
 112         int                             ch_num;
 113         bool                            alloced;
 114         int                             slot[EDMA_MAX_SLOTS];
 115         int                             missed;
 116         struct dma_slave_config         cfg;
 117 };
 118
 119 struct edma_cc {
 120         int                             ctlr;
 121         struct dma_device               dma_slave;
 122         struct edma_chan                slave_chans[EDMA_CHANS];
 123         int                             num_slave_chans;
 124         int                             dummy_slot;
 125 };
 126
 127 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 128 {
 129         return container_of(d, struct edma_cc, dma_slave);
 130 }
 131
 132 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 133 {
 134         return container_of(c, struct edma_chan, vchan.chan);
 135 }
 136
 137 static inline struct edma_desc
 138 *to_edma_desc(struct dma_async_tx_descriptor *tx)
 139 {
 140         return container_of(tx, struct edma_desc, vdesc.tx);
 141 }
 142
 143 static void edma_desc_free(struct virt_dma_desc *vdesc)
 144 {
 145         kfree(container_of(vdesc, struct edma_desc, vdesc));
 146 }
 147
 148 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 149 static void edma_execute(struct edma_chan *echan)
 150 {
 151         struct virt_dma_desc *vdesc;
 152         struct edma_desc *edesc;
 153         struct device *dev = echan->vchan.chan.device->dev;
 154         int i, j, left, nslots;
 155
 156         /* If either we processed all psets or we're still not started */
 157         if (!echan->edesc ||
 158             echan->edesc->pset_nr == echan->edesc->processed) {
 159                 /* Get next vdesc */
 160                 vdesc = vchan_next_desc(&echan->vchan);
 161                 if (!vdesc) {
 162                         echan->edesc = NULL;
 163                         return;
 164                 }
 165                 list_del(&vdesc->node);
 166                 echan->edesc = to_edma_desc(&vdesc->tx);
 167         }
 168
 169         edesc = echan->edesc;
 170
 171         /* Find out how many left */
 172         left = edesc->pset_nr - edesc->processed;
 173         nslots = min(MAX_NR_SG, left);
 174         edesc->sg_len = 0;
 175
 176         /* Write descriptor PaRAM set(s) */
 177         for (i = 0; i < nslots; i++) {
 178                 j = i + edesc->processed;
 179                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
 180                 edesc->sg_len += edesc->pset[j].len;
 181                 dev_vdbg(echan->vchan.chan.device->dev,
 182                         "\n pset[%d]:\n"
 183                         "  chnum\t%d\n"
 184                         "  slot\t%d\n"
 185                         "  opt\t%08x\n"
 186                         "  src\t%08x\n"
 187                         "  dst\t%08x\n"
 188                         "  abcnt\t%08x\n"
 189                         "  ccnt\t%08x\n"
 190                         "  bidx\t%08x\n"
 191                         "  cidx\t%08x\n"
 192                         "  lkrld\t%08x\n",
 193                         j, echan->ch_num, echan->slot[i],
 194                         edesc->pset[j].param.opt,
 195                         edesc->pset[j].param.src,
 196                         edesc->pset[j].param.dst,
 197                         edesc->pset[j].param.a_b_cnt,
 198                         edesc->pset[j].param.ccnt,
 199                         edesc->pset[j].param.src_dst_bidx,
 200                         edesc->pset[j].param.src_dst_cidx,
 201                         edesc->pset[j].param.link_bcntrld);
 202                 /* Link to the previous slot if not the last set */
 203                 if (i != (nslots - 1))
 204                         edma_link(echan->slot[i], echan->slot[i+1]);
 205         }
 206
 207         edesc->processed += nslots;
 208
 209         /*
 210          * If this is either the last set in a set of SG-list transactions
 211          * then setup a link to the dummy slot, this results in all future
 212          * events being absorbed and that's OK because we're done
 213          */
 214         if (edesc->processed == edesc->pset_nr) {
 215                 if (edesc->cyclic)
 216                         edma_link(echan->slot[nslots-1], echan->slot[1]);
 217                 else
 218                         edma_link(echan->slot[nslots-1],
 219                                   echan->ecc->dummy_slot);
 220         }
 221
 222         if (edesc->processed <= MAX_NR_SG) {
 223                 dev_dbg(dev, "first transfer starting on channel %d\n",
 224                         echan->ch_num);
 225                 edma_start(echan->ch_num);
 226         } else {
 227                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 228                         echan->ch_num, edesc->processed);
 229                 edma_resume(echan->ch_num);
 230         }
 231
 232         /*
 233          * This happens due to setup times between intermediate transfers
 234          * in long SG lists which have to be broken up into transfers of
 235          * MAX_NR_SG
 236          */
 237         if (echan->missed) {
 238                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 239                 edma_clean_channel(echan->ch_num);
 240                 edma_stop(echan->ch_num);
 241                 edma_start(echan->ch_num);
 242                 edma_trigger_channel(echan->ch_num);
 243                 echan->missed = 0;
 244         }
 245 }
 246
 247 static int edma_terminate_all(struct edma_chan *echan)
 248 {
 249         unsigned long flags;
 250         LIST_HEAD(head);
 251
 252         spin_lock_irqsave(&echan->vchan.lock, flags);
 253
 254         /*
 255          * Stop DMA activity: we assume the callback will not be called
 256          * after edma_dma() returns (even if it does, it will see
 257          * echan->edesc is NULL and exit.)
 258          */
 259         if (echan->edesc) {
 260                 int cyclic = echan->edesc->cyclic;
 261                 echan->edesc = NULL;
 262                 edma_stop(echan->ch_num);
 263                 /* Move the cyclic channel back to default queue */
 264                 if (cyclic)
 265                         edma_assign_channel_eventq(echan->ch_num,
 266                                                    EVENTQ_DEFAULT);
 267         }
 268
 269         vchan_get_all_descriptors(&echan->vchan, &head);
 270         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 271         vchan_dma_desc_free_list(&echan->vchan, &head);
 272
 273         return 0;
 274 }
 275
 276 static int edma_slave_config(struct edma_chan *echan,
 277         struct dma_slave_config *cfg)
 278 {
 279         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 280             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 281                 return -EINVAL;
 282
 283         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 284
 285         return 0;
 286 }
 287
 288 static int edma_dma_pause(struct edma_chan *echan)
 289 {
 290         /* Pause/Resume only allowed with cyclic mode */
 291         if (!echan->edesc || !echan->edesc->cyclic)
 292                 return -EINVAL;
 293
 294         edma_pause(echan->ch_num);
 295         return 0;
 296 }
 297
 298 static int edma_dma_resume(struct edma_chan *echan)
 299 {
 300         /* Pause/Resume only allowed with cyclic mode */
 301         if (!echan->edesc->cyclic)
 302                 return -EINVAL;
 303
 304         edma_resume(echan->ch_num);
 305         return 0;
 306 }
 307
 308 static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 309                         unsigned long arg)
 310 {
 311         int ret = 0;
 312         struct dma_slave_config *config;
 313         struct edma_chan *echan = to_edma_chan(chan);
 314
 315         switch (cmd) {
 316         case DMA_TERMINATE_ALL:
 317                 edma_terminate_all(echan);
 318                 break;
 319         case DMA_SLAVE_CONFIG:
 320                 config = (struct dma_slave_config *)arg;
 321                 ret = edma_slave_config(echan, config);
 322                 break;
 323         case DMA_PAUSE:
 324                 ret = edma_dma_pause(echan);
 325                 break;
 326
 327         case DMA_RESUME:
 328                 ret = edma_dma_resume(echan);
 329                 break;
 330
 331         default:
 332                 ret = -ENOSYS;
 333         }
 334
 335         return ret;
 336 }
 337
 338 /*
 339  * A PaRAM set configuration abstraction used by other modes
 340  * @chan: Channel who's PaRAM set we're configuring
 341  * @pset: PaRAM set to initialize and setup.
 342  * @src_addr: Source address of the DMA
 343  * @dst_addr: Destination address of the DMA
 344  * @burst: In units of dev_width, how much to send
 345  * @dev_width: How much is the dev_width
 346  * @dma_length: Total length of the DMA transfer
 347  * @direction: Direction of the transfer
 348  */
 349 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 350         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 351         enum dma_slave_buswidth dev_width, unsigned int dma_length,
 352         enum dma_transfer_direction direction)
 353 {
 354         struct edma_chan *echan = to_edma_chan(chan);
 355         struct device *dev = chan->device->dev;
 356         struct edmacc_param *param = &epset->param;
 357         int acnt, bcnt, ccnt, cidx;
 358         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 359         int absync;
 360
 361         acnt = dev_width;
 362
 363         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 364         if (!burst)
 365                 burst = 1;
 366         /*
 367          * If the maxburst is equal to the fifo width, use
 368          * A-synced transfers. This allows for large contiguous
 369          * buffer transfers using only one PaRAM set.
 370          */
 371         if (burst == 1) {
 372                 /*
 373                  * For the A-sync case, bcnt and ccnt are the remainder
 374                  * and quotient respectively of the division of:
 375                  * (dma_length / acnt) by (SZ_64K -1). This is so
 376                  * that in case bcnt over flows, we have ccnt to use.
 377                  * Note: In A-sync tranfer only, bcntrld is used, but it
 378                  * only applies for sg_dma_len(sg) >= SZ_64K.
 379                  * In this case, the best way adopted is- bccnt for the
 380                  * first frame will be the remainder below. Then for
 381                  * every successive frame, bcnt will be SZ_64K-1. This
 382                  * is assured as bcntrld = 0xffff in end of function.
 383                  */
 384                 absync = false;
 385                 ccnt = dma_length / acnt / (SZ_64K - 1);
 386                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 387                 /*
 388                  * If bcnt is non-zero, we have a remainder and hence an
 389                  * extra frame to transfer, so increment ccnt.
 390                  */
 391                 if (bcnt)
 392                         ccnt++;
 393                 else
 394                         bcnt = SZ_64K - 1;
 395                 cidx = acnt;
 396         } else {
 397                 /*
 398                  * If maxburst is greater than the fifo address_width,
 399                  * use AB-synced transfers where A count is the fifo
 400                  * address_width and B count is the maxburst. In this
 401                  * case, we are limited to transfers of C count frames
 402                  * of (address_width * maxburst) where C count is limited
 403                  * to SZ_64K-1. This places an upper bound on the length
 404                  * of an SG segment that can be handled.
 405                  */
 406                 absync = true;
 407                 bcnt = burst;
 408                 ccnt = dma_length / (acnt * bcnt);
 409                 if (ccnt > (SZ_64K - 1)) {
 410                         dev_err(dev, "Exceeded max SG segment size\n");
 411                         return -EINVAL;
 412                 }
 413                 cidx = acnt * bcnt;
 414         }
 415
 416         epset->len = dma_length;
 417
 418         if (direction == DMA_MEM_TO_DEV) {
 419                 src_bidx = acnt;
 420                 src_cidx = cidx;
 421                 dst_bidx = 0;
 422                 dst_cidx = 0;
 423                 epset->addr = src_addr;
 424         } else if (direction == DMA_DEV_TO_MEM)  {
 425                 src_bidx = 0;
 426                 src_cidx = 0;
 427                 dst_bidx = acnt;
 428                 dst_cidx = cidx;
 429                 epset->addr = dst_addr;
 430         } else if (direction == DMA_MEM_TO_MEM)  {
 431                 src_bidx = acnt;
 432                 src_cidx = cidx;
 433                 dst_bidx = acnt;
 434                 dst_cidx = cidx;
 435         } else {
 436                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
 437                 return -EINVAL;
 438         }
 439
 440         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
 441         /* Configure A or AB synchronized transfers */
 442         if (absync)
 443                 param->opt |= SYNCDIM;
 444
 445         param->src = src_addr;
 446         param->dst = dst_addr;
 447
 448         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
 449         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
 450
 451         param->a_b_cnt = bcnt << 16 | acnt;
 452         param->ccnt = ccnt;
 453         /*
 454          * Only time when (bcntrld) auto reload is required is for
 455          * A-sync case, and in this case, a requirement of reload value
 456          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
 457          * and then later will be populated by edma_execute.
 458          */
 459         param->link_bcntrld = 0xffffffff;
 460         return absync;
 461 }
 462
 463 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
 464         struct dma_chan *chan, struct scatterlist *sgl,
 465         unsigned int sg_len, enum dma_transfer_direction direction,
 466         unsigned long tx_flags, void *context)
 467 {
 468         struct edma_chan *echan = to_edma_chan(chan);
 469         struct device *dev = chan->device->dev;
 470         struct edma_desc *edesc;
 471         dma_addr_t src_addr = 0, dst_addr = 0;
 472         enum dma_slave_buswidth dev_width;
 473         u32 burst;
 474         struct scatterlist *sg;
 475         int i, nslots, ret;
 476
 477         if (unlikely(!echan || !sgl || !sg_len))
 478                 return NULL;
 479
 480         if (direction == DMA_DEV_TO_MEM) {
 481                 src_addr = echan->cfg.src_addr;
 482                 dev_width = echan->cfg.src_addr_width;
 483                 burst = echan->cfg.src_maxburst;
 484         } else if (direction == DMA_MEM_TO_DEV) {
 485                 dst_addr = echan->cfg.dst_addr;
 486                 dev_width = echan->cfg.dst_addr_width;
 487                 burst = echan->cfg.dst_maxburst;
 488         } else {
 489                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 490                 return NULL;
 491         }
 492
 493         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 494                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 495                 return NULL;
 496         }
 497
 498         edesc = kzalloc(sizeof(*edesc) + sg_len *
 499                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 500         if (!edesc) {
 501                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 502                 return NULL;
 503         }
 504
 505         edesc->pset_nr = sg_len;
 506         edesc->residue = 0;
 507         edesc->direction = direction;
 508         edesc->echan = echan;
 509
 510         /* Allocate a PaRAM slot, if needed */
 511         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
 512
 513         for (i = 0; i < nslots; i++) {
 514                 if (echan->slot[i] < 0) {
 515                         echan->slot[i] =
 516                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 517                                                 EDMA_SLOT_ANY);
 518                         if (echan->slot[i] < 0) {
 519                                 kfree(edesc);
 520                                 dev_err(dev, "%s: Failed to allocate slot\n",
 521                                         __func__);
 522                                 return NULL;
 523                         }
 524                 }
 525         }
 526
 527         /* Configure PaRAM sets for each SG */
 528         for_each_sg(sgl, sg, sg_len, i) {
 529                 /* Get address for each SG */
 530                 if (direction == DMA_DEV_TO_MEM)
 531                         dst_addr = sg_dma_address(sg);
 532                 else
 533                         src_addr = sg_dma_address(sg);
 534
 535                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 536                                        dst_addr, burst, dev_width,
 537                                        sg_dma_len(sg), direction);
 538                 if (ret < 0) {
 539                         kfree(edesc);
 540                         return NULL;
 541                 }
 542
 543                 edesc->absync = ret;
 544                 edesc->residue += sg_dma_len(sg);
 545
 546                 /* If this is the last in a current SG set of transactions,
 547                    enable interrupts so that next set is processed */
 548                 if (!((i+1) % MAX_NR_SG))
 549                         edesc->pset[i].param.opt |= TCINTEN;
 550
 551                 /* If this is the last set, enable completion interrupt flag */
 552                 if (i == sg_len - 1)
 553                         edesc->pset[i].param.opt |= TCINTEN;
 554         }
 555         edesc->residue_stat = edesc->residue;
 556
 557         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 558 }
 559
 560 struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
 561         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 562         size_t len, unsigned long tx_flags)
 563 {
 564         int ret;
 565         struct edma_desc *edesc;
 566         struct device *dev = chan->device->dev;
 567         struct edma_chan *echan = to_edma_chan(chan);
 568
 569         if (unlikely(!echan || !len))
 570                 return NULL;
 571
 572         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
 573         if (!edesc) {
 574                 dev_dbg(dev, "Failed to allocate a descriptor\n");
 575                 return NULL;
 576         }
 577
 578         edesc->pset_nr = 1;
 579
 580         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
 581                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
 582         if (ret < 0)
 583                 return NULL;
 584
 585         edesc->absync = ret;
 586
 587         /*
 588          * Enable intermediate transfer chaining to re-trigger channel
 589          * on completion of every TR, and enable transfer-completion
 590          * interrupt on completion of the whole transfer.
 591          */
 592         edesc->pset[0].param.opt |= ITCCHEN;
 593         edesc->pset[0].param.opt |= TCINTEN;
 594
 595         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 596 }
 597
 598 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
 599         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 600         size_t period_len, enum dma_transfer_direction direction,
 601         unsigned long tx_flags)
 602 {
 603         struct edma_chan *echan = to_edma_chan(chan);
 604         struct device *dev = chan->device->dev;
 605         struct edma_desc *edesc;
 606         dma_addr_t src_addr, dst_addr;
 607         enum dma_slave_buswidth dev_width;
 608         u32 burst;
 609         int i, ret, nslots;
 610
 611         if (unlikely(!echan || !buf_len || !period_len))
 612                 return NULL;
 613
 614         if (direction == DMA_DEV_TO_MEM) {
 615                 src_addr = echan->cfg.src_addr;
 616                 dst_addr = buf_addr;
 617                 dev_width = echan->cfg.src_addr_width;
 618                 burst = echan->cfg.src_maxburst;
 619         } else if (direction == DMA_MEM_TO_DEV) {
 620                 src_addr = buf_addr;
 621                 dst_addr = echan->cfg.dst_addr;
 622                 dev_width = echan->cfg.dst_addr_width;
 623                 burst = echan->cfg.dst_maxburst;
 624         } else {
 625                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 626                 return NULL;
 627         }
 628
 629         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 630                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 631                 return NULL;
 632         }
 633
 634         if (unlikely(buf_len % period_len)) {
 635                 dev_err(dev, "Period should be multiple of Buffer length\n");
 636                 return NULL;
 637         }
 638
 639         nslots = (buf_len / period_len) + 1;
 640
 641         /*
 642          * Cyclic DMA users such as audio cannot tolerate delays introduced
 643          * by cases where the number of periods is more than the maximum
 644          * number of SGs the EDMA driver can handle at a time. For DMA types
 645          * such as Slave SGs, such delays are tolerable and synchronized,
 646          * but the synchronization is difficult to achieve with Cyclic and
 647          * cannot be guaranteed, so we error out early.
 648          */
 649         if (nslots > MAX_NR_SG)
 650                 return NULL;
 651
 652         edesc = kzalloc(sizeof(*edesc) + nslots *
 653                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 654         if (!edesc) {
 655                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 656                 return NULL;
 657         }
 658
 659         edesc->cyclic = 1;
 660         edesc->pset_nr = nslots;
 661         edesc->residue = edesc->residue_stat = buf_len;
 662         edesc->direction = direction;
 663         edesc->echan = echan;
 664
 665         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
 666                 __func__, echan->ch_num, nslots, period_len, buf_len);
 667
 668         for (i = 0; i < nslots; i++) {
 669                 /* Allocate a PaRAM slot, if needed */
 670                 if (echan->slot[i] < 0) {
 671                         echan->slot[i] =
 672                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 673                                                 EDMA_SLOT_ANY);
 674                         if (echan->slot[i] < 0) {
 675                                 kfree(edesc);
 676                                 dev_err(dev, "%s: Failed to allocate slot\n",
 677                                         __func__);
 678                                 return NULL;
 679                         }
 680                 }
 681
 682                 if (i == nslots - 1) {
 683                         memcpy(&edesc->pset[i], &edesc->pset[0],
 684                                sizeof(edesc->pset[0]));
 685                         break;
 686                 }
 687
 688                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 689                                        dst_addr, burst, dev_width, period_len,
 690                                        direction);
 691                 if (ret < 0) {
 692                         kfree(edesc);
 693                         return NULL;
 694                 }
 695
 696                 if (direction == DMA_DEV_TO_MEM)
 697                         dst_addr += period_len;
 698                 else
 699                         src_addr += period_len;
 700
 701                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
 702                 dev_vdbg(dev,
 703                         "\n pset[%d]:\n"
 704                         "  chnum\t%d\n"
 705                         "  slot\t%d\n"
 706                         "  opt\t%08x\n"
 707                         "  src\t%08x\n"
 708                         "  dst\t%08x\n"
 709                         "  abcnt\t%08x\n"
 710                         "  ccnt\t%08x\n"
 711                         "  bidx\t%08x\n"
 712                         "  cidx\t%08x\n"
 713                         "  lkrld\t%08x\n",
 714                         i, echan->ch_num, echan->slot[i],
 715                         edesc->pset[i].param.opt,
 716                         edesc->pset[i].param.src,
 717                         edesc->pset[i].param.dst,
 718                         edesc->pset[i].param.a_b_cnt,
 719                         edesc->pset[i].param.ccnt,
 720                         edesc->pset[i].param.src_dst_bidx,
 721                         edesc->pset[i].param.src_dst_cidx,
 722                         edesc->pset[i].param.link_bcntrld);
 723
 724                 edesc->absync = ret;
 725
 726                 /*
 727                  * Enable period interrupt only if it is requested
 728                  */
 729                 if (tx_flags & DMA_PREP_INTERRUPT)
 730                         edesc->pset[i].param.opt |= TCINTEN;
 731         }
 732
 733         /* Place the cyclic channel to highest priority queue */
 734         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
 735
 736         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 737 }
 738
 739 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
 740 {
 741         struct edma_chan *echan = data;
 742         struct device *dev = echan->vchan.chan.device->dev;
 743         struct edma_desc *edesc;
 744         struct edmacc_param p;
 745
 746         edesc = echan->edesc;
 747
 748         /* Pause the channel for non-cyclic */
 749         if (!edesc || (edesc && !edesc->cyclic))
 750                 edma_pause(echan->ch_num);
 751
 752         switch (ch_status) {
 753         case EDMA_DMA_COMPLETE:
 754                 spin_lock(&echan->vchan.lock);
 755
 756                 if (edesc) {
 757                         if (edesc->cyclic) {
 758                                 vchan_cyclic_callback(&edesc->vdesc);
 759                         } else if (edesc->processed == edesc->pset_nr) {
 760                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
 761                                 edesc->residue = 0;
 762                                 edma_stop(echan->ch_num);
 763                                 vchan_cookie_complete(&edesc->vdesc);
 764                                 edma_execute(echan);
 765                         } else {
 766                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
 767
 768                                 /* Update statistics for tx_status */
 769                                 edesc->residue -= edesc->sg_len;
 770                                 edesc->residue_stat = edesc->residue;
 771                                 edesc->processed_stat = edesc->processed;
 772
 773                                 edma_execute(echan);
 774                         }
 775                 }
 776
 777                 spin_unlock(&echan->vchan.lock);
 778
 779                 break;
 780         case EDMA_DMA_CC_ERROR:
 781                 spin_lock(&echan->vchan.lock);
 782
 783                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
 784
 785                 /*
 786                  * Issue later based on missed flag which will be sure
 787                  * to happen as:
 788                  * (1) we finished transmitting an intermediate slot and
 789                  *     edma_execute is coming up.
 790                  * (2) or we finished current transfer and issue will
 791                  *     call edma_execute.
 792                  *
 793                  * Important note: issuing can be dangerous here and
 794                  * lead to some nasty recursion when we are in a NULL
 795                  * slot. So we avoid doing so and set the missed flag.
 796                  */
 797                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
 798                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
 799                         echan->missed = 1;
 800                 } else {
 801                         /*
 802                          * The slot is already programmed but the event got
 803                          * missed, so its safe to issue it here.
 804                          */
 805                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
 806                         edma_clean_channel(echan->ch_num);
 807                         edma_stop(echan->ch_num);
 808                         edma_start(echan->ch_num);
 809                         edma_trigger_channel(echan->ch_num);
 810                 }
 811
 812                 spin_unlock(&echan->vchan.lock);
 813
 814                 break;
 815         default:
 816                 break;
 817         }
 818 }
 819
 820 /* Alloc channel resources */
 821 static int edma_alloc_chan_resources(struct dma_chan *chan)
 822 {
 823         struct edma_chan *echan = to_edma_chan(chan);
 824         struct device *dev = chan->device->dev;
 825         int ret;
 826         int a_ch_num;
 827         LIST_HEAD(descs);
 828
 829         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
 830                                         chan, EVENTQ_DEFAULT);
 831
 832         if (a_ch_num < 0) {
 833                 ret = -ENODEV;
 834                 goto err_no_chan;
 835         }
 836
 837         if (a_ch_num != echan->ch_num) {
 838                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
 839                         EDMA_CTLR(echan->ch_num),
 840                         EDMA_CHAN_SLOT(echan->ch_num));
 841                 ret = -ENODEV;
 842                 goto err_wrong_chan;
 843         }
 844
 845         echan->alloced = true;
 846         echan->slot[0] = echan->ch_num;
 847
 848         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
 849                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
 850
 851         return 0;
 852
 853 err_wrong_chan:
 854         edma_free_channel(a_ch_num);
 855 err_no_chan:
 856         return ret;
 857 }
 858
 859 /* Free channel resources */
 860 static void edma_free_chan_resources(struct dma_chan *chan)
 861 {
 862         struct edma_chan *echan = to_edma_chan(chan);
 863         struct device *dev = chan->device->dev;
 864         int i;
 865
 866         /* Terminate transfers */
 867         edma_stop(echan->ch_num);
 868
 869         vchan_free_chan_resources(&echan->vchan);
 870
 871         /* Free EDMA PaRAM slots */
 872         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
 873                 if (echan->slot[i] >= 0) {
 874                         edma_free_slot(echan->slot[i]);
 875                         echan->slot[i] = -1;
 876                 }
 877         }
 878
 879         /* Free EDMA channel */
 880         if (echan->alloced) {
 881                 edma_free_channel(echan->ch_num);
 882                 echan->alloced = false;
 883         }
 884
 885         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
 886 }
 887
 888 /* Send pending descriptor to hardware */
 889 static void edma_issue_pending(struct dma_chan *chan)
 890 {
 891         struct edma_chan *echan = to_edma_chan(chan);
 892         unsigned long flags;
 893
 894         spin_lock_irqsave(&echan->vchan.lock, flags);
 895         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
 896                 edma_execute(echan);
 897         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 898 }
 899
 900 static u32 edma_residue(struct edma_desc *edesc)
 901 {
 902         bool dst = edesc->direction == DMA_DEV_TO_MEM;
 903         struct edma_pset *pset = edesc->pset;
 904         dma_addr_t done, pos;
 905         int i;
 906
 907         /*
 908          * We always read the dst/src position from the first RamPar
 909          * pset. That's the one which is active now.
 910          */
 911         pos = edma_get_position(edesc->echan->slot[0], dst);
 912
 913         /*
 914          * Cyclic is simple. Just subtract pset[0].addr from pos.
 915          *
 916          * We never update edesc->residue in the cyclic case, so we
 917          * can tell the remaining room to the end of the circular
 918          * buffer.
 919          */
 920         if (edesc->cyclic) {
 921                 done = pos - pset->addr;
 922                 edesc->residue_stat = edesc->residue - done;
 923                 return edesc->residue_stat;
 924         }
 925
 926         /*
 927          * For SG operation we catch up with the last processed
 928          * status.
 929          */
 930         pset += edesc->processed_stat;
 931
 932         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
 933                 /*
 934                  * If we are inside this pset address range, we know
 935                  * this is the active one. Get the current delta and
 936                  * stop walking the psets.
 937                  */
 938                 if (pos >= pset->addr && pos < pset->addr + pset->len)
 939                         return edesc->residue_stat - (pos - pset->addr);
 940
 941                 /* Otherwise mark it done and update residue_stat. */
 942                 edesc->processed_stat++;
 943                 edesc->residue_stat -= pset->len;
 944         }
 945         return edesc->residue_stat;
 946 }
 947
 948 /* Check request completion status */
 949 static enum dma_status edma_tx_status(struct dma_chan *chan,
 950                                       dma_cookie_t cookie,
 951                                       struct dma_tx_state *txstate)
 952 {
 953         struct edma_chan *echan = to_edma_chan(chan);
 954         struct virt_dma_desc *vdesc;
 955         enum dma_status ret;
 956         unsigned long flags;
 957
 958         ret = dma_cookie_status(chan, cookie, txstate);
 959         if (ret == DMA_COMPLETE || !txstate)
 960                 return ret;
 961
 962         spin_lock_irqsave(&echan->vchan.lock, flags);
 963         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
 964                 txstate->residue = edma_residue(echan->edesc);
 965         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
 966                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
 967         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 968
 969         return ret;
 970 }
 971
 972 static void __init edma_chan_init(struct edma_cc *ecc,
 973                                   struct dma_device *dma,
 974                                   struct edma_chan *echans)
 975 {
 976         int i, j;
 977
 978         for (i = 0; i < EDMA_CHANS; i++) {
 979                 struct edma_chan *echan = &echans[i];
 980                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
 981                 echan->ecc = ecc;
 982                 echan->vchan.desc_free = edma_desc_free;
 983
 984                 vchan_init(&echan->vchan, dma);
 985
 986                 INIT_LIST_HEAD(&echan->node);
 987                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
 988                         echan->slot[j] = -1;
 989         }
 990 }
 991
 992 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
 993                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
 994                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
 995                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
 996
 997 static int edma_dma_device_slave_caps(struct dma_chan *dchan,
 998                                       struct dma_slave_caps *caps)
 999 {
1000         caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1001         caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
1002         caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1003         caps->cmd_pause = true;
1004         caps->cmd_terminate = true;
1005         caps->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1006
1007         return 0;
1008 }
1009
1010 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1011                           struct device *dev)
1012 {
1013         dma->device_prep_slave_sg = edma_prep_slave_sg;
1014         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1015         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1016         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1017         dma->device_free_chan_resources = edma_free_chan_resources;
1018         dma->device_issue_pending = edma_issue_pending;
1019         dma->device_tx_status = edma_tx_status;
1020         dma->device_control = edma_control;
1021         dma->device_slave_caps = edma_dma_device_slave_caps;
1022         dma->dev = dev;
1023
1024         /*
1025          * code using dma memcpy must make sure alignment of
1026          * length is at dma->copy_align boundary.
1027          */
1028         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1029
1030         INIT_LIST_HEAD(&dma->channels);
1031 }
1032
1033 static int edma_probe(struct platform_device *pdev)
1034 {
1035         struct edma_cc *ecc;
1036         int ret;
1037
1038         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1039         if (ret)
1040                 return ret;
1041
1042         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1043         if (!ecc) {
1044                 dev_err(&pdev->dev, "Can't allocate controller\n");
1045                 return -ENOMEM;
1046         }
1047
1048         ecc->ctlr = pdev->id;
1049         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1050         if (ecc->dummy_slot < 0) {
1051                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1052                 return ecc->dummy_slot;
1053         }
1054
1055         dma_cap_zero(ecc->dma_slave.cap_mask);
1056         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1057         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1058         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1059
1060         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1061
1062         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1063
1064         ret = dma_async_device_register(&ecc->dma_slave);
1065         if (ret)
1066                 goto err_reg1;
1067
1068         platform_set_drvdata(pdev, ecc);
1069
1070         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1071
1072         return 0;
1073
1074 err_reg1:
1075         edma_free_slot(ecc->dummy_slot);
1076         return ret;
1077 }
1078
1079 static int edma_remove(struct platform_device *pdev)
1080 {
1081         struct device *dev = &pdev->dev;
1082         struct edma_cc *ecc = dev_get_drvdata(dev);
1083
1084         dma_async_device_unregister(&ecc->dma_slave);
1085         edma_free_slot(ecc->dummy_slot);
1086
1087         return 0;
1088 }
1089
1090 static struct platform_driver edma_driver = {
1091         .probe          = edma_probe,
1092         .remove         = edma_remove,
1093         .driver = {
1094                 .name = "edma-dma-engine",
1095                 .owner = THIS_MODULE,
1096         },
1097 };
1098
1099 bool edma_filter_fn(struct dma_chan *chan, void *param)
1100 {
1101         if (chan->device->dev->driver == &edma_driver.driver) {
1102                 struct edma_chan *echan = to_edma_chan(chan);
1103                 unsigned ch_req = *(unsigned *)param;
1104                 return ch_req == echan->ch_num;
1105         }
1106         return false;
1107 }
1108 EXPORT_SYMBOL(edma_filter_fn);
1109
1110 static struct platform_device *pdev0, *pdev1;
1111
1112 static const struct platform_device_info edma_dev_info0 = {
1113         .name = "edma-dma-engine",
1114         .id = 0,
1115         .dma_mask = DMA_BIT_MASK(32),
1116 };
1117
1118 static const struct platform_device_info edma_dev_info1 = {
1119         .name = "edma-dma-engine",
1120         .id = 1,
1121         .dma_mask = DMA_BIT_MASK(32),
1122 };
1123
1124 static int edma_init(void)
1125 {
1126         int ret = platform_driver_register(&edma_driver);
1127
1128         if (ret == 0) {
1129                 pdev0 = platform_device_register_full(&edma_dev_info0);
1130                 if (IS_ERR(pdev0)) {
1131                         platform_driver_unregister(&edma_driver);
1132                         ret = PTR_ERR(pdev0);
1133                         goto out;
1134                 }
1135         }
1136
1137         if (!of_have_populated_dt() && EDMA_CTLRS == 2) {
1138                 pdev1 = platform_device_register_full(&edma_dev_info1);
1139                 if (IS_ERR(pdev1)) {
1140                         platform_driver_unregister(&edma_driver);
1141                         platform_device_unregister(pdev0);
1142                         ret = PTR_ERR(pdev1);
1143                 }
1144         }
1145
1146 out:
1147         return ret;
1148 }
1149 subsys_initcall(edma_init);
1150
1151 static void __exit edma_exit(void)
1152 {
1153         platform_device_unregister(pdev0);
1154         if (pdev1)
1155                 platform_device_unregister(pdev1);
1156         platform_driver_unregister(&edma_driver);
1157 }
1158 module_exit(edma_exit);
1159
1160 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1161 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1162 MODULE_LICENSE("GPL v2");