4 * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
6 * based on amba-pl08x.c
8 * Copyright (c) 2006 ARM Ltd.
9 * Copyright (c) 2010 ST-Ericsson SA
11 * Author: Peter Pearse <peter.pearse@arm.com>
12 * Author: Linus Walleij <linus.walleij@stericsson.com>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
19 * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals
20 * that can be routed to any of the 4 to 8 hardware-channels.
22 * Therefore on these DMA controllers the number of channels
23 * and the number of incoming DMA signals are two totally different things.
24 * It is usually not possible to theoretically handle all physical signals,
25 * so a multiplexing scheme with possible denial of use is necessary.
31 #include <linux/platform_device.h>
32 #include <linux/types.h>
33 #include <linux/dmaengine.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/interrupt.h>
36 #include <linux/clk.h>
37 #include <linux/module.h>
38 #include <linux/slab.h>
39 #include <linux/platform_data/dma-s3c24xx.h>
41 #include "dmaengine.h"
44 #define MAX_DMA_CHANNELS 8
46 #define S3C24XX_DISRC 0x00
47 #define S3C24XX_DISRCC 0x04
48 #define S3C24XX_DISRCC_INC_INCREMENT 0
49 #define S3C24XX_DISRCC_INC_FIXED BIT(0)
50 #define S3C24XX_DISRCC_LOC_AHB 0
51 #define S3C24XX_DISRCC_LOC_APB BIT(1)
53 #define S3C24XX_DIDST 0x08
54 #define S3C24XX_DIDSTC 0x0c
55 #define S3C24XX_DIDSTC_INC_INCREMENT 0
56 #define S3C24XX_DIDSTC_INC_FIXED BIT(0)
57 #define S3C24XX_DIDSTC_LOC_AHB 0
58 #define S3C24XX_DIDSTC_LOC_APB BIT(1)
59 #define S3C24XX_DIDSTC_INT_TC0 0
60 #define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
62 #define S3C24XX_DCON 0x10
64 #define S3C24XX_DCON_TC_MASK 0xfffff
65 #define S3C24XX_DCON_DSZ_BYTE (0 << 20)
66 #define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
67 #define S3C24XX_DCON_DSZ_WORD (2 << 20)
68 #define S3C24XX_DCON_DSZ_MASK (3 << 20)
69 #define S3C24XX_DCON_DSZ_SHIFT 20
70 #define S3C24XX_DCON_AUTORELOAD 0
71 #define S3C24XX_DCON_NORELOAD BIT(22)
72 #define S3C24XX_DCON_HWTRIG BIT(23)
73 #define S3C24XX_DCON_HWSRC_SHIFT 24
74 #define S3C24XX_DCON_SERV_SINGLE 0
75 #define S3C24XX_DCON_SERV_WHOLE BIT(27)
76 #define S3C24XX_DCON_TSZ_UNIT 0
77 #define S3C24XX_DCON_TSZ_BURST4 BIT(28)
78 #define S3C24XX_DCON_INT BIT(29)
79 #define S3C24XX_DCON_SYNC_PCLK 0
80 #define S3C24XX_DCON_SYNC_HCLK BIT(30)
81 #define S3C24XX_DCON_DEMAND 0
82 #define S3C24XX_DCON_HANDSHAKE BIT(31)
84 #define S3C24XX_DSTAT 0x14
85 #define S3C24XX_DSTAT_STAT_BUSY BIT(20)
86 #define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
88 #define S3C24XX_DMASKTRIG 0x20
89 #define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
90 #define S3C24XX_DMASKTRIG_ON BIT(1)
91 #define S3C24XX_DMASKTRIG_STOP BIT(2)
93 #define S3C24XX_DMAREQSEL 0x24
94 #define S3C24XX_DMAREQSEL_HW BIT(0)
97 * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel
98 * for a DMA source. Instead only specific channels are valid.
99 * All of these SoCs have 4 physical channels and the number of request
100 * source bits is 3. Additionally we also need 1 bit to mark the channel
102 * Therefore we separate the chansel element of the channel data into 4
103 * parts of 4 bits each, to hold the information if the channel is valid
104 * and the hw request source to use.
107 * SDI is valid on channels 0, 2 and 3 - with varying hw request sources.
108 * For it the chansel field would look like
110 * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1
111 * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2
112 * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2
114 #define S3C24XX_CHANSEL_WIDTH 4
115 #define S3C24XX_CHANSEL_VALID BIT(3)
116 #define S3C24XX_CHANSEL_REQ_MASK 7
119 * struct soc_data - vendor-specific config parameters for individual SoCs
120 * @stride: spacing between the registers of each channel
121 * @has_reqsel: does the controller use the newer requestselection mechanism
122 * @has_clocks: are controllable dma-clocks present
131 * enum s3c24xx_dma_chan_state - holds the virtual channel states
132 * @S3C24XX_DMA_CHAN_IDLE: the channel is idle
133 * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport
134 * channel and is running a transfer on it
135 * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport
136 * channel to become available (only pertains to memcpy channels)
138 enum s3c24xx_dma_chan_state {
139 S3C24XX_DMA_CHAN_IDLE,
140 S3C24XX_DMA_CHAN_RUNNING,
141 S3C24XX_DMA_CHAN_WAITING,
145 * struct s3c24xx_sg - structure containing data per sg
146 * @src_addr: src address of sg
147 * @dst_addr: dst address of sg
148 * @len: transfer len in bytes
149 * @node: node for txd's dsg_list
155 struct list_head node;
159 * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor
160 * @vd: virtual DMA descriptor
161 * @dsg_list: list of children sg's
162 * @at: sg currently being transfered
163 * @width: transfer width
164 * @disrcc: value for source control register
165 * @didstc: value for destination control register
166 * @dcon: base value for dcon register
167 * @cyclic: indicate cyclic transfer
170 struct virt_dma_desc vd;
171 struct list_head dsg_list;
172 struct list_head *at;
180 struct s3c24xx_dma_chan;
183 * struct s3c24xx_dma_phy - holder for the physical channels
184 * @id: physical index to this channel
185 * @valid: does the channel have all required elements
186 * @base: virtual memory base (remapped) for the this channel
187 * @irq: interrupt for this channel
188 * @clk: clock for this channel
189 * @lock: a lock to use when altering an instance of this struct
190 * @serving: virtual channel currently being served by this physicalchannel
191 * @host: a pointer to the host (internal use)
193 struct s3c24xx_dma_phy {
200 struct s3c24xx_dma_chan *serving;
201 struct s3c24xx_dma_engine *host;
205 * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel
206 * @id: the id of the channel
207 * @name: name of the channel
208 * @vc: wrappped virtual channel
209 * @phy: the physical channel utilized by this channel, if there is one
210 * @runtime_addr: address for RX/TX according to the runtime config
211 * @at: active transaction on this channel
212 * @lock: a lock for this channel data
213 * @host: a pointer to the host (internal use)
214 * @state: whether the channel is idle, running etc
215 * @slave: whether this channel is a device (slave) or for memcpy
217 struct s3c24xx_dma_chan {
220 struct virt_dma_chan vc;
221 struct s3c24xx_dma_phy *phy;
222 struct dma_slave_config cfg;
223 struct s3c24xx_txd *at;
224 struct s3c24xx_dma_engine *host;
225 enum s3c24xx_dma_chan_state state;
230 * struct s3c24xx_dma_engine - the local state holder for the S3C24XX
231 * @pdev: the corresponding platform device
232 * @pdata: platform data passed in from the platform/machine
233 * @base: virtual memory base (remapped)
234 * @slave: slave engine for this instance
235 * @memcpy: memcpy engine for this instance
236 * @phy_chans: array of data for the physical channels
238 struct s3c24xx_dma_engine {
239 struct platform_device *pdev;
240 const struct s3c24xx_dma_platdata *pdata;
241 struct soc_data *sdata;
243 struct dma_device slave;
244 struct dma_device memcpy;
245 struct s3c24xx_dma_phy *phy_chans;
249 * Physical channel handling
253 * Check whether a certain channel is busy or not.
255 static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
257 unsigned int val = readl(phy->base + S3C24XX_DSTAT);
258 return val & S3C24XX_DSTAT_STAT_BUSY;
261 static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
262 struct s3c24xx_dma_phy *phy)
264 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
265 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
266 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
269 /* every phy is valid for memcopy channels */
273 /* On newer variants all phys can be used for all virtual channels */
274 if (s3cdma->sdata->has_reqsel)
277 phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
278 return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
282 * Allocate a physical channel for a virtual channel
284 * Try to locate a physical channel to be used for this transfer. If all
285 * are taken return NULL and the requester will have to cope by using
286 * some fallback PIO mode or retrying later.
289 struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
291 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
292 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
293 struct s3c24xx_dma_channel *cdata;
294 struct s3c24xx_dma_phy *phy = NULL;
300 cdata = &pdata->channels[s3cchan->id];
302 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
303 phy = &s3cdma->phy_chans[i];
308 if (!s3c24xx_dma_phy_valid(s3cchan, phy))
311 spin_lock_irqsave(&phy->lock, flags);
314 phy->serving = s3cchan;
315 spin_unlock_irqrestore(&phy->lock, flags);
319 spin_unlock_irqrestore(&phy->lock, flags);
322 /* No physical channel available, cope with it */
323 if (i == s3cdma->pdata->num_phy_channels) {
324 dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
328 /* start the phy clock */
329 if (s3cdma->sdata->has_clocks) {
330 ret = clk_enable(phy->clk);
332 dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
343 * Mark the physical channel as free.
345 * This drops the link between the physical and virtual channel.
347 static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
349 struct s3c24xx_dma_engine *s3cdma = phy->host;
351 if (s3cdma->sdata->has_clocks)
352 clk_disable(phy->clk);
358 * Stops the channel by writing the stop bit.
359 * This should not be used for an on-going transfer, but as a method of
360 * shutting down a channel (eg, when it's no longer used) or terminating a
363 static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
365 writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
369 * Virtual channel handling
373 struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
375 return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
378 static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
380 struct s3c24xx_dma_phy *phy = s3cchan->phy;
381 struct s3c24xx_txd *txd = s3cchan->at;
382 u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
384 return tc * txd->width;
387 static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
388 struct dma_slave_config *config)
390 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
394 /* Reject definitely invalid configurations */
395 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
396 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
399 spin_lock_irqsave(&s3cchan->vc.lock, flags);
401 if (!s3cchan->slave) {
406 s3cchan->cfg = *config;
409 spin_lock_irqrestore(&s3cchan->vc.lock, flags);
418 struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
420 return container_of(tx, struct s3c24xx_txd, vd.tx);
423 static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
425 struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
428 INIT_LIST_HEAD(&txd->dsg_list);
429 txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
435 static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
437 struct s3c24xx_sg *dsg, *_dsg;
439 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
440 list_del(&dsg->node);
447 static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
448 struct s3c24xx_txd *txd)
450 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
451 struct s3c24xx_dma_phy *phy = s3cchan->phy;
452 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
453 struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
454 u32 dcon = txd->dcon;
457 /* transfer-size and -count from len and width */
458 switch (txd->width) {
460 dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
463 dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
466 dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
470 if (s3cchan->slave) {
471 struct s3c24xx_dma_channel *cdata =
472 &pdata->channels[s3cchan->id];
474 if (s3cdma->sdata->has_reqsel) {
475 writel_relaxed((cdata->chansel << 1) |
476 S3C24XX_DMAREQSEL_HW,
477 phy->base + S3C24XX_DMAREQSEL);
479 int csel = cdata->chansel >> (phy->id *
480 S3C24XX_CHANSEL_WIDTH);
482 csel &= S3C24XX_CHANSEL_REQ_MASK;
483 dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
484 dcon |= S3C24XX_DCON_HWTRIG;
487 if (s3cdma->sdata->has_reqsel)
488 writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
491 writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
492 writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
493 writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
494 writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
495 writel_relaxed(dcon, phy->base + S3C24XX_DCON);
497 val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
498 val &= ~S3C24XX_DMASKTRIG_STOP;
499 val |= S3C24XX_DMASKTRIG_ON;
501 /* trigger the dma operation for memcpy transfers */
503 val |= S3C24XX_DMASKTRIG_SWTRIG;
505 writel(val, phy->base + S3C24XX_DMASKTRIG);
509 * Set the initial DMA register values and start first sg.
511 static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
513 struct s3c24xx_dma_phy *phy = s3cchan->phy;
514 struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
515 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
517 list_del(&txd->vd.node);
521 /* Wait for channel inactive */
522 while (s3c24xx_dma_phy_busy(phy))
525 /* point to the first element of the sg list */
526 txd->at = txd->dsg_list.next;
527 s3c24xx_dma_start_next_sg(s3cchan, txd);
530 static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
531 struct s3c24xx_dma_chan *s3cchan)
535 vchan_get_all_descriptors(&s3cchan->vc, &head);
536 vchan_dma_desc_free_list(&s3cchan->vc, &head);
540 * Try to allocate a physical channel. When successful, assign it to
541 * this virtual channel, and initiate the next descriptor. The
542 * virtual channel lock must be held at this point.
544 static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
546 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
547 struct s3c24xx_dma_phy *phy;
549 phy = s3c24xx_dma_get_phy(s3cchan);
551 dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
553 s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
557 dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
558 phy->id, s3cchan->name);
561 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
563 s3c24xx_dma_start_next_txd(s3cchan);
566 static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
567 struct s3c24xx_dma_chan *s3cchan)
569 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
571 dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
572 phy->id, s3cchan->name);
575 * We do this without taking the lock; we're really only concerned
576 * about whether this pointer is NULL or not, and we're guaranteed
577 * that this will only be called when it _already_ is non-NULL.
579 phy->serving = s3cchan;
581 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
582 s3c24xx_dma_start_next_txd(s3cchan);
586 * Free a physical DMA channel, potentially reallocating it to another
587 * virtual channel if we have any pending.
589 static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
591 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
592 struct s3c24xx_dma_chan *p, *next;
597 /* Find a waiting virtual channel for the next transfer. */
598 list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
599 if (p->state == S3C24XX_DMA_CHAN_WAITING) {
605 list_for_each_entry(p, &s3cdma->slave.channels,
607 if (p->state == S3C24XX_DMA_CHAN_WAITING &&
608 s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
614 /* Ensure that the physical channel is stopped */
615 s3c24xx_dma_terminate_phy(s3cchan->phy);
621 * Eww. We know this isn't going to deadlock
622 * but lockdep probably doesn't.
624 spin_lock(&next->vc.lock);
625 /* Re-check the state now that we have the lock */
626 success = next->state == S3C24XX_DMA_CHAN_WAITING;
628 s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
629 spin_unlock(&next->vc.lock);
631 /* If the state changed, try to find another channel */
635 /* No more jobs, so free up the physical channel */
636 s3c24xx_dma_put_phy(s3cchan->phy);
640 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
643 static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
645 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
646 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
649 dma_descriptor_unmap(&vd->tx);
651 s3c24xx_dma_free_txd(txd);
654 static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
656 struct s3c24xx_dma_phy *phy = data;
657 struct s3c24xx_dma_chan *s3cchan = phy->serving;
658 struct s3c24xx_txd *txd;
660 dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
663 * Interrupts happen to notify the completion of a transfer and the
664 * channel should have moved into its stop state already on its own.
665 * Therefore interrupts on channels not bound to a virtual channel
666 * should never happen. Nevertheless send a terminate command to the
667 * channel if the unlikely case happens.
669 if (unlikely(!s3cchan)) {
670 dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
673 s3c24xx_dma_terminate_phy(phy);
678 spin_lock(&s3cchan->vc.lock);
681 /* when more sg's are in this txd, start the next one */
682 if (!list_is_last(txd->at, &txd->dsg_list)) {
683 txd->at = txd->at->next;
685 vchan_cyclic_callback(&txd->vd);
686 s3c24xx_dma_start_next_sg(s3cchan, txd);
687 } else if (!txd->cyclic) {
689 vchan_cookie_complete(&txd->vd);
692 * And start the next descriptor (if any),
693 * otherwise free this channel.
695 if (vchan_next_desc(&s3cchan->vc))
696 s3c24xx_dma_start_next_txd(s3cchan);
698 s3c24xx_dma_phy_free(s3cchan);
700 vchan_cyclic_callback(&txd->vd);
702 /* Cyclic: reset at beginning */
703 txd->at = txd->dsg_list.next;
704 s3c24xx_dma_start_next_sg(s3cchan, txd);
707 spin_unlock(&s3cchan->vc.lock);
716 static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
718 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
719 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
722 spin_lock_irqsave(&s3cchan->vc.lock, flags);
724 if (!s3cchan->phy && !s3cchan->at) {
725 dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
730 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
732 /* Mark physical channel as free */
734 s3c24xx_dma_phy_free(s3cchan);
736 /* Dequeue current job */
738 s3c24xx_dma_desc_free(&s3cchan->at->vd);
742 /* Dequeue jobs not yet fired as well */
743 s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
745 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
750 static int s3c24xx_dma_alloc_chan_resources(struct dma_chan *chan)
755 static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
757 /* Ensure all queued descriptors are freed */
758 vchan_free_chan_resources(to_virt_chan(chan));
761 static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
762 dma_cookie_t cookie, struct dma_tx_state *txstate)
764 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
765 struct s3c24xx_txd *txd;
766 struct s3c24xx_sg *dsg;
767 struct virt_dma_desc *vd;
772 spin_lock_irqsave(&s3cchan->vc.lock, flags);
773 ret = dma_cookie_status(chan, cookie, txstate);
774 if (ret == DMA_COMPLETE) {
775 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
780 * There's no point calculating the residue if there's
781 * no txstate to store the value.
784 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
788 vd = vchan_find_desc(&s3cchan->vc, cookie);
790 /* On the issued list, so hasn't been processed yet */
791 txd = to_s3c24xx_txd(&vd->tx);
793 list_for_each_entry(dsg, &txd->dsg_list, node)
797 * Currently running, so sum over the pending sg's and
798 * the currently active one.
802 dsg = list_entry(txd->at, struct s3c24xx_sg, node);
803 list_for_each_entry_from(dsg, &txd->dsg_list, node)
806 bytes += s3c24xx_dma_getbytes_chan(s3cchan);
808 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
811 * This cookie not complete yet
812 * Get number of bytes left in the active transactions and queue
814 dma_set_residue(txstate, bytes);
816 /* Whether waiting or running, we're in progress */
821 * Initialize a descriptor to be used by memcpy submit
823 static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
824 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
825 size_t len, unsigned long flags)
827 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
828 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
829 struct s3c24xx_txd *txd;
830 struct s3c24xx_sg *dsg;
831 int src_mod, dest_mod;
833 dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %d bytes from %s\n",
836 if ((len & S3C24XX_DCON_TC_MASK) != len) {
837 dev_err(&s3cdma->pdev->dev, "memcpy size %d to large\n", len);
841 txd = s3c24xx_dma_get_txd();
845 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
847 s3c24xx_dma_free_txd(txd);
850 list_add_tail(&dsg->node, &txd->dsg_list);
853 dsg->dst_addr = dest;
857 * Determine a suitable transfer width.
858 * The DMA controller cannot fetch/store information which is not
859 * naturally aligned on the bus, i.e., a 4 byte fetch must start at
860 * an address divisible by 4 - more generally addr % width must be 0.
866 txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
869 txd->width = ((src_mod == 2 || src_mod == 0) &&
870 (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
877 txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
878 txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
879 txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
880 S3C24XX_DCON_SERV_WHOLE;
882 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
885 static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
886 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
887 enum dma_transfer_direction direction, unsigned long flags)
889 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
890 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
891 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
892 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
893 struct s3c24xx_txd *txd;
894 struct s3c24xx_sg *dsg;
896 dma_addr_t slave_addr;
900 dev_dbg(&s3cdma->pdev->dev,
901 "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
902 size, period, s3cchan->name);
904 if (!is_slave_direction(direction)) {
905 dev_err(&s3cdma->pdev->dev,
906 "direction %d unsupported\n", direction);
910 txd = s3c24xx_dma_get_txd();
916 if (cdata->handshake)
917 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
919 switch (cdata->bus) {
920 case S3C24XX_DMA_APB:
921 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
922 hwcfg |= S3C24XX_DISRCC_LOC_APB;
924 case S3C24XX_DMA_AHB:
925 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
926 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
931 * Always assume our peripheral desintation is a fixed
934 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
937 * Individual dma operations are requested by the slave,
938 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
940 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
942 if (direction == DMA_MEM_TO_DEV) {
943 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
944 S3C24XX_DISRCC_INC_INCREMENT;
946 slave_addr = s3cchan->cfg.dst_addr;
947 txd->width = s3cchan->cfg.dst_addr_width;
950 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
951 S3C24XX_DIDSTC_INC_INCREMENT;
952 slave_addr = s3cchan->cfg.src_addr;
953 txd->width = s3cchan->cfg.src_addr_width;
956 sg_len = size / period;
958 for (i = 0; i < sg_len; i++) {
959 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
961 s3c24xx_dma_free_txd(txd);
964 list_add_tail(&dsg->node, &txd->dsg_list);
967 /* Check last period length */
969 dsg->len = size - period * i;
970 if (direction == DMA_MEM_TO_DEV) {
971 dsg->src_addr = addr + period * i;
972 dsg->dst_addr = slave_addr;
973 } else { /* DMA_DEV_TO_MEM */
974 dsg->src_addr = slave_addr;
975 dsg->dst_addr = addr + period * i;
979 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
982 static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
983 struct dma_chan *chan, struct scatterlist *sgl,
984 unsigned int sg_len, enum dma_transfer_direction direction,
985 unsigned long flags, void *context)
987 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
988 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
989 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
990 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
991 struct s3c24xx_txd *txd;
992 struct s3c24xx_sg *dsg;
993 struct scatterlist *sg;
994 dma_addr_t slave_addr;
998 dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
999 sg_dma_len(sgl), s3cchan->name);
1001 txd = s3c24xx_dma_get_txd();
1005 if (cdata->handshake)
1006 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
1008 switch (cdata->bus) {
1009 case S3C24XX_DMA_APB:
1010 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
1011 hwcfg |= S3C24XX_DISRCC_LOC_APB;
1013 case S3C24XX_DMA_AHB:
1014 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
1015 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
1020 * Always assume our peripheral desintation is a fixed
1021 * address in memory.
1023 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
1026 * Individual dma operations are requested by the slave,
1027 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
1029 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
1031 if (direction == DMA_MEM_TO_DEV) {
1032 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
1033 S3C24XX_DISRCC_INC_INCREMENT;
1034 txd->didstc = hwcfg;
1035 slave_addr = s3cchan->cfg.dst_addr;
1036 txd->width = s3cchan->cfg.dst_addr_width;
1037 } else if (direction == DMA_DEV_TO_MEM) {
1038 txd->disrcc = hwcfg;
1039 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
1040 S3C24XX_DIDSTC_INC_INCREMENT;
1041 slave_addr = s3cchan->cfg.src_addr;
1042 txd->width = s3cchan->cfg.src_addr_width;
1044 s3c24xx_dma_free_txd(txd);
1045 dev_err(&s3cdma->pdev->dev,
1046 "direction %d unsupported\n", direction);
1050 for_each_sg(sgl, sg, sg_len, tmp) {
1051 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
1053 s3c24xx_dma_free_txd(txd);
1056 list_add_tail(&dsg->node, &txd->dsg_list);
1058 dsg->len = sg_dma_len(sg);
1059 if (direction == DMA_MEM_TO_DEV) {
1060 dsg->src_addr = sg_dma_address(sg);
1061 dsg->dst_addr = slave_addr;
1062 } else { /* DMA_DEV_TO_MEM */
1063 dsg->src_addr = slave_addr;
1064 dsg->dst_addr = sg_dma_address(sg);
1068 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
1072 * Slave transactions callback to the slave device to allow
1073 * synchronization of slave DMA signals with the DMAC enable
1075 static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
1077 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
1078 unsigned long flags;
1080 spin_lock_irqsave(&s3cchan->vc.lock, flags);
1081 if (vchan_issue_pending(&s3cchan->vc)) {
1082 if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
1083 s3c24xx_dma_phy_alloc_and_start(s3cchan);
1085 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
1089 * Bringup and teardown
1093 * Initialise the DMAC memcpy/slave channels.
1094 * Make a local wrapper to hold required data
1096 static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
1097 struct dma_device *dmadev, unsigned int channels, bool slave)
1099 struct s3c24xx_dma_chan *chan;
1102 INIT_LIST_HEAD(&dmadev->channels);
1105 * Register as many many memcpy as we have physical channels,
1106 * we won't always be able to use all but the code will have
1107 * to cope with that situation.
1109 for (i = 0; i < channels; i++) {
1110 chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
1112 dev_err(dmadev->dev,
1113 "%s no memory for channel\n", __func__);
1118 chan->host = s3cdma;
1119 chan->state = S3C24XX_DMA_CHAN_IDLE;
1123 chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
1127 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1131 dev_dbg(dmadev->dev,
1132 "initialize virtual channel \"%s\"\n",
1135 chan->vc.desc_free = s3c24xx_dma_desc_free;
1136 vchan_init(&chan->vc, dmadev);
1138 dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
1139 i, slave ? "slave" : "memcpy");
1143 static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
1145 struct s3c24xx_dma_chan *chan = NULL;
1146 struct s3c24xx_dma_chan *next;
1148 list_for_each_entry_safe(chan,
1149 next, &dmadev->channels, vc.chan.device_node)
1150 list_del(&chan->vc.chan.device_node);
1153 /* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */
1154 static struct soc_data soc_s3c2410 = {
1156 .has_reqsel = false,
1157 .has_clocks = false,
1160 /* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */
1161 static struct soc_data soc_s3c2412 = {
1167 /* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */
1168 static struct soc_data soc_s3c2443 = {
1174 static struct platform_device_id s3c24xx_dma_driver_ids[] = {
1176 .name = "s3c2410-dma",
1177 .driver_data = (kernel_ulong_t)&soc_s3c2410,
1179 .name = "s3c2412-dma",
1180 .driver_data = (kernel_ulong_t)&soc_s3c2412,
1182 .name = "s3c2443-dma",
1183 .driver_data = (kernel_ulong_t)&soc_s3c2443,
1188 static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
1190 return (struct soc_data *)
1191 platform_get_device_id(pdev)->driver_data;
1194 static int s3c24xx_dma_probe(struct platform_device *pdev)
1196 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1197 struct s3c24xx_dma_engine *s3cdma;
1198 struct soc_data *sdata;
1199 struct resource *res;
1204 dev_err(&pdev->dev, "platform data missing\n");
1208 /* Basic sanity check */
1209 if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
1210 dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
1211 pdata->num_phy_channels, MAX_DMA_CHANNELS);
1215 sdata = s3c24xx_dma_get_soc_data(pdev);
1219 s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
1223 s3cdma->pdev = pdev;
1224 s3cdma->pdata = pdata;
1225 s3cdma->sdata = sdata;
1227 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1228 s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
1229 if (IS_ERR(s3cdma->base))
1230 return PTR_ERR(s3cdma->base);
1232 s3cdma->phy_chans = devm_kzalloc(&pdev->dev,
1233 sizeof(struct s3c24xx_dma_phy) *
1234 pdata->num_phy_channels,
1236 if (!s3cdma->phy_chans)
1239 /* aquire irqs and clocks for all physical channels */
1240 for (i = 0; i < pdata->num_phy_channels; i++) {
1241 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1245 phy->base = s3cdma->base + (i * sdata->stride);
1248 phy->irq = platform_get_irq(pdev, i);
1250 dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
1255 ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
1256 0, pdev->name, phy);
1258 dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
1263 if (sdata->has_clocks) {
1264 sprintf(clk_name, "dma.%d", i);
1265 phy->clk = devm_clk_get(&pdev->dev, clk_name);
1266 if (IS_ERR(phy->clk) && sdata->has_clocks) {
1267 dev_err(&pdev->dev, "unable to aquire clock for channel %d, error %lu",
1268 i, PTR_ERR(phy->clk));
1272 ret = clk_prepare(phy->clk);
1274 dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
1280 spin_lock_init(&phy->lock);
1283 dev_dbg(&pdev->dev, "physical channel %d is %s\n",
1284 i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
1287 /* Initialize memcpy engine */
1288 dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
1289 dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
1290 s3cdma->memcpy.dev = &pdev->dev;
1291 s3cdma->memcpy.device_alloc_chan_resources =
1292 s3c24xx_dma_alloc_chan_resources;
1293 s3cdma->memcpy.device_free_chan_resources =
1294 s3c24xx_dma_free_chan_resources;
1295 s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
1296 s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
1297 s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
1298 s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
1299 s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
1301 /* Initialize slave engine for SoC internal dedicated peripherals */
1302 dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
1303 dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
1304 dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
1305 s3cdma->slave.dev = &pdev->dev;
1306 s3cdma->slave.device_alloc_chan_resources =
1307 s3c24xx_dma_alloc_chan_resources;
1308 s3cdma->slave.device_free_chan_resources =
1309 s3c24xx_dma_free_chan_resources;
1310 s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
1311 s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
1312 s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
1313 s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
1314 s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
1315 s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
1317 /* Register as many memcpy channels as there are physical channels */
1318 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
1319 pdata->num_phy_channels, false);
1321 dev_warn(&pdev->dev,
1322 "%s failed to enumerate memcpy channels - %d\n",
1327 /* Register slave channels */
1328 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
1329 pdata->num_channels, true);
1331 dev_warn(&pdev->dev,
1332 "%s failed to enumerate slave channels - %d\n",
1337 ret = dma_async_device_register(&s3cdma->memcpy);
1339 dev_warn(&pdev->dev,
1340 "%s failed to register memcpy as an async device - %d\n",
1342 goto err_memcpy_reg;
1345 ret = dma_async_device_register(&s3cdma->slave);
1347 dev_warn(&pdev->dev,
1348 "%s failed to register slave as an async device - %d\n",
1353 platform_set_drvdata(pdev, s3cdma);
1354 dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
1355 pdata->num_phy_channels);
1360 dma_async_device_unregister(&s3cdma->memcpy);
1362 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1364 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1366 if (sdata->has_clocks)
1367 for (i = 0; i < pdata->num_phy_channels; i++) {
1368 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1370 clk_unprepare(phy->clk);
1376 static int s3c24xx_dma_remove(struct platform_device *pdev)
1378 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1379 struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
1380 struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
1383 dma_async_device_unregister(&s3cdma->slave);
1384 dma_async_device_unregister(&s3cdma->memcpy);
1386 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1387 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1389 if (sdata->has_clocks)
1390 for (i = 0; i < pdata->num_phy_channels; i++) {
1391 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1393 clk_unprepare(phy->clk);
1399 static struct platform_driver s3c24xx_dma_driver = {
1401 .name = "s3c24xx-dma",
1403 .id_table = s3c24xx_dma_driver_ids,
1404 .probe = s3c24xx_dma_probe,
1405 .remove = s3c24xx_dma_remove,
1408 module_platform_driver(s3c24xx_dma_driver);
1410 bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
1412 struct s3c24xx_dma_chan *s3cchan;
1414 if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
1417 s3cchan = to_s3c24xx_dma_chan(chan);
1419 return s3cchan->id == (int)param;
1421 EXPORT_SYMBOL(s3c24xx_dma_filter);
1423 MODULE_DESCRIPTION("S3C24XX DMA Driver");
1424 MODULE_AUTHOR("Heiko Stuebner");
1425 MODULE_LICENSE("GPL v2");