2 * Copyright 2015 Linaro.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/sched.h>
9 #include <linux/device.h>
10 #include <linux/dmaengine.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmapool.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/platform_device.h>
18 #include <linux/slab.h>
19 #include <linux/spinlock.h>
20 #include <linux/of_device.h>
22 #include <linux/clk.h>
23 #include <linux/of_dma.h>
27 #define DRIVER_NAME "zx-dma"
29 #define DMA_MAX_SIZE (0x10000 - PAGE_SIZE)
30 #define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
32 #define REG_ZX_SRC_ADDR 0x00
33 #define REG_ZX_DST_ADDR 0x04
34 #define REG_ZX_TX_X_COUNT 0x08
35 #define REG_ZX_TX_ZY_COUNT 0x0c
36 #define REG_ZX_SRC_ZY_STEP 0x10
37 #define REG_ZX_DST_ZY_STEP 0x14
38 #define REG_ZX_LLI_ADDR 0x1c
39 #define REG_ZX_CTRL 0x20
40 #define REG_ZX_TC_IRQ 0x800
41 #define REG_ZX_SRC_ERR_IRQ 0x804
42 #define REG_ZX_DST_ERR_IRQ 0x808
43 #define REG_ZX_CFG_ERR_IRQ 0x80c
44 #define REG_ZX_TC_IRQ_RAW 0x810
45 #define REG_ZX_SRC_ERR_IRQ_RAW 0x814
46 #define REG_ZX_DST_ERR_IRQ_RAW 0x818
47 #define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
48 #define REG_ZX_STATUS 0x820
49 #define REG_ZX_DMA_GRP_PRIO 0x824
50 #define REG_ZX_DMA_ARB 0x828
52 #define ZX_FORCE_CLOSE BIT(31)
53 #define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
54 #define ZX_MAX_BURST_LEN 16
55 #define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
56 #define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
57 #define ZX_IRQ_ENABLE_ALL (3 << 4)
58 #define ZX_DST_FIFO_MODE BIT(3)
59 #define ZX_SRC_FIFO_MODE BIT(2)
60 #define ZX_SOFT_REQ BIT(1)
61 #define ZX_CH_ENABLE BIT(0)
63 #define ZX_DMA_BUSWIDTHS \
64 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
65 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
66 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
67 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
68 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
70 enum zx_dma_burst_width {
71 ZX_DMA_WIDTH_8BIT = 0,
72 ZX_DMA_WIDTH_16BIT = 1,
73 ZX_DMA_WIDTH_32BIT = 2,
74 ZX_DMA_WIDTH_64BIT = 3,
87 u32 reserved[7]; /* pack as hardware registers region size */
90 struct zx_dma_desc_sw {
91 struct virt_dma_desc vd;
92 dma_addr_t desc_hw_lli;
95 struct zx_desc_hw *desc_hw;
101 struct dma_slave_config slave_cfg;
102 int id; /* Request phy chan id */
104 struct virt_dma_chan vc;
105 struct zx_dma_phy *phy;
106 struct list_head node;
108 enum dma_status status;
114 struct zx_dma_chan *vchan;
115 struct zx_dma_desc_sw *ds_run;
116 struct zx_dma_desc_sw *ds_done;
120 struct dma_device slave;
122 spinlock_t lock; /* lock for ch and phy */
123 struct list_head chan_pending;
124 struct zx_dma_phy *phy;
125 struct zx_dma_chan *chans;
127 struct dma_pool *pool;
132 #define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
134 static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan)
136 return container_of(chan, struct zx_dma_chan, vc.chan);
139 static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d)
143 val = readl_relaxed(phy->base + REG_ZX_CTRL);
144 val &= ~ZX_CH_ENABLE;
145 writel_relaxed(val, phy->base + REG_ZX_CTRL);
147 val = 0x1 << phy->idx;
148 writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW);
149 writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
150 writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW);
151 writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
154 static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw)
156 writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR);
157 writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR);
158 writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT);
159 writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT);
160 writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP);
161 writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP);
162 writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR);
163 writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL);
166 static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy)
168 return readl_relaxed(phy->base + REG_ZX_LLI_ADDR);
171 static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d)
173 return readl_relaxed(d->base + REG_ZX_STATUS);
176 static void zx_dma_init_state(struct zx_dma_dev *d)
178 /* set same priority */
179 writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB);
181 writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW);
182 writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
183 writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW);
184 writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
187 static int zx_dma_start_txd(struct zx_dma_chan *c)
189 struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device);
190 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
195 if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d))
199 struct zx_dma_desc_sw *ds =
200 container_of(vd, struct zx_dma_desc_sw, vd);
202 * fetch and remove request from vc->desc_issued
203 * so vc->desc_issued only contains desc pending
205 list_del(&ds->vd.node);
207 c->phy->ds_done = NULL;
209 zx_dma_set_desc(c->phy, ds->desc_hw);
212 c->phy->ds_done = NULL;
213 c->phy->ds_run = NULL;
217 static void zx_dma_task(struct zx_dma_dev *d)
219 struct zx_dma_phy *p;
220 struct zx_dma_chan *c, *cn;
221 unsigned pch, pch_alloc = 0;
224 /* check new dma request of running channel in vc->desc_issued */
225 list_for_each_entry_safe(c, cn, &d->slave.channels,
226 vc.chan.device_node) {
227 spin_lock_irqsave(&c->vc.lock, flags);
229 if (p && p->ds_done && zx_dma_start_txd(c)) {
230 /* No current txd associated with this channel */
231 dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx);
232 /* Mark this channel free */
236 spin_unlock_irqrestore(&c->vc.lock, flags);
239 /* check new channel request in d->chan_pending */
240 spin_lock_irqsave(&d->lock, flags);
241 while (!list_empty(&d->chan_pending)) {
242 c = list_first_entry(&d->chan_pending,
243 struct zx_dma_chan, node);
246 /* remove from d->chan_pending */
247 list_del_init(&c->node);
248 pch_alloc |= 1 << c->id;
249 /* Mark this channel allocated */
253 dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id);
256 spin_unlock_irqrestore(&d->lock, flags);
258 for (pch = 0; pch < d->dma_channels; pch++) {
259 if (pch_alloc & (1 << pch)) {
263 spin_lock_irqsave(&c->vc.lock, flags);
265 spin_unlock_irqrestore(&c->vc.lock, flags);
271 static irqreturn_t zx_dma_int_handler(int irq, void *dev_id)
273 struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id;
274 struct zx_dma_phy *p;
275 struct zx_dma_chan *c;
276 u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ);
277 u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ);
278 u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ);
279 u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ);
290 spin_lock_irqsave(&c->vc.lock, flags);
291 vchan_cookie_complete(&p->ds_run->vd);
292 p->ds_done = p->ds_run;
293 spin_unlock_irqrestore(&c->vc.lock, flags);
298 if (serr || derr || cfg)
299 dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n",
302 writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW);
303 writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
304 writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW);
305 writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
315 static void zx_dma_free_chan_resources(struct dma_chan *chan)
317 struct zx_dma_chan *c = to_zx_chan(chan);
318 struct zx_dma_dev *d = to_zx_dma(chan->device);
321 spin_lock_irqsave(&d->lock, flags);
322 list_del_init(&c->node);
323 spin_unlock_irqrestore(&d->lock, flags);
325 vchan_free_chan_resources(&c->vc);
329 static enum dma_status zx_dma_tx_status(struct dma_chan *chan,
331 struct dma_tx_state *state)
333 struct zx_dma_chan *c = to_zx_chan(chan);
334 struct zx_dma_phy *p;
335 struct virt_dma_desc *vd;
340 ret = dma_cookie_status(&c->vc.chan, cookie, state);
341 if (ret == DMA_COMPLETE || !state)
344 spin_lock_irqsave(&c->vc.lock, flags);
349 * If the cookie is on our issue queue, then the residue is
352 vd = vchan_find_desc(&c->vc, cookie);
354 bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size;
355 } else if ((!p) || (!p->ds_run)) {
358 struct zx_dma_desc_sw *ds = p->ds_run;
359 u32 clli = 0, index = 0;
362 clli = zx_dma_get_curr_lli(p);
363 index = (clli - ds->desc_hw_lli) / sizeof(struct zx_desc_hw);
364 for (; index < ds->desc_num; index++) {
365 bytes += ds->desc_hw[index].src_x;
367 if (!ds->desc_hw[index].lli)
371 spin_unlock_irqrestore(&c->vc.lock, flags);
372 dma_set_residue(state, bytes);
376 static void zx_dma_issue_pending(struct dma_chan *chan)
378 struct zx_dma_chan *c = to_zx_chan(chan);
379 struct zx_dma_dev *d = to_zx_dma(chan->device);
383 spin_lock_irqsave(&c->vc.lock, flags);
384 /* add request to vc->desc_issued */
385 if (vchan_issue_pending(&c->vc)) {
387 if (!c->phy && list_empty(&c->node)) {
388 /* if new channel, add chan_pending */
389 list_add_tail(&c->node, &d->chan_pending);
391 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
393 spin_unlock(&d->lock);
395 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
397 spin_unlock_irqrestore(&c->vc.lock, flags);
403 static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst,
404 dma_addr_t src, size_t len, u32 num, u32 ccfg)
406 if ((num + 1) < ds->desc_num)
407 ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
408 sizeof(struct zx_desc_hw);
409 ds->desc_hw[num].saddr = src;
410 ds->desc_hw[num].daddr = dst;
411 ds->desc_hw[num].src_x = len;
412 ds->desc_hw[num].ctr = ccfg;
415 static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num,
416 struct dma_chan *chan)
418 struct zx_dma_chan *c = to_zx_chan(chan);
419 struct zx_dma_desc_sw *ds;
420 struct zx_dma_dev *d = to_zx_dma(chan->device);
421 int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw);
423 if (num > lli_limit) {
424 dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
425 &c->vc, num, lli_limit);
429 ds = kzalloc(sizeof(*ds), GFP_ATOMIC);
433 ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
435 dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
439 memset(ds->desc_hw, sizeof(struct zx_desc_hw) * num, 0);
444 static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width)
447 case DMA_SLAVE_BUSWIDTH_1_BYTE:
448 case DMA_SLAVE_BUSWIDTH_2_BYTES:
449 case DMA_SLAVE_BUSWIDTH_4_BYTES:
450 case DMA_SLAVE_BUSWIDTH_8_BYTES:
451 return ffs(width) - 1;
453 return ZX_DMA_WIDTH_32BIT;
457 static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir)
459 struct dma_slave_config *cfg = &c->slave_cfg;
460 enum zx_dma_burst_width src_width;
461 enum zx_dma_burst_width dst_width;
466 c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ
467 | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1)
468 | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT)
469 | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT);
472 c->dev_addr = cfg->dst_addr;
473 /* dst len is calculated from src width, len and dst width.
474 * We need make sure dst len not exceed MAX LEN.
476 dst_width = zx_dma_burst_width(cfg->dst_addr_width);
477 maxburst = cfg->dst_maxburst * cfg->dst_addr_width
478 / DMA_SLAVE_BUSWIDTH_8_BYTES;
479 maxburst = maxburst < ZX_MAX_BURST_LEN ?
480 maxburst : ZX_MAX_BURST_LEN;
481 c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE
482 | ZX_SRC_BURST_LEN(maxburst - 1)
483 | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_64BIT)
484 | ZX_DST_BURST_WIDTH(dst_width);
487 c->dev_addr = cfg->src_addr;
488 src_width = zx_dma_burst_width(cfg->src_addr_width);
489 maxburst = cfg->src_maxburst;
490 maxburst = maxburst < ZX_MAX_BURST_LEN ?
491 maxburst : ZX_MAX_BURST_LEN;
492 c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE
493 | ZX_SRC_BURST_LEN(maxburst - 1)
494 | ZX_SRC_BURST_WIDTH(src_width)
495 | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_64BIT);
503 static struct dma_async_tx_descriptor *zx_dma_prep_memcpy(
504 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
505 size_t len, unsigned long flags)
507 struct zx_dma_chan *c = to_zx_chan(chan);
508 struct zx_dma_desc_sw *ds;
515 if (zx_pre_config(c, DMA_MEM_TO_MEM))
518 num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
520 ds = zx_alloc_desc_resource(num, chan);
528 copy = min_t(size_t, len, DMA_MAX_SIZE);
529 zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg);
536 ds->desc_hw[num - 1].lli = 0; /* end of link */
537 ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
538 return vchan_tx_prep(&c->vc, &ds->vd, flags);
541 static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg(
542 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
543 enum dma_transfer_direction dir, unsigned long flags, void *context)
545 struct zx_dma_chan *c = to_zx_chan(chan);
546 struct zx_dma_desc_sw *ds;
547 size_t len, avail, total = 0;
548 struct scatterlist *sg;
549 dma_addr_t addr, src = 0, dst = 0;
555 if (zx_pre_config(c, dir))
558 for_each_sg(sgl, sg, sglen, i) {
559 avail = sg_dma_len(sg);
560 if (avail > DMA_MAX_SIZE)
561 num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
564 ds = zx_alloc_desc_resource(num, chan);
569 for_each_sg(sgl, sg, sglen, i) {
570 addr = sg_dma_address(sg);
571 avail = sg_dma_len(sg);
575 len = min_t(size_t, avail, DMA_MAX_SIZE);
577 if (dir == DMA_MEM_TO_DEV) {
580 } else if (dir == DMA_DEV_TO_MEM) {
585 zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg);
592 ds->desc_hw[num - 1].lli = 0; /* end of link */
593 ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
595 return vchan_tx_prep(&c->vc, &ds->vd, flags);
598 static int zx_dma_config(struct dma_chan *chan,
599 struct dma_slave_config *cfg)
601 struct zx_dma_chan *c = to_zx_chan(chan);
606 memcpy(&c->slave_cfg, cfg, sizeof(*cfg));
611 static int zx_dma_terminate_all(struct dma_chan *chan)
613 struct zx_dma_chan *c = to_zx_chan(chan);
614 struct zx_dma_dev *d = to_zx_dma(chan->device);
615 struct zx_dma_phy *p = c->phy;
619 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
621 /* Prevent this channel being scheduled */
623 list_del_init(&c->node);
624 spin_unlock(&d->lock);
626 /* Clear the tx descriptor lists */
627 spin_lock_irqsave(&c->vc.lock, flags);
628 vchan_get_all_descriptors(&c->vc, &head);
630 /* vchan is assigned to a pchan - stop the channel */
631 zx_dma_terminate_chan(p, d);
637 spin_unlock_irqrestore(&c->vc.lock, flags);
638 vchan_dma_desc_free_list(&c->vc, &head);
643 static void zx_dma_free_desc(struct virt_dma_desc *vd)
645 struct zx_dma_desc_sw *ds =
646 container_of(vd, struct zx_dma_desc_sw, vd);
647 struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device);
649 dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
653 static const struct of_device_id zx6702_dma_dt_ids[] = {
654 { .compatible = "zte,zx296702-dma", },
657 MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids);
659 static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec,
660 struct of_dma *ofdma)
662 struct zx_dma_dev *d = ofdma->of_dma_data;
663 unsigned int request = dma_spec->args[0];
664 struct dma_chan *chan;
665 struct zx_dma_chan *c;
667 if (request > d->dma_requests)
670 chan = dma_get_any_slave_channel(&d->slave);
672 dev_err(d->slave.dev, "get channel fail in %s.\n", __func__);
675 c = to_zx_chan(chan);
677 dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n",
682 static int zx_dma_probe(struct platform_device *op)
684 struct zx_dma_dev *d;
685 struct resource *iores;
686 int i, ret = 0, irq = 0;
688 iores = platform_get_resource(op, IORESOURCE_MEM, 0);
692 d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL);
696 d->base = devm_ioremap_resource(&op->dev, iores);
698 return PTR_ERR(d->base);
700 of_property_read_u32((&op->dev)->of_node,
701 "dma-channels", &d->dma_channels);
702 of_property_read_u32((&op->dev)->of_node,
703 "dma-requests", &d->dma_requests);
704 if (!d->dma_requests || !d->dma_channels)
707 d->clk = devm_clk_get(&op->dev, NULL);
708 if (IS_ERR(d->clk)) {
709 dev_err(&op->dev, "no dma clk\n");
710 return PTR_ERR(d->clk);
713 irq = platform_get_irq(op, 0);
714 ret = devm_request_irq(&op->dev, irq, zx_dma_int_handler,
719 /* A DMA memory pool for LLIs, align on 32-byte boundary */
720 d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
721 LLI_BLOCK_SIZE, 32, 0);
725 /* init phy channel */
726 d->phy = devm_kzalloc(&op->dev,
727 d->dma_channels * sizeof(struct zx_dma_phy), GFP_KERNEL);
731 for (i = 0; i < d->dma_channels; i++) {
732 struct zx_dma_phy *p = &d->phy[i];
735 p->base = d->base + i * 0x40;
738 INIT_LIST_HEAD(&d->slave.channels);
739 dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
740 dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
741 dma_cap_set(DMA_PRIVATE, d->slave.cap_mask);
742 d->slave.dev = &op->dev;
743 d->slave.device_free_chan_resources = zx_dma_free_chan_resources;
744 d->slave.device_tx_status = zx_dma_tx_status;
745 d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy;
746 d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg;
747 d->slave.device_issue_pending = zx_dma_issue_pending;
748 d->slave.device_config = zx_dma_config;
749 d->slave.device_terminate_all = zx_dma_terminate_all;
750 d->slave.copy_align = DMA_ALIGN;
751 d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS;
752 d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS;
753 d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV)
754 | BIT(DMA_DEV_TO_MEM);
755 d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
757 /* init virtual channel */
758 d->chans = devm_kzalloc(&op->dev,
759 d->dma_requests * sizeof(struct zx_dma_chan), GFP_KERNEL);
763 for (i = 0; i < d->dma_requests; i++) {
764 struct zx_dma_chan *c = &d->chans[i];
766 c->status = DMA_IN_PROGRESS;
767 INIT_LIST_HEAD(&c->node);
768 c->vc.desc_free = zx_dma_free_desc;
769 vchan_init(&c->vc, &d->slave);
772 /* Enable clock before accessing registers */
773 ret = clk_prepare_enable(d->clk);
775 dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret);
779 zx_dma_init_state(d);
781 spin_lock_init(&d->lock);
782 INIT_LIST_HEAD(&d->chan_pending);
783 platform_set_drvdata(op, d);
785 ret = dma_async_device_register(&d->slave);
789 ret = of_dma_controller_register((&op->dev)->of_node,
790 zx_of_dma_simple_xlate, d);
792 goto of_dma_register_fail;
794 dev_info(&op->dev, "initialized\n");
797 of_dma_register_fail:
798 dma_async_device_unregister(&d->slave);
800 clk_disable_unprepare(d->clk);
805 static int zx_dma_remove(struct platform_device *op)
807 struct zx_dma_chan *c, *cn;
808 struct zx_dma_dev *d = platform_get_drvdata(op);
810 dma_async_device_unregister(&d->slave);
811 of_dma_controller_free((&op->dev)->of_node);
813 list_for_each_entry_safe(c, cn, &d->slave.channels,
814 vc.chan.device_node) {
815 list_del(&c->vc.chan.device_node);
817 clk_disable_unprepare(d->clk);
818 dmam_pool_destroy(d->pool);
823 #ifdef CONFIG_PM_SLEEP
824 static int zx_dma_suspend_dev(struct device *dev)
826 struct zx_dma_dev *d = dev_get_drvdata(dev);
829 stat = zx_dma_get_chan_stat(d);
831 dev_warn(d->slave.dev,
832 "chan %d is running fail to suspend\n", stat);
835 clk_disable_unprepare(d->clk);
839 static int zx_dma_resume_dev(struct device *dev)
841 struct zx_dma_dev *d = dev_get_drvdata(dev);
844 ret = clk_prepare_enable(d->clk);
846 dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret);
849 zx_dma_init_state(d);
854 static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev);
856 static struct platform_driver zx_pdma_driver = {
860 .of_match_table = zx6702_dma_dt_ids,
862 .probe = zx_dma_probe,
863 .remove = zx_dma_remove,
866 module_platform_driver(zx_pdma_driver);
868 MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver");
869 MODULE_AUTHOR("Jun Nie jun.nie@linaro.org");
870 MODULE_LICENSE("GPL v2");