From c9784a467380dfbd8070c735e651af07331172d9 Mon Sep 17 00:00:00 2001 From: Jie Yang Date: Thu, 7 Jan 2016 08:39:33 +0800 Subject: dmaengine: dw: fix a typo for bitfields of CTL_LO The offset of SINC should be 9, not 7, here fix this typo. Signed-off-by: Jie Yang Acked-by: Andy Shevchenko Signed-off-by: Vinod Koul --- drivers/dma/dw/regs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index 241ff2b1402b..0a50c18d85b8 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -150,7 +150,7 @@ enum dw_dma_msize { #define DWC_CTLL_DST_INC (0<<7) /* DAR update/not */ #define DWC_CTLL_DST_DEC (1<<7) #define DWC_CTLL_DST_FIX (2<<7) -#define DWC_CTLL_SRC_INC (0<<7) /* SAR update/not */ +#define DWC_CTLL_SRC_INC (0<<9) /* SAR update/not */ #define DWC_CTLL_SRC_DEC (1<<9) #define DWC_CTLL_SRC_FIX (2<<9) #define DWC_CTLL_DST_MSIZE(n) ((n)<<11) /* burst, #elements */ -- cgit v1.2.3 From 94901e1b24496f04bd59176b7f87406e71505e27 Mon Sep 17 00:00:00 2001 From: Alexander Sverdlin Date: Thu, 7 Jan 2016 23:05:48 +0100 Subject: dmaengine: ep93xx: Treat STALL and NFB IRQs same way Debugging ALSA hangups it was found that EP9302 (latest E2 rev.) DMA controller sometimes asserts STALL interrupt instead of NFB interrupt. Simply ignoring the difference and simply acting upon the amount of data we still have to transfer seems to work fine. This somehow sounds similar to M2M issue which is already dealt with in the driver, when the controller asserts DONE interrupt too early. The issue is not documented in Cirrus Logic erratas for EP93XX, but original Cirrus DMA driver from 2003 (not based on DMA API) did the similar handling of STALL interrupt. In-tree driver (6d831c65) did it also, before conversion to DMA engine API. Signed-off-by: Alexander Sverdlin Signed-off-by: Vinod Koul --- drivers/dma/ep93xx_dma.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 57ff46284f15..21f08cc3352b 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -421,23 +421,25 @@ static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac) desc->size); } - switch (irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)) { - case M2P_INTERRUPT_STALL: - /* Disable interrupts */ - control = readl(edmac->regs + M2P_CONTROL); - control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT); - m2p_set_control(edmac, control); - - return INTERRUPT_DONE; - - case M2P_INTERRUPT_NFB: - if (ep93xx_dma_advance_active(edmac)) - m2p_fill_desc(edmac); + /* + * Even latest E2 silicon revision sometimes assert STALL interrupt + * instead of NFB. Therefore we treat them equally, basing on the + * amount of data we still have to transfer. + */ + if (!(irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB))) + return INTERRUPT_UNKNOWN; + if (ep93xx_dma_advance_active(edmac)) { + m2p_fill_desc(edmac); return INTERRUPT_NEXT_BUFFER; } - return INTERRUPT_UNKNOWN; + /* Disable interrupts */ + control = readl(edmac->regs + M2P_CONTROL); + control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT); + m2p_set_control(edmac, control); + + return INTERRUPT_DONE; } /* -- cgit v1.2.3 From 8319f84adb49b5c6d69bbce86442e26c2277376f Mon Sep 17 00:00:00 2001 From: Tim Gardner Date: Fri, 8 Jan 2016 14:48:17 -0700 Subject: dmaengine: ioatdma: Squelch framesize warnings CC [M] drivers/dma/ioat/prep.o drivers/dma/ioat/prep.c: In function 'ioat_prep_pqxor': drivers/dma/ioat/prep.c:682:1: warning: the frame size of 1048 bytes is larger than 1024 bytes [-Wframe-larger-than=] } ^ drivers/dma/ioat/prep.c: In function 'ioat_prep_pqxor_val': drivers/dma/ioat/prep.c:714:1: warning: the frame size of 1048 bytes is larger than 1024 bytes [-Wframe-larger-than=] } gcc version 5.3.1 20151219 (Ubuntu 5.3.1-4ubuntu1) Cc: Vinod Koul Cc: Dan Williams Cc: Dave Jiang Cc: Prarit Bhargava Cc: Nicholas Mc Guire Cc: Jarkko Nikula Signed-off-by: Tim Gardner Signed-off-by: Vinod Koul --- drivers/dma/ioat/prep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/prep.c b/drivers/dma/ioat/prep.c index 6bb4a13a8fbd..243421af888f 100644 --- a/drivers/dma/ioat/prep.c +++ b/drivers/dma/ioat/prep.c @@ -26,7 +26,7 @@ #include "hw.h" #include "dma.h" -#define MAX_SCF 1024 +#define MAX_SCF 256 /* provide a lookup table for setting the source address in the base or * extended descriptor of an xor or pq descriptor -- cgit v1.2.3 From 5c229d00a58df469550dbc1f4e3a360eb8958661 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Wed, 3 Feb 2016 11:55:34 +0100 Subject: dmaengine: sh: Use ARCH_RENESAS Make use of ARCH_RENESAS in place of ARCH_SHMOBILE. This is part of an ongoing process to migrate from ARCH_SHMOBILE to ARCH_RENESAS the motivation for which being that RENESAS seems to be a more appropriate name than SHMOBILE for the majority of Renesas ARM based SoCs. Signed-off-by: Simon Horman Signed-off-by: Vinod Koul --- drivers/dma/sh/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index f32c430eb16c..6e0685f1a838 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -12,7 +12,7 @@ config RENESAS_DMA config SH_DMAE_BASE bool "Renesas SuperH DMA Engine support" - depends on SUPERH || ARCH_SHMOBILE || COMPILE_TEST + depends on SUPERH || ARCH_RENESAS || COMPILE_TEST depends on !SUPERH || SH_DMA depends on !SH_DMA_API default y @@ -41,7 +41,7 @@ endif config RCAR_DMAC tristate "Renesas R-Car Gen2 DMA Controller" - depends on ARCH_SHMOBILE || COMPILE_TEST + depends on ARCH_RENESAS || COMPILE_TEST select RENESAS_DMA help This driver supports the general purpose DMA controller found in the @@ -49,7 +49,7 @@ config RCAR_DMAC config RENESAS_USB_DMAC tristate "Renesas USB-DMA Controller" - depends on ARCH_SHMOBILE || COMPILE_TEST + depends on ARCH_RENESAS || COMPILE_TEST select RENESAS_DMA select DMA_VIRTUAL_CHANNELS help -- cgit v1.2.3 From 848e9776fee424b9368c72377de5d3509b17937c Mon Sep 17 00:00:00 2001 From: Boojin Kim Date: Fri, 22 Jan 2016 19:06:44 +0800 Subject: dmaengine: pl330: support burst mode for dev-to-mem and mem-to-dev transmit This patch adds to support burst mode for dev-to-mem and mem-to-dev transmit. Signed-off-by: Boojin Kim Signed-off-by: Addy Ke Signed-off-by: Shawn Lin cc: Heiko Stuebner cc: Doug Anderson cc: Olof Johansson Reviewed-by: Sonny Rao Signed-off-by: Caesar Wang Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 17ee758b419f..7cdf8d4c7947 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1141,10 +1141,13 @@ static inline int _ldst_devtomem(unsigned dry_run, u8 buf[], const struct _xfer_spec *pxs, int cyc) { int off = 0; + enum pl330_cond cond; + + cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; while (cyc--) { - off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri); - off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->desc->peri); + off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); + off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_ST(dry_run, &buf[off], ALWAYS); off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); } @@ -1156,11 +1159,14 @@ static inline int _ldst_memtodev(unsigned dry_run, u8 buf[], const struct _xfer_spec *pxs, int cyc) { int off = 0; + enum pl330_cond cond; + + cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; while (cyc--) { - off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri); + off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_LD(dry_run, &buf[off], ALWAYS); - off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->desc->peri); + off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); } @@ -1199,7 +1205,7 @@ static inline int _loop(unsigned dry_run, u8 buf[], struct _arg_LPEND lpend; if (*bursts == 1) - return _bursts(dry_run, buf, pxs, 1); + return _bursts(pl330, dry_run, buf, pxs, 1); /* Max iterations possible in DMALP is 256 */ if (*bursts >= 256*256) { @@ -2560,7 +2566,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( desc->rqtype = direction; desc->rqcfg.brst_size = pch->burst_sz; - desc->rqcfg.brst_len = 1; + desc->rqcfg.brst_len = pch->burst_len; desc->bytes_requested = period_len; fill_px(&desc->px, dst, src, period_len); @@ -2705,7 +2711,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } desc->rqcfg.brst_size = pch->burst_sz; - desc->rqcfg.brst_len = 1; + desc->rqcfg.brst_len = pch->burst_len; desc->rqtype = direction; desc->bytes_requested = sg_dma_len(sg); } -- cgit v1.2.3 From 271e1b86e69140fe65718ae8a264284c46d3129d Mon Sep 17 00:00:00 2001 From: Addy Ke Date: Fri, 22 Jan 2016 19:06:46 +0800 Subject: dmaengine: pl330: add quirk for broken no flushp This patch add "arm,pl330-broken-no-flushp" quirk to avoid execute DMAFLUSHP if Soc doesn't support it. Signed-off-by: Addy Ke Signed-off-by: Shawn Lin cc: Doug Anderson cc: Heiko Stuebner cc: Olof Johansson Reviewed-by: Sonny Rao Signed-off-by: Caesar Wang Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 87 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 25 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 7cdf8d4c7947..008408db6ae0 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -34,6 +34,8 @@ #define PL330_MAX_IRQS 32 #define PL330_MAX_PERI 32 +#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0) + enum pl330_cachectrl { CCTRL0, /* Noncacheable and nonbufferable */ CCTRL1, /* Bufferable only */ @@ -488,6 +490,17 @@ struct pl330_dmac { /* Peripheral channels connected to this DMAC */ unsigned int num_peripherals; struct dma_pl330_chan *peripherals; /* keep at end */ + int quirks; +}; + +static struct pl330_of_quirks { + char *quirk; + int id; +} of_quirks[] = { + { + .quirk = "arm,pl330-broken-no-flushp", + .id = PL330_QUIRK_BROKEN_NO_FLUSHP, + } }; struct dma_pl330_desc { @@ -1137,53 +1150,68 @@ static inline int _ldst_memtomem(unsigned dry_run, u8 buf[], return off; } -static inline int _ldst_devtomem(unsigned dry_run, u8 buf[], - const struct _xfer_spec *pxs, int cyc) +static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run, + u8 buf[], const struct _xfer_spec *pxs, + int cyc) { int off = 0; enum pl330_cond cond; - cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; + if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) + cond = BURST; + else + cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; while (cyc--) { off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_ST(dry_run, &buf[off], ALWAYS); - off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); + + if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)) + off += _emit_FLUSHP(dry_run, &buf[off], + pxs->desc->peri); } return off; } -static inline int _ldst_memtodev(unsigned dry_run, u8 buf[], - const struct _xfer_spec *pxs, int cyc) +static inline int _ldst_memtodev(struct pl330_dmac *pl330, + unsigned dry_run, u8 buf[], + const struct _xfer_spec *pxs, int cyc) { int off = 0; enum pl330_cond cond; - cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; + if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) + cond = BURST; + else + cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; + while (cyc--) { off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); off += _emit_LD(dry_run, &buf[off], ALWAYS); off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri); - off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); + + if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)) + off += _emit_FLUSHP(dry_run, &buf[off], + pxs->desc->peri); } return off; } -static int _bursts(unsigned dry_run, u8 buf[], +static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], const struct _xfer_spec *pxs, int cyc) { int off = 0; switch (pxs->desc->rqtype) { case DMA_MEM_TO_DEV: - off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc); + off += _ldst_memtodev(pl330, dry_run, &buf[off], pxs, cyc); break; case DMA_DEV_TO_MEM: - off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc); + off += _ldst_devtomem(pl330, dry_run, &buf[off], pxs, cyc); break; case DMA_MEM_TO_MEM: off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc); @@ -1197,7 +1225,7 @@ static int _bursts(unsigned dry_run, u8 buf[], } /* Returns bytes consumed and updates bursts */ -static inline int _loop(unsigned dry_run, u8 buf[], +static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], unsigned long *bursts, const struct _xfer_spec *pxs) { int cyc, cycmax, szlp, szlpend, szbrst, off; @@ -1223,7 +1251,7 @@ static inline int _loop(unsigned dry_run, u8 buf[], } szlp = _emit_LP(1, buf, 0, 0); - szbrst = _bursts(1, buf, pxs, 1); + szbrst = _bursts(pl330, 1, buf, pxs, 1); lpend.cond = ALWAYS; lpend.forever = false; @@ -1255,7 +1283,7 @@ static inline int _loop(unsigned dry_run, u8 buf[], off += _emit_LP(dry_run, &buf[off], 1, lcnt1); ljmp1 = off; - off += _bursts(dry_run, &buf[off], pxs, cyc); + off += _bursts(pl330, dry_run, &buf[off], pxs, cyc); lpend.cond = ALWAYS; lpend.forever = false; @@ -1278,8 +1306,9 @@ static inline int _loop(unsigned dry_run, u8 buf[], return off; } -static inline int _setup_loops(unsigned dry_run, u8 buf[], - const struct _xfer_spec *pxs) +static inline int _setup_loops(struct pl330_dmac *pl330, + unsigned dry_run, u8 buf[], + const struct _xfer_spec *pxs) { struct pl330_xfer *x = &pxs->desc->px; u32 ccr = pxs->ccr; @@ -1288,15 +1317,16 @@ static inline int _setup_loops(unsigned dry_run, u8 buf[], while (bursts) { c = bursts; - off += _loop(dry_run, &buf[off], &c, pxs); + off += _loop(pl330, dry_run, &buf[off], &c, pxs); bursts -= c; } return off; } -static inline int _setup_xfer(unsigned dry_run, u8 buf[], - const struct _xfer_spec *pxs) +static inline int _setup_xfer(struct pl330_dmac *pl330, + unsigned dry_run, u8 buf[], + const struct _xfer_spec *pxs) { struct pl330_xfer *x = &pxs->desc->px; int off = 0; @@ -1307,7 +1337,7 @@ static inline int _setup_xfer(unsigned dry_run, u8 buf[], off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); /* Setup Loop(s) */ - off += _setup_loops(dry_run, &buf[off], pxs); + off += _setup_loops(pl330, dry_run, &buf[off], pxs); return off; } @@ -1316,8 +1346,9 @@ static inline int _setup_xfer(unsigned dry_run, u8 buf[], * A req is a sequence of one or more xfer units. * Returns the number of bytes taken to setup the MC for the req. */ -static int _setup_req(unsigned dry_run, struct pl330_thread *thrd, - unsigned index, struct _xfer_spec *pxs) +static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run, + struct pl330_thread *thrd, unsigned index, + struct _xfer_spec *pxs) { struct _pl330_req *req = &thrd->req[index]; struct pl330_xfer *x; @@ -1334,7 +1365,7 @@ static int _setup_req(unsigned dry_run, struct pl330_thread *thrd, if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr))) return -EINVAL; - off += _setup_xfer(dry_run, &buf[off], pxs); + off += _setup_xfer(pl330, dry_run, &buf[off], pxs); /* DMASEV peripheral/event */ off += _emit_SEV(dry_run, &buf[off], thrd->ev); @@ -1428,7 +1459,7 @@ static int pl330_submit_req(struct pl330_thread *thrd, xs.desc = desc; /* First dry run to check if req is acceptable */ - ret = _setup_req(1, thrd, idx, &xs); + ret = _setup_req(pl330, 1, thrd, idx, &xs); if (ret < 0) goto xfer_exit; @@ -1442,7 +1473,7 @@ static int pl330_submit_req(struct pl330_thread *thrd, /* Hook the request */ thrd->lstenq = idx; thrd->req[idx].desc = desc; - _setup_req(0, thrd, idx, &xs); + _setup_req(pl330, 0, thrd, idx, &xs); ret = 0; @@ -2787,6 +2818,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) struct resource *res; int i, ret, irq; int num_chan; + struct device_node *np = adev->dev.of_node; pdat = dev_get_platdata(&adev->dev); @@ -2806,6 +2838,11 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0; + /* get quirk */ + for (i = 0; i < ARRAY_SIZE(of_quirks); i++) + if (of_property_read_bool(np, of_quirks[i].quirk)) + pl330->quirks |= of_quirks[i].id; + res = &adev->res; pl330->base = devm_ioremap_resource(&adev->dev, res); if (IS_ERR(pl330->base)) -- cgit v1.2.3 From 6d5bbed30f89acd2ae0d23b3fff5b13b307525d9 Mon Sep 17 00:00:00 2001 From: Shawn Lin Date: Fri, 22 Jan 2016 19:06:50 +0800 Subject: dmaengine: core: expose max burst capability to clients This patch add max_burst to dma_get_slave_caps for clients to get the burst capability of slave dma controller. Signed-off-by: Shawn Lin Signed-off-by: Caesar Wang Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/dma') diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index c50a247be2e0..0cb259c59916 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -496,6 +496,7 @@ int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps) caps->src_addr_widths = device->src_addr_widths; caps->dst_addr_widths = device->dst_addr_widths; caps->directions = device->directions; + caps->max_burst = device->max_burst; caps->residue_granularity = device->residue_granularity; caps->descriptor_reuse = device->descriptor_reuse; -- cgit v1.2.3 From 86a8ce7d4103f6680236af6916abf38e12477df8 Mon Sep 17 00:00:00 2001 From: Shawn Lin Date: Fri, 22 Jan 2016 19:06:51 +0800 Subject: dmaengine: pl330: add max burst for dmaengine This patch add max burst capability for dmaengine and limit burst capability to one for PL330_QUIRK_BROKEN_NO_FLUSHP Signed-off-by: Shawn Lin Signed-off-by: Caesar Wang Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 008408db6ae0..1b0453b9e32d 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -33,6 +33,7 @@ #define PL330_MAX_CHAN 8 #define PL330_MAX_IRQS 32 #define PL330_MAX_PERI 32 +#define PL330_MAX_BURST 16 #define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0) @@ -2938,6 +2939,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) pd->dst_addr_widths = PL330_DMA_BUSWIDTHS; pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); pd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ? + 1 : PL330_MAX_BURST); ret = dma_async_device_register(pd); if (ret) { -- cgit v1.2.3 From a2826e662eb13a041078260692b4e023ed5cbf62 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 26 Jan 2016 13:43:13 +0200 Subject: dmaengine: idma64: clear LLP_[SD]_EN bits in last descriptor The datasheet requires that the user must clear LLP_[SD]_EN bits whenever LLP.LOC is zero, i.e. in the last descriptor of a multi-block chain. Make the driver do this. Signed-off-by: Andy Shevchenko Signed-off-by: Vinod Koul --- drivers/dma/idma64.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index 3cb7b2c78197..1953e57505f4 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -289,6 +289,9 @@ static void idma64_desc_fill(struct idma64_chan *idma64c, /* Trigger an interrupt after the last block is transfered */ lli->ctllo |= IDMA64C_CTLL_INT_EN; + + /* Disable LLP transfer in the last block */ + lli->ctllo &= ~(IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN); } static struct dma_async_tx_descriptor *idma64_prep_slave_sg( -- cgit v1.2.3 From 679cfbf79b4eb7d7d81195e6b9ab98106fd78a54 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 10 Feb 2016 15:00:21 -0700 Subject: dmaengine: IOATDMA: Convert pci_pool_* to dma_pool_* Converting old pci_pool_* calls to "new" dma_pool_* to make everything uniform. Signed-off-by: Dave Jiang Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 6 +++--- drivers/dma/ioat/dma.h | 4 ++-- drivers/dma/ioat/init.c | 20 ++++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 1d5df2ef148b..7a04c16a0bfa 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -298,14 +298,14 @@ ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) dma_addr_t phys; ioat_dma = to_ioatdma_device(chan->device); - hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys); + hw = dma_pool_alloc(ioat_dma->dma_pool, flags, &phys); if (!hw) return NULL; memset(hw, 0, sizeof(*hw)); desc = kmem_cache_zalloc(ioat_cache, flags); if (!desc) { - pci_pool_free(ioat_dma->dma_pool, hw, phys); + dma_pool_free(ioat_dma->dma_pool, hw, phys); return NULL; } @@ -321,7 +321,7 @@ void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) struct ioatdma_device *ioat_dma; ioat_dma = to_ioatdma_device(chan->device); - pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys); + dma_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys); kmem_cache_free(ioat_cache, desc); } diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index b8f48074789f..f471092440d3 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -76,8 +76,8 @@ enum ioat_irq_mode { struct ioatdma_device { struct pci_dev *pdev; void __iomem *reg_base; - struct pci_pool *dma_pool; - struct pci_pool *completion_pool; + struct dma_pool *dma_pool; + struct dma_pool *completion_pool; #define MAX_SED_POOLS 5 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; struct dma_device dma_dev; diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 4ef0c5e07912..b02b63b719db 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -505,7 +505,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) struct device *dev = &pdev->dev; /* DMA coherent memory pool for DMA descriptor allocations */ - ioat_dma->dma_pool = pci_pool_create("dma_desc_pool", pdev, + ioat_dma->dma_pool = dma_pool_create("dma_desc_pool", dev, sizeof(struct ioat_dma_descriptor), 64, 0); if (!ioat_dma->dma_pool) { @@ -513,7 +513,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) goto err_dma_pool; } - ioat_dma->completion_pool = pci_pool_create("completion_pool", pdev, + ioat_dma->completion_pool = dma_pool_create("completion_pool", dev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); @@ -546,9 +546,9 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) err_self_test: ioat_disable_interrupts(ioat_dma); err_setup_interrupts: - pci_pool_destroy(ioat_dma->completion_pool); + dma_pool_destroy(ioat_dma->completion_pool); err_completion_pool: - pci_pool_destroy(ioat_dma->dma_pool); + dma_pool_destroy(ioat_dma->dma_pool); err_dma_pool: return err; } @@ -559,8 +559,8 @@ static int ioat_register(struct ioatdma_device *ioat_dma) if (err) { ioat_disable_interrupts(ioat_dma); - pci_pool_destroy(ioat_dma->completion_pool); - pci_pool_destroy(ioat_dma->dma_pool); + dma_pool_destroy(ioat_dma->completion_pool); + dma_pool_destroy(ioat_dma->dma_pool); } return err; @@ -576,8 +576,8 @@ static void ioat_dma_remove(struct ioatdma_device *ioat_dma) dma_async_device_unregister(dma); - pci_pool_destroy(ioat_dma->dma_pool); - pci_pool_destroy(ioat_dma->completion_pool); + dma_pool_destroy(ioat_dma->dma_pool); + dma_pool_destroy(ioat_dma->completion_pool); INIT_LIST_HEAD(&dma->channels); } @@ -669,7 +669,7 @@ static void ioat_free_chan_resources(struct dma_chan *c) kfree(ioat_chan->ring); ioat_chan->ring = NULL; ioat_chan->alloc_order = 0; - pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion, + dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion, ioat_chan->completion_dma); spin_unlock_bh(&ioat_chan->prep_lock); spin_unlock_bh(&ioat_chan->cleanup_lock); @@ -701,7 +701,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c) /* allocate a completion writeback area */ /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ ioat_chan->completion = - pci_pool_alloc(ioat_chan->ioat_dma->completion_pool, + dma_pool_alloc(ioat_chan->ioat_dma->completion_pool, GFP_KERNEL, &ioat_chan->completion_dma); if (!ioat_chan->completion) return -ENOMEM; -- cgit v1.2.3 From cd60cd96137f6cb3ea82cace9225626619e7a52d Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 10 Feb 2016 15:00:26 -0700 Subject: dmaengine: IOATDMA: Removing descriptor ring reshape Moving to contingous memory backed descriptor rings. This makes is really difficult and complex to do reshape. Going to remove this as I don't think we need to do it anymore. Signed-off-by: Dave Jiang Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 143 ------------------------------------------------ drivers/dma/ioat/dma.h | 5 -- drivers/dma/ioat/init.c | 10 +--- 3 files changed, 1 insertion(+), 157 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 7a04c16a0bfa..9c4d3b20f520 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -332,9 +332,6 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) int descs = 1 << order; int i; - if (order > ioat_get_max_alloc_order()) - return NULL; - /* allocate the array to hold the software ring */ ring = kcalloc(descs, sizeof(*ring), flags); if (!ring) @@ -362,114 +359,6 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) return ring; } -static bool reshape_ring(struct ioatdma_chan *ioat_chan, int order) -{ - /* reshape differs from normal ring allocation in that we want - * to allocate a new software ring while only - * extending/truncating the hardware ring - */ - struct dma_chan *c = &ioat_chan->dma_chan; - const u32 curr_size = ioat_ring_size(ioat_chan); - const u16 active = ioat_ring_active(ioat_chan); - const u32 new_size = 1 << order; - struct ioat_ring_ent **ring; - u32 i; - - if (order > ioat_get_max_alloc_order()) - return false; - - /* double check that we have at least 1 free descriptor */ - if (active == curr_size) - return false; - - /* when shrinking, verify that we can hold the current active - * set in the new ring - */ - if (active >= new_size) - return false; - - /* allocate the array to hold the software ring */ - ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT); - if (!ring) - return false; - - /* allocate/trim descriptors as needed */ - if (new_size > curr_size) { - /* copy current descriptors to the new ring */ - for (i = 0; i < curr_size; i++) { - u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); - u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - - ring[new_idx] = ioat_chan->ring[curr_idx]; - set_desc_id(ring[new_idx], new_idx); - } - - /* add new descriptors to the ring */ - for (i = curr_size; i < new_size; i++) { - u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - - ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT); - if (!ring[new_idx]) { - while (i--) { - u16 new_idx = (ioat_chan->tail+i) & - (new_size-1); - - ioat_free_ring_ent(ring[new_idx], c); - } - kfree(ring); - return false; - } - set_desc_id(ring[new_idx], new_idx); - } - - /* hw link new descriptors */ - for (i = curr_size-1; i < new_size; i++) { - u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - struct ioat_ring_ent *next = - ring[(new_idx+1) & (new_size-1)]; - struct ioat_dma_descriptor *hw = ring[new_idx]->hw; - - hw->next = next->txd.phys; - } - } else { - struct ioat_dma_descriptor *hw; - struct ioat_ring_ent *next; - - /* copy current descriptors to the new ring, dropping the - * removed descriptors - */ - for (i = 0; i < new_size; i++) { - u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); - u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - - ring[new_idx] = ioat_chan->ring[curr_idx]; - set_desc_id(ring[new_idx], new_idx); - } - - /* free deleted descriptors */ - for (i = new_size; i < curr_size; i++) { - struct ioat_ring_ent *ent; - - ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i); - ioat_free_ring_ent(ent, c); - } - - /* fix up hardware ring */ - hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw; - next = ring[(ioat_chan->tail+new_size) & (new_size-1)]; - hw->next = next->txd.phys; - } - - dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n", - __func__, new_size); - - kfree(ioat_chan->ring); - ioat_chan->ring = ring; - ioat_chan->alloc_order = order; - - return true; -} - /** * ioat_check_space_lock - verify space and grab ring producer lock * @ioat: ioat,3 channel (ring) to operate on @@ -478,9 +367,6 @@ static bool reshape_ring(struct ioatdma_chan *ioat_chan, int order) int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) __acquires(&ioat_chan->prep_lock) { - bool retry; - - retry: spin_lock_bh(&ioat_chan->prep_lock); /* never allow the last descriptor to be consumed, we need at * least one free at all times to allow for on-the-fly ring @@ -493,24 +379,8 @@ int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) ioat_chan->produce = num_descs; return 0; /* with ioat->prep_lock held */ } - retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); spin_unlock_bh(&ioat_chan->prep_lock); - /* is another cpu already trying to expand the ring? */ - if (retry) - goto retry; - - spin_lock_bh(&ioat_chan->cleanup_lock); - spin_lock_bh(&ioat_chan->prep_lock); - retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1); - clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); - spin_unlock_bh(&ioat_chan->prep_lock); - spin_unlock_bh(&ioat_chan->cleanup_lock); - - /* if we were able to expand the ring retry the allocation */ - if (retry) - goto retry; - dev_dbg_ratelimited(to_dev(ioat_chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", __func__, num_descs, ioat_chan->head, @@ -823,19 +693,6 @@ static void check_active(struct ioatdma_chan *ioat_chan) if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); - else if (ioat_chan->alloc_order > ioat_get_alloc_order()) { - /* if the ring is idle, empty, and oversized try to step - * down the size - */ - reshape_ring(ioat_chan, ioat_chan->alloc_order - 1); - - /* keep shrinking until we get back to our minimum - * default size - */ - if (ioat_chan->alloc_order > ioat_get_alloc_order()) - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); - } - } void ioat_timer_event(unsigned long data) diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index f471092440d3..5f2f9fbcf184 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -100,7 +100,6 @@ struct ioatdma_chan { #define IOAT_COMPLETION_ACK 1 #define IOAT_RESET_PENDING 2 #define IOAT_KOBJ_INIT_FAIL 3 - #define IOAT_RESHAPE_PENDING 4 #define IOAT_RUN 5 #define IOAT_CHAN_ACTIVE 6 struct timer_list timer; @@ -302,10 +301,6 @@ static inline bool is_ioat_bug(unsigned long err) } #define IOAT_MAX_ORDER 16 -#define ioat_get_alloc_order() \ - (min(ioat_ring_alloc_order, IOAT_MAX_ORDER)) -#define ioat_get_max_alloc_order() \ - (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan) { diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index b02b63b719db..66369204896a 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -136,14 +136,6 @@ int ioat_pending_level = 4; module_param(ioat_pending_level, int, 0644); MODULE_PARM_DESC(ioat_pending_level, "high-water mark for pushing ioat descriptors (default: 4)"); -int ioat_ring_alloc_order = 8; -module_param(ioat_ring_alloc_order, int, 0644); -MODULE_PARM_DESC(ioat_ring_alloc_order, - "ioat+: allocate 2^n descriptors per channel (default: 8 max: 16)"); -int ioat_ring_max_alloc_order = IOAT_MAX_ORDER; -module_param(ioat_ring_max_alloc_order, int, 0644); -MODULE_PARM_DESC(ioat_ring_max_alloc_order, - "ioat+: upper limit for ring size (default: 16)"); static char ioat_interrupt_style[32] = "msix"; module_param_string(ioat_interrupt_style, ioat_interrupt_style, sizeof(ioat_interrupt_style), 0644); @@ -712,7 +704,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c) writel(((u64)ioat_chan->completion_dma) >> 32, ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); - order = ioat_get_alloc_order(); + order = IOAT_MAX_ORDER; ring = ioat_alloc_ring(c, order, GFP_KERNEL); if (!ring) return -ENOMEM; -- cgit v1.2.3 From dd4645ebb7d100bb04ba38ec58b499cbe95322fa Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 10 Feb 2016 15:00:32 -0700 Subject: dmaengine: IOATDMA: Allocate DMA descriptor ring in contig DMA memory Future IOATDMA hardware will take advantage of descriptors residing in contiguous memory. Setting the descriptor ring in max config DMA memory of 2MB. Each channel will need 2 of these chunks. This should provide 64k of 64B descriptors. Signed-off-by: Dave Jiang Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 75 ++++++++++++++++++++++++++++++++++++++----------- drivers/dma/ioat/dma.h | 11 ++++++-- drivers/dma/ioat/hw.h | 2 ++ drivers/dma/ioat/init.c | 27 ++++++++---------- 4 files changed, 81 insertions(+), 34 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 9c4d3b20f520..14ae9a0994fd 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "dma.h" #include "registers.h" #include "hw.h" @@ -290,24 +291,30 @@ static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx) } static struct ioat_ring_ent * -ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) +ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags) { struct ioat_dma_descriptor *hw; struct ioat_ring_ent *desc; struct ioatdma_device *ioat_dma; + struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); + int chunk; dma_addr_t phys; + u8 *pos; + off_t offs; ioat_dma = to_ioatdma_device(chan->device); - hw = dma_pool_alloc(ioat_dma->dma_pool, flags, &phys); - if (!hw) - return NULL; + + chunk = idx / IOAT_DESCS_PER_2M; + idx &= (IOAT_DESCS_PER_2M - 1); + offs = idx * IOAT_DESC_SZ; + pos = (u8 *)ioat_chan->descs[chunk].virt + offs; + phys = ioat_chan->descs[chunk].hw + offs; + hw = (struct ioat_dma_descriptor *)pos; memset(hw, 0, sizeof(*hw)); desc = kmem_cache_zalloc(ioat_cache, flags); - if (!desc) { - dma_pool_free(ioat_dma->dma_pool, hw, phys); + if (!desc) return NULL; - } dma_async_tx_descriptor_init(&desc->txd, chan); desc->txd.tx_submit = ioat_tx_submit_unlock; @@ -318,29 +325,63 @@ ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) { - struct ioatdma_device *ioat_dma; - - ioat_dma = to_ioatdma_device(chan->device); - dma_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys); kmem_cache_free(ioat_cache, desc); } struct ioat_ring_ent ** ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) { + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct ioat_ring_ent **ring; - int descs = 1 << order; - int i; + int total_descs = 1 << order; + int i, chunks; /* allocate the array to hold the software ring */ - ring = kcalloc(descs, sizeof(*ring), flags); + ring = kcalloc(total_descs, sizeof(*ring), flags); if (!ring) return NULL; - for (i = 0; i < descs; i++) { - ring[i] = ioat_alloc_ring_ent(c, flags); + + ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M; + + for (i = 0; i < chunks; i++) { + struct ioat_descs *descs = &ioat_chan->descs[i]; + + descs->virt = dma_alloc_coherent(to_dev(ioat_chan), + SZ_2M, &descs->hw, flags); + if (!descs->virt && (i > 0)) { + int idx; + + for (idx = 0; idx < i; idx++) { + dma_free_coherent(to_dev(ioat_chan), SZ_2M, + descs->virt, descs->hw); + descs->virt = NULL; + descs->hw = 0; + } + + ioat_chan->desc_chunks = 0; + kfree(ring); + return NULL; + } + } + + for (i = 0; i < total_descs; i++) { + ring[i] = ioat_alloc_ring_ent(c, i, flags); if (!ring[i]) { + int idx; + while (i--) ioat_free_ring_ent(ring[i], c); + + for (idx = 0; idx < ioat_chan->desc_chunks; idx++) { + dma_free_coherent(to_dev(ioat_chan), + SZ_2M, + ioat_chan->descs[idx].virt, + ioat_chan->descs[idx].hw); + ioat_chan->descs[idx].virt = NULL; + ioat_chan->descs[idx].hw = 0; + } + + ioat_chan->desc_chunks = 0; kfree(ring); return NULL; } @@ -348,7 +389,7 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) } /* link descs */ - for (i = 0; i < descs-1; i++) { + for (i = 0; i < total_descs-1; i++) { struct ioat_ring_ent *next = ring[i+1]; struct ioat_dma_descriptor *hw = ring[i]->hw; diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 5f2f9fbcf184..57a9b83db455 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -62,7 +62,6 @@ enum ioat_irq_mode { * struct ioatdma_device - internal representation of a IOAT device * @pdev: PCI-Express device * @reg_base: MMIO register space base address - * @dma_pool: for allocating DMA descriptors * @completion_pool: DMA buffers for completion ops * @sed_hw_pool: DMA super descriptor pools * @dma_dev: embedded struct dma_device @@ -76,7 +75,6 @@ enum ioat_irq_mode { struct ioatdma_device { struct pci_dev *pdev; void __iomem *reg_base; - struct dma_pool *dma_pool; struct dma_pool *completion_pool; #define MAX_SED_POOLS 5 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; @@ -90,6 +88,11 @@ struct ioatdma_device { u32 cap; }; +struct ioat_descs { + void *virt; + dma_addr_t hw; +}; + struct ioatdma_chan { struct dma_chan dma_chan; void __iomem *reg_base; @@ -132,6 +135,8 @@ struct ioatdma_chan { u16 produce; struct ioat_ring_ent **ring; spinlock_t prep_lock; + struct ioat_descs descs[2]; + int desc_chunks; }; struct ioat_sysfs_entry { @@ -301,6 +306,8 @@ static inline bool is_ioat_bug(unsigned long err) } #define IOAT_MAX_ORDER 16 +#define IOAT_MAX_DESCS 65536 +#define IOAT_DESCS_PER_2M 32768 static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan) { diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 690e3b4f8202..8e67895bcca3 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -73,6 +73,8 @@ int system_has_dca_enabled(struct pci_dev *pdev); +#define IOAT_DESC_SZ 64 + struct ioat_dma_descriptor { uint32_t size; union { diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 66369204896a..efdee1a69fc4 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "dma.h" #include "registers.h" #include "hw.h" @@ -496,15 +497,6 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) struct pci_dev *pdev = ioat_dma->pdev; struct device *dev = &pdev->dev; - /* DMA coherent memory pool for DMA descriptor allocations */ - ioat_dma->dma_pool = dma_pool_create("dma_desc_pool", dev, - sizeof(struct ioat_dma_descriptor), - 64, 0); - if (!ioat_dma->dma_pool) { - err = -ENOMEM; - goto err_dma_pool; - } - ioat_dma->completion_pool = dma_pool_create("completion_pool", dev, sizeof(u64), SMP_CACHE_BYTES, @@ -512,7 +504,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) if (!ioat_dma->completion_pool) { err = -ENOMEM; - goto err_completion_pool; + goto err_out; } ioat_enumerate_channels(ioat_dma); @@ -539,9 +531,7 @@ err_self_test: ioat_disable_interrupts(ioat_dma); err_setup_interrupts: dma_pool_destroy(ioat_dma->completion_pool); -err_completion_pool: - dma_pool_destroy(ioat_dma->dma_pool); -err_dma_pool: +err_out: return err; } @@ -552,7 +542,6 @@ static int ioat_register(struct ioatdma_device *ioat_dma) if (err) { ioat_disable_interrupts(ioat_dma); dma_pool_destroy(ioat_dma->completion_pool); - dma_pool_destroy(ioat_dma->dma_pool); } return err; @@ -568,7 +557,6 @@ static void ioat_dma_remove(struct ioatdma_device *ioat_dma) dma_async_device_unregister(dma); - dma_pool_destroy(ioat_dma->dma_pool); dma_pool_destroy(ioat_dma->completion_pool); INIT_LIST_HEAD(&dma->channels); @@ -658,6 +646,15 @@ static void ioat_free_chan_resources(struct dma_chan *c) ioat_free_ring_ent(desc, c); } + for (i = 0; i < ioat_chan->desc_chunks; i++) { + dma_free_coherent(to_dev(ioat_chan), SZ_2M, + ioat_chan->descs[i].virt, + ioat_chan->descs[i].hw); + ioat_chan->descs[i].virt = NULL; + ioat_chan->descs[i].hw = 0; + } + ioat_chan->desc_chunks = 0; + kfree(ioat_chan->ring); ioat_chan->ring = NULL; ioat_chan->alloc_order = 0; -- cgit v1.2.3 From 9bef6d8288eb568121d3a9ef8ada80960ad9bdec Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Thu, 11 Feb 2016 11:08:34 +0200 Subject: dmaengine: omap-dma: Implement device_synchronize callback We need the callback to support the dmaengine_terminate_sync(). Signed-off-by: Peter Ujfalusi Signed-off-by: Vinod Koul --- drivers/dma/omap-dma.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 9794b073d7d7..43bd5aee7ffe 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -1009,6 +1009,13 @@ static int omap_dma_terminate_all(struct dma_chan *chan) return 0; } +static void omap_dma_synchronize(struct dma_chan *chan) +{ + struct omap_chan *c = to_omap_dma_chan(chan); + + vchan_synchronize(&c->vc); +} + static int omap_dma_pause(struct dma_chan *chan) { struct omap_chan *c = to_omap_dma_chan(chan); @@ -1112,6 +1119,7 @@ static int omap_dma_probe(struct platform_device *pdev) od->ddev.device_pause = omap_dma_pause; od->ddev.device_resume = omap_dma_resume; od->ddev.device_terminate_all = omap_dma_terminate_all; + od->ddev.device_synchronize = omap_dma_synchronize; od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS; od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS; od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); -- cgit v1.2.3 From b84730ffcfccbed76e7f623b336e9bba8d78d93e Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Thu, 11 Feb 2016 11:08:42 +0200 Subject: dmaengine: edma: Implement device_synchronize callback We need the callback to support the dmaengine_terminate_sync(). Signed-off-by: Peter Ujfalusi Signed-off-by: Vinod Koul --- drivers/dma/edma.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index d92d65549406..81bbfdc20591 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -866,6 +866,13 @@ static int edma_terminate_all(struct dma_chan *chan) return 0; } +static void edma_synchronize(struct dma_chan *chan) +{ + struct edma_chan *echan = to_edma_chan(chan); + + vchan_synchronize(&echan->vchan); +} + static int edma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { @@ -1798,6 +1805,7 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) s_ddev->device_pause = edma_dma_pause; s_ddev->device_resume = edma_dma_resume; s_ddev->device_terminate_all = edma_terminate_all; + s_ddev->device_synchronize = edma_synchronize; s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS; s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS; @@ -1823,6 +1831,7 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) m_ddev->device_pause = edma_dma_pause; m_ddev->device_resume = edma_dma_resume; m_ddev->device_terminate_all = edma_terminate_all; + m_ddev->device_synchronize = edma_synchronize; m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS; m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS; -- cgit v1.2.3 From e4d8817cbe9dad5da7a8487184d1b9ee8c927d92 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Thu, 11 Feb 2016 15:17:48 +0200 Subject: dmaengine: edma: Fetch echan->edesc while holding lock in edma_comletion_handler In order to avoid possible race condition when client drivers are using dmaengine_terminate_sync() call to disable the channel. Signed-off-by: Peter Ujfalusi Suggested-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/edma.c | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 81bbfdc20591..29a7723918d9 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -1369,36 +1369,36 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( static void edma_completion_handler(struct edma_chan *echan) { struct device *dev = echan->vchan.chan.device->dev; - struct edma_desc *edesc = echan->edesc; - - if (!edesc) - return; + struct edma_desc *edesc; spin_lock(&echan->vchan.lock); - if (edesc->cyclic) { - vchan_cyclic_callback(&edesc->vdesc); - spin_unlock(&echan->vchan.lock); - return; - } else if (edesc->processed == edesc->pset_nr) { - edesc->residue = 0; - edma_stop(echan); - vchan_cookie_complete(&edesc->vdesc); - echan->edesc = NULL; - - dev_dbg(dev, "Transfer completed on channel %d\n", - echan->ch_num); - } else { - dev_dbg(dev, "Sub transfer completed on channel %d\n", - echan->ch_num); - - edma_pause(echan); - - /* Update statistics for tx_status */ - edesc->residue -= edesc->sg_len; - edesc->residue_stat = edesc->residue; - edesc->processed_stat = edesc->processed; + edesc = echan->edesc; + if (edesc) { + if (edesc->cyclic) { + vchan_cyclic_callback(&edesc->vdesc); + spin_unlock(&echan->vchan.lock); + return; + } else if (edesc->processed == edesc->pset_nr) { + edesc->residue = 0; + edma_stop(echan); + vchan_cookie_complete(&edesc->vdesc); + echan->edesc = NULL; + + dev_dbg(dev, "Transfer completed on channel %d\n", + echan->ch_num); + } else { + dev_dbg(dev, "Sub transfer completed on channel %d\n", + echan->ch_num); + + edma_pause(echan); + + /* Update statistics for tx_status */ + edesc->residue -= edesc->sg_len; + edesc->residue_stat = edesc->residue; + edesc->processed_stat = edesc->processed; + } + edma_execute(echan); } - edma_execute(echan); spin_unlock(&echan->vchan.lock); } -- cgit v1.2.3 From a02eb37ac6dc240e0903fdcdd0d7be300b0e3d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lothar=20Wa=C3=9Fmann?= Date: Thu, 11 Feb 2016 14:19:54 +0100 Subject: dmaengine: enable mxs-dma for imx6ul MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mxs-dma unit is also available on i.MX6UL. Make it possible to select it in Kconfig. Signed-off-by: Lothar Waßmann Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 79b1390f2016..c77f214c9466 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -341,12 +341,13 @@ config MV_XOR config MXS_DMA bool "MXS DMA support" - depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q + depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q || SOC_IMX6UL select STMP_DEVICE select DMA_ENGINE help Support the MXS DMA engine. This engine including APBH-DMA - and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips. + and APBX-DMA is integrated into Freescale + i.MX23/28/MX6Q/MX6DL/MX6UL chips. config MX3_IPU bool "MX3x Image Processing Unit support" -- cgit v1.2.3 From 8e2067be0aa71d119ade237c5351245aff78e024 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 16 Feb 2016 11:28:05 +0200 Subject: dmaengine: idma64: fix typo in WR_ISSUE_THD(x) There is a typo in the definition of IDMA64C_CFGH_WR_ISSUE_THD(x). Fix it by swapping characters. Signed-off-by: Andy Shevchenko Signed-off-by: Vinod Koul --- drivers/dma/idma64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/idma64.h b/drivers/dma/idma64.h index 8423f13ed0da..dc6874424188 100644 --- a/drivers/dma/idma64.h +++ b/drivers/dma/idma64.h @@ -71,7 +71,7 @@ #define IDMA64C_CFGH_SRC_PER(x) ((x) << 0) /* src peripheral */ #define IDMA64C_CFGH_DST_PER(x) ((x) << 4) /* dst peripheral */ #define IDMA64C_CFGH_RD_ISSUE_THD(x) ((x) << 8) -#define IDMA64C_CFGH_RW_ISSUE_THD(x) ((x) << 18) +#define IDMA64C_CFGH_WR_ISSUE_THD(x) ((x) << 18) /* Interrupt registers */ -- cgit v1.2.3 From 36bf8fc42ea41110c02f47456dbc4283ee7981dc Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 16 Feb 2016 11:26:53 +0200 Subject: dmaengine: acpi-dma: align debug message with flow In acpi_dma_request_slave_chan_by_name() the debug message is printed before the actual matching happens. Correct the message itself to be in align with the flow. Signed-off-by: Andy Shevchenko Acked-by: Mika Westerberg Signed-off-by: Vinod Koul --- drivers/dma/acpi-dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c index eed6bda01790..4a748c3435d7 100644 --- a/drivers/dma/acpi-dma.c +++ b/drivers/dma/acpi-dma.c @@ -438,7 +438,7 @@ struct dma_chan *acpi_dma_request_slave_chan_by_name(struct device *dev, return ERR_PTR(-ENODEV); } - dev_dbg(dev, "found DMA channel \"%s\" at index %d\n", name, index); + dev_dbg(dev, "Looking for DMA channel \"%s\" at index %d...\n", name, index); return acpi_dma_request_slave_chan_by_index(dev, index); } EXPORT_SYMBOL_GPL(acpi_dma_request_slave_chan_by_name); -- cgit v1.2.3 From 0a18f9b268dd66c84c806a74b3bc0be6438e7ba7 Mon Sep 17 00:00:00 2001 From: Caesar Wang Date: Thu, 25 Feb 2016 09:00:53 +0800 Subject: dmaengine: pl330: fix to support the burst mode This patch fixes the burst mode that will break DMA uart on SoCFPGA. In some cases, some SoCS didn't support the multi-burst even if the devices who use the pl330 claim support the maxburst. Fixes: commit 848e977 "dmaengine: pl330: support burst mode for dev-to-mem and mem-to-dev transmit" Reported-by: Dinh Nguyen Signed-off-by: Caesar Wang Tested-by: Bartlomiej Zolnierkiewicz Tested-by: Dinh Nguyen Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 1b0453b9e32d..372b4359da97 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1161,7 +1161,7 @@ static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run, if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) cond = BURST; else - cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; + cond = SINGLE; while (cyc--) { off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); @@ -1186,8 +1186,7 @@ static inline int _ldst_memtodev(struct pl330_dmac *pl330, if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) cond = BURST; else - cond = (pxs->desc->rqcfg.brst_len == 1) ? SINGLE : BURST; - + cond = SINGLE; while (cyc--) { off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri); @@ -2598,7 +2597,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( desc->rqtype = direction; desc->rqcfg.brst_size = pch->burst_sz; - desc->rqcfg.brst_len = pch->burst_len; + desc->rqcfg.brst_len = 1; desc->bytes_requested = period_len; fill_px(&desc->px, dst, src, period_len); @@ -2743,7 +2742,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } desc->rqcfg.brst_size = pch->burst_sz; - desc->rqcfg.brst_len = pch->burst_len; + desc->rqcfg.brst_len = 1; desc->rqtype = direction; desc->bytes_requested = sg_dma_len(sg); } -- cgit v1.2.3 From 7096f36e5388beb06b80576d3ccc3a0df2760035 Mon Sep 17 00:00:00 2001 From: Kedareswara rao Appana Date: Fri, 26 Feb 2016 19:33:51 +0530 Subject: dmaengine: xilinx_vdma: Improve SG engine handling The current driver allows user to queue up multiple segments on to a single transaction descriptor. User will submit this single desc and in the issue_pending() we decode multiple segments and submit to SG HW engine. We free up the allocated_desc when it is submitted to the HW. Existing code prevents the user to prepare multiple trasactions at same time as we are overwrite with the allocated_desc. The best utilization of HW SG engine would happen if we collate the pending list when we start dma this patch updates the same. Signed-off-by: Kedareswara rao Appana Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_vdma.c | 127 ++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 55 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index 6f4b5017ca3b..06bffec934d2 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -190,8 +190,7 @@ struct xilinx_vdma_tx_descriptor { * @desc_offset: TX descriptor registers offset * @lock: Descriptor operation lock * @pending_list: Descriptors waiting - * @active_desc: Active descriptor - * @allocated_desc: Allocated descriptor + * @active_list: Descriptors ready to submit * @done_list: Complete descriptors * @common: DMA common channel * @desc_pool: Descriptors pool @@ -206,6 +205,7 @@ struct xilinx_vdma_tx_descriptor { * @tasklet: Cleanup work after irq * @config: Device configuration info * @flush_on_fsync: Flush on Frame sync + * @desc_pendingcount: Descriptor pending count */ struct xilinx_vdma_chan { struct xilinx_vdma_device *xdev; @@ -213,8 +213,7 @@ struct xilinx_vdma_chan { u32 desc_offset; spinlock_t lock; struct list_head pending_list; - struct xilinx_vdma_tx_descriptor *active_desc; - struct xilinx_vdma_tx_descriptor *allocated_desc; + struct list_head active_list; struct list_head done_list; struct dma_chan common; struct dma_pool *desc_pool; @@ -229,6 +228,7 @@ struct xilinx_vdma_chan { struct tasklet_struct tasklet; struct xilinx_vdma_config config; bool flush_on_fsync; + u32 desc_pendingcount; }; /** @@ -342,19 +342,11 @@ static struct xilinx_vdma_tx_descriptor * xilinx_vdma_alloc_tx_descriptor(struct xilinx_vdma_chan *chan) { struct xilinx_vdma_tx_descriptor *desc; - unsigned long flags; - - if (chan->allocated_desc) - return chan->allocated_desc; desc = kzalloc(sizeof(*desc), GFP_KERNEL); if (!desc) return NULL; - spin_lock_irqsave(&chan->lock, flags); - chan->allocated_desc = desc; - spin_unlock_irqrestore(&chan->lock, flags); - INIT_LIST_HEAD(&desc->segments); return desc; @@ -412,9 +404,7 @@ static void xilinx_vdma_free_descriptors(struct xilinx_vdma_chan *chan) xilinx_vdma_free_desc_list(chan, &chan->pending_list); xilinx_vdma_free_desc_list(chan, &chan->done_list); - - xilinx_vdma_free_tx_descriptor(chan, chan->active_desc); - chan->active_desc = NULL; + xilinx_vdma_free_desc_list(chan, &chan->active_list); spin_unlock_irqrestore(&chan->lock, flags); } @@ -614,25 +604,26 @@ static void xilinx_vdma_start(struct xilinx_vdma_chan *chan) static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) { struct xilinx_vdma_config *config = &chan->config; - struct xilinx_vdma_tx_descriptor *desc; + struct xilinx_vdma_tx_descriptor *desc, *tail_desc; unsigned long flags; u32 reg; - struct xilinx_vdma_tx_segment *head, *tail = NULL; + struct xilinx_vdma_tx_segment *tail_segment; if (chan->err) return; spin_lock_irqsave(&chan->lock, flags); - /* There's already an active descriptor, bail out. */ - if (chan->active_desc) - goto out_unlock; - if (list_empty(&chan->pending_list)) goto out_unlock; desc = list_first_entry(&chan->pending_list, struct xilinx_vdma_tx_descriptor, node); + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_vdma_tx_descriptor, node); + + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_vdma_tx_segment, node); /* If it is SG mode and hardware is busy, cannot submit */ if (chan->has_sg && xilinx_vdma_is_running(chan) && @@ -645,14 +636,9 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) * If hardware is idle, then all descriptors on the running lists are * done, start new transfers */ - if (chan->has_sg) { - head = list_first_entry(&desc->segments, - struct xilinx_vdma_tx_segment, node); - tail = list_entry(desc->segments.prev, - struct xilinx_vdma_tx_segment, node); - - vdma_ctrl_write(chan, XILINX_VDMA_REG_CURDESC, head->phys); - } + if (chan->has_sg) + vdma_ctrl_write(chan, XILINX_VDMA_REG_CURDESC, + desc->async_tx.phys); /* Configure the hardware using info in the config structure */ reg = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR); @@ -694,12 +680,15 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) /* Start the transfer */ if (chan->has_sg) { - vdma_ctrl_write(chan, XILINX_VDMA_REG_TAILDESC, tail->phys); + vdma_ctrl_write(chan, XILINX_VDMA_REG_TAILDESC, + tail_segment->phys); } else { struct xilinx_vdma_tx_segment *segment, *last = NULL; int i = 0; - list_for_each_entry(segment, &desc->segments, node) { + list_for_each_entry(desc, &chan->pending_list, node) { + segment = list_first_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); vdma_desc_write(chan, XILINX_VDMA_REG_START_ADDRESS(i++), segment->hw.buf_addr); @@ -716,8 +705,8 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) vdma_desc_write(chan, XILINX_VDMA_REG_VSIZE, last->hw.vsize); } - list_del(&desc->node); - chan->active_desc = desc; + list_splice_tail_init(&chan->pending_list, &chan->active_list); + chan->desc_pendingcount = 0; out_unlock: spin_unlock_irqrestore(&chan->lock, flags); @@ -742,21 +731,19 @@ static void xilinx_vdma_issue_pending(struct dma_chan *dchan) */ static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan) { - struct xilinx_vdma_tx_descriptor *desc; + struct xilinx_vdma_tx_descriptor *desc, *next; unsigned long flags; spin_lock_irqsave(&chan->lock, flags); - desc = chan->active_desc; - if (!desc) { - dev_dbg(chan->dev, "no running descriptors\n"); + if (list_empty(&chan->active_list)) goto out_unlock; - } - - dma_cookie_complete(&desc->async_tx); - list_add_tail(&desc->node, &chan->done_list); - chan->active_desc = NULL; + list_for_each_entry_safe(desc, next, &chan->active_list, node) { + list_del(&desc->node); + dma_cookie_complete(&desc->async_tx); + list_add_tail(&desc->node, &chan->done_list); + } out_unlock: spin_unlock_irqrestore(&chan->lock, flags); @@ -878,6 +865,44 @@ static irqreturn_t xilinx_vdma_irq_handler(int irq, void *data) return IRQ_HANDLED; } +/** + * append_desc_queue - Queuing descriptor + * @chan: Driver specific dma channel + * @desc: dma transaction descriptor + */ +static void append_desc_queue(struct xilinx_vdma_chan *chan, + struct xilinx_vdma_tx_descriptor *desc) +{ + struct xilinx_vdma_tx_segment *tail_segment; + struct xilinx_vdma_tx_descriptor *tail_desc; + + if (list_empty(&chan->pending_list)) + goto append; + + /* + * Add the hardware descriptor to the chain of hardware descriptors + * that already exists in memory. + */ + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_vdma_tx_descriptor, node); + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_vdma_tx_segment, node); + tail_segment->hw.next_desc = (u32)desc->async_tx.phys; + + /* + * Add the software descriptor and all children to the list + * of pending transactions + */ +append: + list_add_tail(&desc->node, &chan->pending_list); + chan->desc_pendingcount++; + + if (unlikely(chan->desc_pendingcount > chan->num_frms)) { + dev_dbg(chan->dev, "desc pendingcount is too high\n"); + chan->desc_pendingcount = chan->num_frms; + } +} + /** * xilinx_vdma_tx_submit - Submit DMA transaction * @tx: Async transaction descriptor @@ -906,11 +931,8 @@ static dma_cookie_t xilinx_vdma_tx_submit(struct dma_async_tx_descriptor *tx) cookie = dma_cookie_assign(tx); - /* Append the transaction to the pending transactions queue. */ - list_add_tail(&desc->node, &chan->pending_list); - - /* Free the allocated desc */ - chan->allocated_desc = NULL; + /* Put this transaction onto the tail of the pending queue */ + append_desc_queue(chan, desc); spin_unlock_irqrestore(&chan->lock, flags); @@ -973,13 +995,6 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, else hw->buf_addr = xt->src_start; - /* Link the previous next descriptor to current */ - if (!list_empty(&desc->segments)) { - prev = list_last_entry(&desc->segments, - struct xilinx_vdma_tx_segment, node); - prev->hw.next_desc = segment->phys; - } - /* Insert the segment into the descriptor segments list. */ list_add_tail(&segment->node, &desc->segments); @@ -988,7 +1003,7 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, /* Link the last hardware descriptor with the first. */ segment = list_first_entry(&desc->segments, struct xilinx_vdma_tx_segment, node); - prev->hw.next_desc = segment->phys; + desc->async_tx.phys = segment->phys; return &desc->async_tx; @@ -1127,10 +1142,12 @@ static int xilinx_vdma_chan_probe(struct xilinx_vdma_device *xdev, chan->dev = xdev->dev; chan->xdev = xdev; chan->has_sg = xdev->has_sg; + chan->desc_pendingcount = 0x0; spin_lock_init(&chan->lock); INIT_LIST_HEAD(&chan->pending_list); INIT_LIST_HEAD(&chan->done_list); + INIT_LIST_HEAD(&chan->active_list); /* Retrieve the channel properties from the device tree */ has_dre = of_property_read_bool(node, "xlnx,include-dre"); -- cgit v1.2.3 From e2b538a77d99706a9d2dc12925f3f8477c724dcc Mon Sep 17 00:00:00 2001 From: Kedareswara rao Appana Date: Fri, 26 Feb 2016 19:33:53 +0530 Subject: dmaengine: xilinx_vdma: Fix issues with non-parking mode This patch fixes issues with the Non-parking mode(Cirular mode). With the existing driver in cirular mode if we submit frames less than h/w configured we simply end-up having misconfigured vdma h/w. This patch fixes this issue by configuring the frame count register. Signed-off-by: Kedareswara rao Appana Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_vdma.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index 06bffec934d2..e96ff9dea40d 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -648,6 +648,10 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) else reg &= ~XILINX_VDMA_DMACR_FRAMECNT_EN; + /* Configure channel to allow number frame buffers */ + vdma_ctrl_write(chan, XILINX_VDMA_REG_FRMSTORE, + chan->desc_pendingcount); + /* * With SG, start with circular mode, so that BDs can be fetched. * In direct register mode, if not parking, enable circular mode -- cgit v1.2.3 From 26c5e36931b3fea7461ba6f0d8b65eb25ce2b917 Mon Sep 17 00:00:00 2001 From: Kedareswara rao Appana Date: Fri, 26 Feb 2016 19:33:52 +0530 Subject: dmaengine: xilinx_vdma: Simplify spin lock handling This patch simplifies the spin lock handling in the driver by moving locking out of xilinx_dma_start_transfer() API and xilinx_dma_update_completed_cookie() API. Signed-off-by: Kedareswara rao Appana Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_vdma.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index e96ff9dea40d..70b2b32cba15 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -605,17 +605,15 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) { struct xilinx_vdma_config *config = &chan->config; struct xilinx_vdma_tx_descriptor *desc, *tail_desc; - unsigned long flags; u32 reg; struct xilinx_vdma_tx_segment *tail_segment; + /* This function was invoked with lock held */ if (chan->err) return; - spin_lock_irqsave(&chan->lock, flags); - if (list_empty(&chan->pending_list)) - goto out_unlock; + return; desc = list_first_entry(&chan->pending_list, struct xilinx_vdma_tx_descriptor, node); @@ -629,7 +627,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) if (chan->has_sg && xilinx_vdma_is_running(chan) && !xilinx_vdma_is_idle(chan)) { dev_dbg(chan->dev, "DMA controller still busy\n"); - goto out_unlock; + return; } /* @@ -680,7 +678,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) xilinx_vdma_start(chan); if (chan->err) - goto out_unlock; + return; /* Start the transfer */ if (chan->has_sg) { @@ -700,7 +698,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) } if (!last) - goto out_unlock; + return; /* HW expects these parameters to be same for one transaction */ vdma_desc_write(chan, XILINX_VDMA_REG_HSIZE, last->hw.hsize); @@ -711,9 +709,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) list_splice_tail_init(&chan->pending_list, &chan->active_list); chan->desc_pendingcount = 0; - -out_unlock: - spin_unlock_irqrestore(&chan->lock, flags); } /** @@ -723,8 +718,11 @@ out_unlock: static void xilinx_vdma_issue_pending(struct dma_chan *dchan) { struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + unsigned long flags; + spin_lock_irqsave(&chan->lock, flags); xilinx_vdma_start_transfer(chan); + spin_unlock_irqrestore(&chan->lock, flags); } /** @@ -736,21 +734,16 @@ static void xilinx_vdma_issue_pending(struct dma_chan *dchan) static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan) { struct xilinx_vdma_tx_descriptor *desc, *next; - unsigned long flags; - - spin_lock_irqsave(&chan->lock, flags); + /* This function was invoked with lock held */ if (list_empty(&chan->active_list)) - goto out_unlock; + return; list_for_each_entry_safe(desc, next, &chan->active_list, node) { list_del(&desc->node); dma_cookie_complete(&desc->async_tx); list_add_tail(&desc->node, &chan->done_list); } - -out_unlock: - spin_unlock_irqrestore(&chan->lock, flags); } /** @@ -861,8 +854,10 @@ static irqreturn_t xilinx_vdma_irq_handler(int irq, void *data) } if (status & XILINX_VDMA_DMASR_FRM_CNT_IRQ) { + spin_lock(&chan->lock); xilinx_vdma_complete_descriptor(chan); xilinx_vdma_start_transfer(chan); + spin_unlock(&chan->lock); } tasklet_schedule(&chan->tasklet); -- cgit v1.2.3 From 9495f2648287029fb5545c34a0fa318426ebe84c Mon Sep 17 00:00:00 2001 From: Kedareswara rao Appana Date: Fri, 26 Feb 2016 19:33:54 +0530 Subject: dmaengine: xilinx_vdma: Use readl_poll_timeout instead of do while loop's It is sometimes necessary to poll a memory-mapped register until its value satisfies some condition use convenience macros that do this instead of do while loop's. This patch updates the same in the driver. Signed-off-by: Kedareswara rao Appana Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_vdma.c | 46 +++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index 70b2b32cba15..bc2ca457247e 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -254,6 +255,9 @@ struct xilinx_vdma_device { container_of(chan, struct xilinx_vdma_chan, common) #define to_vdma_tx_descriptor(tx) \ container_of(tx, struct xilinx_vdma_tx_descriptor, async_tx) +#define xilinx_vdma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \ + readl_poll_timeout(chan->xdev->regs + chan->ctrl_offset + reg, val, \ + cond, delay_us, timeout_us) /* IO accessors */ static inline u32 vdma_read(struct xilinx_vdma_chan *chan, u32 reg) @@ -550,18 +554,17 @@ static bool xilinx_vdma_is_idle(struct xilinx_vdma_chan *chan) */ static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan) { - int loop = XILINX_VDMA_LOOP_COUNT; + int err = 0; + u32 val; vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); /* Wait for the hardware to halt */ - do { - if (vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & - XILINX_VDMA_DMASR_HALTED) - break; - } while (loop--); + err = xilinx_vdma_poll_timeout(chan, XILINX_VDMA_REG_DMASR, val, + (val & XILINX_VDMA_DMASR_HALTED), 0, + XILINX_VDMA_LOOP_COUNT); - if (!loop) { + if (err) { dev_err(chan->dev, "Cannot stop channel %p: %x\n", chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); chan->err = true; @@ -576,18 +579,17 @@ static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan) */ static void xilinx_vdma_start(struct xilinx_vdma_chan *chan) { - int loop = XILINX_VDMA_LOOP_COUNT; + int err = 0; + u32 val; vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); /* Wait for the hardware to start */ - do { - if (!(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & - XILINX_VDMA_DMASR_HALTED)) - break; - } while (loop--); + err = xilinx_vdma_poll_timeout(chan, XILINX_VDMA_REG_DMASR, val, + !(val & XILINX_VDMA_DMASR_HALTED), 0, + XILINX_VDMA_LOOP_COUNT); - if (!loop) { + if (err) { dev_err(chan->dev, "Cannot start channel %p: %x\n", chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); @@ -754,21 +756,17 @@ static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan) */ static int xilinx_vdma_reset(struct xilinx_vdma_chan *chan) { - int loop = XILINX_VDMA_LOOP_COUNT; + int err = 0; u32 tmp; vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RESET); - tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) & - XILINX_VDMA_DMACR_RESET; - /* Wait for the hardware to finish reset */ - do { - tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) & - XILINX_VDMA_DMACR_RESET; - } while (loop-- && tmp); + err = xilinx_vdma_poll_timeout(chan, XILINX_VDMA_REG_DMACR, tmp, + !(tmp & XILINX_VDMA_DMACR_RESET), 0, + XILINX_VDMA_LOOP_COUNT); - if (!loop) { + if (err) { dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR), vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); @@ -777,7 +775,7 @@ static int xilinx_vdma_reset(struct xilinx_vdma_chan *chan) chan->err = false; - return 0; + return err; } /** -- cgit v1.2.3 From d387ef021a218ee66fe6b64cf447da09cac5c53b Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 29 Feb 2016 12:51:07 +0300 Subject: dma/mic_x100_dma: IS_ERR() vs PTR_ERR() typo This is harmless because the caller only cares about zero vs non-zero but we should be returning PTR_ERR() here. Signed-off-by: Dan Carpenter Signed-off-by: Vinod Koul --- drivers/dma/mic_x100_dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c index 068e920ecb68..1502b24b7c7d 100644 --- a/drivers/dma/mic_x100_dma.c +++ b/drivers/dma/mic_x100_dma.c @@ -483,7 +483,7 @@ static int mic_dma_setup_irq(struct mic_dma_chan *ch) mic_dma_intr_handler, mic_dma_thread_fn, "mic dma_channel", ch, ch->ch_num); if (IS_ERR(ch->cookie)) - return IS_ERR(ch->cookie); + return PTR_ERR(ch->cookie); return 0; } -- cgit v1.2.3 From 94c622b2a742c6793d74a71280df0c3a5365a156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Sun, 21 Feb 2016 22:26:35 -0300 Subject: dmaengine: sun4i: support module autoloading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MODULE_DEVICE_TABLE() is missing, so the module isn't auto-loading on supported systems. This commit adds the missing line so it loads automatically when building it as a module and running on a system with the early sunxi DMA engine. Signed-off-by: Emilio López Reviewed-by: Javier Martinez Canillas Signed-off-by: Vinod Koul --- drivers/dma/sun4i-dma.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/dma') diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index 1661d518224a..e0df233dde92 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -1271,6 +1271,7 @@ static const struct of_device_id sun4i_dma_match[] = { { .compatible = "allwinner,sun4i-a10-dma" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, sun4i_dma_match); static struct platform_driver sun4i_dma_driver = { .probe = sun4i_dma_probe, -- cgit v1.2.3 From 458f5884a1972f26c5260539822b0d158dd85e92 Mon Sep 17 00:00:00 2001 From: Andy Gross Date: Mon, 29 Feb 2016 17:15:19 -0600 Subject: dmaengine: qcom_bam_dma: Make driver work for BE This patch fixes the Qualcomm BAM dmaenging driver to work with big endian kernels. Signed-off-by: Andy Gross Signed-off-by: Vinod Koul --- drivers/dma/qcom_bam_dma.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/qcom_bam_dma.c b/drivers/dma/qcom_bam_dma.c index 5a250cdc8376..37f7aeca7fe1 100644 --- a/drivers/dma/qcom_bam_dma.c +++ b/drivers/dma/qcom_bam_dma.c @@ -53,9 +53,9 @@ #include "virt-dma.h" struct bam_desc_hw { - u32 addr; /* Buffer physical address */ - u16 size; /* Buffer size in bytes */ - u16 flags; + __le32 addr; /* Buffer physical address */ + __le16 size; /* Buffer size in bytes */ + __le16 flags; }; #define DESC_FLAG_INT BIT(15) @@ -632,14 +632,15 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, unsigned int curr_offset = 0; do { - desc->addr = sg_dma_address(sg) + curr_offset; + desc->addr = cpu_to_le32(sg_dma_address(sg) + + curr_offset); if (remainder > BAM_MAX_DATA_SIZE) { - desc->size = BAM_MAX_DATA_SIZE; + desc->size = cpu_to_le16(BAM_MAX_DATA_SIZE); remainder -= BAM_MAX_DATA_SIZE; curr_offset += BAM_MAX_DATA_SIZE; } else { - desc->size = remainder; + desc->size = cpu_to_le16(remainder); remainder = 0; } @@ -915,9 +916,11 @@ static void bam_start_dma(struct bam_chan *bchan) /* set any special flags on the last descriptor */ if (async_desc->num_desc == async_desc->xfer_len) - desc[async_desc->xfer_len - 1].flags = async_desc->flags; + desc[async_desc->xfer_len - 1].flags = + cpu_to_le16(async_desc->flags); else - desc[async_desc->xfer_len - 1].flags |= DESC_FLAG_INT; + desc[async_desc->xfer_len - 1].flags |= + cpu_to_le16(DESC_FLAG_INT); if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) { u32 partial = MAX_DESCRIPTORS - bchan->tail; -- cgit v1.2.3 From 333f16ec68f6fc6b208130de22717b585e6f3242 Mon Sep 17 00:00:00 2001 From: Laxman Dewangan Date: Tue, 1 Mar 2016 18:54:40 +0530 Subject: dmaengine: tegra: don't open code of_device_get_match_data() Use of_device_get_match_data() for getting matched data instead of implementing this locally. Signed-off-by: Laxman Dewangan Acked-by: Thierry Reding Signed-off-by: Vinod Koul --- drivers/dma/tegra20-apb-dma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 935da8192f59..ced588f22f0d 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -1317,15 +1317,13 @@ static int tegra_dma_probe(struct platform_device *pdev) struct tegra_dma *tdma; int ret; int i; - const struct tegra_dma_chip_data *cdata = NULL; - const struct of_device_id *match; + const struct tegra_dma_chip_data *cdata; - match = of_match_device(tegra_dma_of_match, &pdev->dev); - if (!match) { - dev_err(&pdev->dev, "Error: No device match found\n"); + cdata = of_device_get_match_data(&pdev->dev); + if (!cdata) { + dev_err(&pdev->dev, "Error: No device match data found\n"); return -ENODEV; } - cdata = match->data; tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels * sizeof(struct tegra_dma_channel), GFP_KERNEL); -- cgit v1.2.3 From 6fb5629987239c6a8438fab8fd324b7724e49453 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 1 Mar 2016 17:37:07 +0100 Subject: dmaengine: sh: shdmac: don't open code of_device_get_match_data() This change will also make Coverity happy by avoiding a theoretical NULL pointer dereference; yet another reason is to use the above helper function to tighten the code and make it more readable. Signed-off-by: Wolfram Sang Acked-by: Geert Uytterhoeven Signed-off-by: Vinod Koul --- drivers/dma/sh/shdmac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 11707df1a689..80d86402490e 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -699,7 +699,7 @@ static int sh_dmae_probe(struct platform_device *pdev) struct resource *chan, *dmars, *errirq_res, *chanirq_res; if (pdev->dev.of_node) - pdata = of_match_device(sh_dmae_of_match, &pdev->dev)->data; + pdata = of_device_get_match_data(&pdev->dev); else pdata = dev_get_platdata(&pdev->dev); -- cgit v1.2.3 From 20c169aceb459575681c4f4578d0e312bd1436c1 Mon Sep 17 00:00:00 2001 From: Kuninori Morimoto Date: Thu, 3 Mar 2016 17:25:53 +0900 Subject: dmaengine: rcar-dmac: clear pertinence number of channels DMACHCLR clears each channels, but its channel number is based on its SoC or IP. Current driver is using fixed 0x7fff (= for 15ch), it is not good match for Gen3 or Gen2 Audio DMAC. This patch fixes it Signed-off-by: Kuninori Morimoto Acked-by: Geert Uytterhoeven Acked-by: Laurent Pinchart Signed-off-by: Vinod Koul --- drivers/dma/sh/rcar-dmac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 7820d07e7bee..dfb17926297b 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -413,7 +413,7 @@ static int rcar_dmac_init(struct rcar_dmac *dmac) u16 dmaor; /* Clear all channels and enable the DMAC globally. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, 0x7fff); + rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0)); rcar_dmac_write(dmac, RCAR_DMAOR, RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); -- cgit v1.2.3 From 6ff1cb88a7253ee33116626537a231da528bc57c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 2 Mar 2016 16:58:58 +0100 Subject: dmaengine: sirf: use __maybe_unused to hide pm functions The sirf dma driver uses #ifdef to check for CONFIG_PM_SLEEP for its suspend/resume code but then has no #ifdef for the respective runtime PM code, so we get a warning if CONFIG_PM is disabled altogether: drivers/dma/sirf-dma.c:1000:12: error: 'sirfsoc_dma_runtime_resume' defined but not used [-Werror=unused-function] This removes the existing #ifdef and instead uses __maybe_unused annotations for all four functions to let the compiler know it can silently drop the function definition. Signed-off-by: Arnd Bergmann Signed-off-by: Vinod Koul --- drivers/dma/sirf-dma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index 22ea2419ee56..e48350e65089 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -989,7 +989,7 @@ static int sirfsoc_dma_remove(struct platform_device *op) return 0; } -static int sirfsoc_dma_runtime_suspend(struct device *dev) +static int __maybe_unused sirfsoc_dma_runtime_suspend(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); @@ -997,7 +997,7 @@ static int sirfsoc_dma_runtime_suspend(struct device *dev) return 0; } -static int sirfsoc_dma_runtime_resume(struct device *dev) +static int __maybe_unused sirfsoc_dma_runtime_resume(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); int ret; @@ -1010,8 +1010,7 @@ static int sirfsoc_dma_runtime_resume(struct device *dev) return 0; } -#ifdef CONFIG_PM_SLEEP -static int sirfsoc_dma_pm_suspend(struct device *dev) +static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); struct sirfsoc_dma_regs *save = &sdma->regs_save; @@ -1062,7 +1061,7 @@ static int sirfsoc_dma_pm_suspend(struct device *dev) return 0; } -static int sirfsoc_dma_pm_resume(struct device *dev) +static int __maybe_unused sirfsoc_dma_pm_resume(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); struct sirfsoc_dma_regs *save = &sdma->regs_save; @@ -1121,7 +1120,6 @@ static int sirfsoc_dma_pm_resume(struct device *dev) return 0; } -#endif static const struct dev_pm_ops sirfsoc_dma_pm_ops = { SET_RUNTIME_PM_OPS(sirfsoc_dma_runtime_suspend, sirfsoc_dma_runtime_resume, NULL) -- cgit v1.2.3 From 694906348d0c3a0975a0d75f14ceb3239cfeb4e0 Mon Sep 17 00:00:00 2001 From: Kedareswara rao Appana Date: Thu, 3 Mar 2016 23:02:42 +0530 Subject: dmaengine: xilinx_vdma: Remove unnecessary variable initializations This patch removes the unnecessary variable initializations in the driver. Signed-off-by: Kedareswara rao Appana Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_vdma.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index bc2ca457247e..0ee0321868d3 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -554,7 +554,7 @@ static bool xilinx_vdma_is_idle(struct xilinx_vdma_chan *chan) */ static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan) { - int err = 0; + int err; u32 val; vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); @@ -579,7 +579,7 @@ static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan) */ static void xilinx_vdma_start(struct xilinx_vdma_chan *chan) { - int err = 0; + int err; u32 val; vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); @@ -756,7 +756,7 @@ static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan) */ static int xilinx_vdma_reset(struct xilinx_vdma_chan *chan) { - int err = 0; + int err; u32 tmp; vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RESET); -- cgit v1.2.3 From 242637bac7cd6e9e112d4d119e1ee19eafec3f2a Mon Sep 17 00:00:00 2001 From: Laxman Dewangan Date: Fri, 4 Mar 2016 15:55:11 +0530 Subject: dmaengine: tegra: Move of_device_id table near to its user After using the function of_device_get_match_data(), the of_device_id table for tegra20 dma is not used by probe() and hence moving it near to place where platform driver is defined as this table used only on this data structure. Signed-off-by: Laxman Dewangan Signed-off-by: Vinod Koul --- drivers/dma/tegra20-apb-dma.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index ced588f22f0d..3871f29e523d 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -1292,25 +1292,6 @@ static const struct tegra_dma_chip_data tegra148_dma_chip_data = { .support_separate_wcount_reg = true, }; - -static const struct of_device_id tegra_dma_of_match[] = { - { - .compatible = "nvidia,tegra148-apbdma", - .data = &tegra148_dma_chip_data, - }, { - .compatible = "nvidia,tegra114-apbdma", - .data = &tegra114_dma_chip_data, - }, { - .compatible = "nvidia,tegra30-apbdma", - .data = &tegra30_dma_chip_data, - }, { - .compatible = "nvidia,tegra20-apbdma", - .data = &tegra20_dma_chip_data, - }, { - }, -}; -MODULE_DEVICE_TABLE(of, tegra_dma_of_match); - static int tegra_dma_probe(struct platform_device *pdev) { struct resource *res; @@ -1610,6 +1591,24 @@ static const struct dev_pm_ops tegra_dma_dev_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume) }; +static const struct of_device_id tegra_dma_of_match[] = { + { + .compatible = "nvidia,tegra148-apbdma", + .data = &tegra148_dma_chip_data, + }, { + .compatible = "nvidia,tegra114-apbdma", + .data = &tegra114_dma_chip_data, + }, { + .compatible = "nvidia,tegra30-apbdma", + .data = &tegra30_dma_chip_data, + }, { + .compatible = "nvidia,tegra20-apbdma", + .data = &tegra20_dma_chip_data, + }, { + }, +}; +MODULE_DEVICE_TABLE(of, tegra_dma_of_match); + static struct platform_driver tegra_dmac_driver = { .driver = { .name = "tegra-apbdma", -- cgit v1.2.3 From d9b31efcbf41cd249419bca82065976ca44e3ae9 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 4 Feb 2016 23:34:32 -0500 Subject: dmaengine: qcom_bam_dma: move to qcom directory Creating a QCOM directory for all QCOM DMA source files. Signed-off-by: Sinan Kaya Reviewed-by: Andy Gross Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 11 +- drivers/dma/Makefile | 2 +- drivers/dma/qcom/Kconfig | 8 + drivers/dma/qcom/Makefile | 1 + drivers/dma/qcom/bam_dma.c | 1262 ++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/qcom_bam_dma.c | 1262 -------------------------------------------- 6 files changed, 1274 insertions(+), 1272 deletions(-) create mode 100644 drivers/dma/qcom/Kconfig create mode 100644 drivers/dma/qcom/Makefile create mode 100644 drivers/dma/qcom/bam_dma.c delete mode 100644 drivers/dma/qcom_bam_dma.c (limited to 'drivers/dma') diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 79b1390f2016..245b06152e6f 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -408,15 +408,6 @@ config PXA_DMA 16 to 32 channels for peripheral to memory or memory to memory transfers. -config QCOM_BAM_DMA - tristate "QCOM BAM DMA support" - depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - ---help--- - Enable support for the QCOM BAM DMA controller. This controller - provides DMA capabilities for a variety of on-chip devices. - config SIRF_DMA tristate "CSR SiRFprimaII/SiRFmarco DMA support" depends on ARCH_SIRF @@ -539,6 +530,8 @@ config ZX_DMA # driver files source "drivers/dma/bestcomm/Kconfig" +source "drivers/dma/qcom/Kconfig" + source "drivers/dma/dw/Kconfig" source "drivers/dma/hsu/Kconfig" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 2dd0a067a0ca..6084127c1486 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -52,7 +52,6 @@ obj-$(CONFIG_PCH_DMA) += pch_dma.o obj-$(CONFIG_PL330_DMA) += pl330.o obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o -obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o @@ -67,4 +66,5 @@ obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx296702_dma.o +obj-y += qcom/ obj-y += xilinx/ diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig new file mode 100644 index 000000000000..f17c27288808 --- /dev/null +++ b/drivers/dma/qcom/Kconfig @@ -0,0 +1,8 @@ +config QCOM_BAM_DMA + tristate "QCOM BAM DMA support" + depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + ---help--- + Enable support for the QCOM BAM DMA controller. This controller + provides DMA capabilities for a variety of on-chip devices. diff --git a/drivers/dma/qcom/Makefile b/drivers/dma/qcom/Makefile new file mode 100644 index 000000000000..f612ae303ee0 --- /dev/null +++ b/drivers/dma/qcom/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c new file mode 100644 index 000000000000..2d691a34a0ab --- /dev/null +++ b/drivers/dma/qcom/bam_dma.c @@ -0,0 +1,1262 @@ +/* + * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +/* + * QCOM BAM DMA engine driver + * + * QCOM BAM DMA blocks are distributed amongst a number of the on-chip + * peripherals on the MSM 8x74. The configuration of the channels are dependent + * on the way they are hard wired to that specific peripheral. The peripheral + * device tree entries specify the configuration of each channel. + * + * The DMA controller requires the use of external memory for storage of the + * hardware descriptors for each channel. The descriptor FIFO is accessed as a + * circular buffer and operations are managed according to the offset within the + * FIFO. After pipe/channel reset, all of the pipe registers and internal state + * are back to defaults. + * + * During DMA operations, we write descriptors to the FIFO, being careful to + * handle wrapping and then write the last FIFO offset to that channel's + * P_EVNT_REG register to kick off the transaction. The P_SW_OFSTS register + * indicates the current FIFO offset that is being processed, so there is some + * indication of where the hardware is currently working. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "../virt-dma.h" + +struct bam_desc_hw { + __le32 addr; /* Buffer physical address */ + __le16 size; /* Buffer size in bytes */ + __le16 flags; +}; + +#define DESC_FLAG_INT BIT(15) +#define DESC_FLAG_EOT BIT(14) +#define DESC_FLAG_EOB BIT(13) +#define DESC_FLAG_NWD BIT(12) + +struct bam_async_desc { + struct virt_dma_desc vd; + + u32 num_desc; + u32 xfer_len; + + /* transaction flags, EOT|EOB|NWD */ + u16 flags; + + struct bam_desc_hw *curr_desc; + + enum dma_transfer_direction dir; + size_t length; + struct bam_desc_hw desc[0]; +}; + +enum bam_reg { + BAM_CTRL, + BAM_REVISION, + BAM_NUM_PIPES, + BAM_DESC_CNT_TRSHLD, + BAM_IRQ_SRCS, + BAM_IRQ_SRCS_MSK, + BAM_IRQ_SRCS_UNMASKED, + BAM_IRQ_STTS, + BAM_IRQ_CLR, + BAM_IRQ_EN, + BAM_CNFG_BITS, + BAM_IRQ_SRCS_EE, + BAM_IRQ_SRCS_MSK_EE, + BAM_P_CTRL, + BAM_P_RST, + BAM_P_HALT, + BAM_P_IRQ_STTS, + BAM_P_IRQ_CLR, + BAM_P_IRQ_EN, + BAM_P_EVNT_DEST_ADDR, + BAM_P_EVNT_REG, + BAM_P_SW_OFSTS, + BAM_P_DATA_FIFO_ADDR, + BAM_P_DESC_FIFO_ADDR, + BAM_P_EVNT_GEN_TRSHLD, + BAM_P_FIFO_SIZES, +}; + +struct reg_offset_data { + u32 base_offset; + unsigned int pipe_mult, evnt_mult, ee_mult; +}; + +static const struct reg_offset_data bam_v1_3_reg_info[] = { + [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x0FBC, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x0F88, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x0F8C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x0F90, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x0F94, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x0F98, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x0F9C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x0FFC, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x1800, 0x00, 0x00, 0x80 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x1804, 0x00, 0x00, 0x80 }, + [BAM_P_CTRL] = { 0x0000, 0x80, 0x00, 0x00 }, + [BAM_P_RST] = { 0x0004, 0x80, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x0008, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x0010, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x0014, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x0018, 0x80, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x40, 0x00 }, + [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x40, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x40, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x1024, 0x00, 0x40, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x101C, 0x00, 0x40, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 }, +}; + +static const struct reg_offset_data bam_v1_4_reg_info[] = { + [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x003C, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x000C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x0010, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x0800, 0x00, 0x00, 0x80 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x0804, 0x00, 0x00, 0x80 }, + [BAM_P_CTRL] = { 0x1000, 0x1000, 0x00, 0x00 }, + [BAM_P_RST] = { 0x1004, 0x1000, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x1008, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x1010, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x1014, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x1018, 0x1000, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x182C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_REG] = { 0x1818, 0x00, 0x1000, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x1800, 0x00, 0x1000, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x1824, 0x00, 0x1000, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x181C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 }, +}; + +static const struct reg_offset_data bam_v1_7_reg_info[] = { + [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x01008, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x00008, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x03010, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x03014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x00014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x00018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x0001C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x0007C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x03000, 0x00, 0x00, 0x1000 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x03004, 0x00, 0x00, 0x1000 }, + [BAM_P_CTRL] = { 0x13000, 0x1000, 0x00, 0x00 }, + [BAM_P_RST] = { 0x13004, 0x1000, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x13008, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x13010, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x13014, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x13018, 0x1000, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x1382C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_REG] = { 0x13818, 0x00, 0x1000, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x13800, 0x00, 0x1000, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x13824, 0x00, 0x1000, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x1381C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 }, +}; + +/* BAM CTRL */ +#define BAM_SW_RST BIT(0) +#define BAM_EN BIT(1) +#define BAM_EN_ACCUM BIT(4) +#define BAM_TESTBUS_SEL_SHIFT 5 +#define BAM_TESTBUS_SEL_MASK 0x3F +#define BAM_DESC_CACHE_SEL_SHIFT 13 +#define BAM_DESC_CACHE_SEL_MASK 0x3 +#define BAM_CACHED_DESC_STORE BIT(15) +#define IBC_DISABLE BIT(16) + +/* BAM REVISION */ +#define REVISION_SHIFT 0 +#define REVISION_MASK 0xFF +#define NUM_EES_SHIFT 8 +#define NUM_EES_MASK 0xF +#define CE_BUFFER_SIZE BIT(13) +#define AXI_ACTIVE BIT(14) +#define USE_VMIDMT BIT(15) +#define SECURED BIT(16) +#define BAM_HAS_NO_BYPASS BIT(17) +#define HIGH_FREQUENCY_BAM BIT(18) +#define INACTIV_TMRS_EXST BIT(19) +#define NUM_INACTIV_TMRS BIT(20) +#define DESC_CACHE_DEPTH_SHIFT 21 +#define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT) +#define CMD_DESC_EN BIT(23) +#define INACTIV_TMR_BASE_SHIFT 24 +#define INACTIV_TMR_BASE_MASK 0xFF + +/* BAM NUM PIPES */ +#define BAM_NUM_PIPES_SHIFT 0 +#define BAM_NUM_PIPES_MASK 0xFF +#define PERIPH_NON_PIPE_GRP_SHIFT 16 +#define PERIPH_NON_PIP_GRP_MASK 0xFF +#define BAM_NON_PIPE_GRP_SHIFT 24 +#define BAM_NON_PIPE_GRP_MASK 0xFF + +/* BAM CNFG BITS */ +#define BAM_PIPE_CNFG BIT(2) +#define BAM_FULL_PIPE BIT(11) +#define BAM_NO_EXT_P_RST BIT(12) +#define BAM_IBC_DISABLE BIT(13) +#define BAM_SB_CLK_REQ BIT(14) +#define BAM_PSM_CSW_REQ BIT(15) +#define BAM_PSM_P_RES BIT(16) +#define BAM_AU_P_RES BIT(17) +#define BAM_SI_P_RES BIT(18) +#define BAM_WB_P_RES BIT(19) +#define BAM_WB_BLK_CSW BIT(20) +#define BAM_WB_CSW_ACK_IDL BIT(21) +#define BAM_WB_RETR_SVPNT BIT(22) +#define BAM_WB_DSC_AVL_P_RST BIT(23) +#define BAM_REG_P_EN BIT(24) +#define BAM_PSM_P_HD_DATA BIT(25) +#define BAM_AU_ACCUMED BIT(26) +#define BAM_CMD_ENABLE BIT(27) + +#define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \ + BAM_NO_EXT_P_RST | \ + BAM_IBC_DISABLE | \ + BAM_SB_CLK_REQ | \ + BAM_PSM_CSW_REQ | \ + BAM_PSM_P_RES | \ + BAM_AU_P_RES | \ + BAM_SI_P_RES | \ + BAM_WB_P_RES | \ + BAM_WB_BLK_CSW | \ + BAM_WB_CSW_ACK_IDL | \ + BAM_WB_RETR_SVPNT | \ + BAM_WB_DSC_AVL_P_RST | \ + BAM_REG_P_EN | \ + BAM_PSM_P_HD_DATA | \ + BAM_AU_ACCUMED | \ + BAM_CMD_ENABLE) + +/* PIPE CTRL */ +#define P_EN BIT(1) +#define P_DIRECTION BIT(3) +#define P_SYS_STRM BIT(4) +#define P_SYS_MODE BIT(5) +#define P_AUTO_EOB BIT(6) +#define P_AUTO_EOB_SEL_SHIFT 7 +#define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT) +#define P_PREFETCH_LIMIT_SHIFT 9 +#define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT) +#define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT) +#define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT) +#define P_WRITE_NWD BIT(11) +#define P_LOCK_GROUP_SHIFT 16 +#define P_LOCK_GROUP_MASK 0x1F + +/* BAM_DESC_CNT_TRSHLD */ +#define CNT_TRSHLD 0xffff +#define DEFAULT_CNT_THRSHLD 0x4 + +/* BAM_IRQ_SRCS */ +#define BAM_IRQ BIT(31) +#define P_IRQ 0x7fffffff + +/* BAM_IRQ_SRCS_MSK */ +#define BAM_IRQ_MSK BAM_IRQ +#define P_IRQ_MSK P_IRQ + +/* BAM_IRQ_STTS */ +#define BAM_TIMER_IRQ BIT(4) +#define BAM_EMPTY_IRQ BIT(3) +#define BAM_ERROR_IRQ BIT(2) +#define BAM_HRESP_ERR_IRQ BIT(1) + +/* BAM_IRQ_CLR */ +#define BAM_TIMER_CLR BIT(4) +#define BAM_EMPTY_CLR BIT(3) +#define BAM_ERROR_CLR BIT(2) +#define BAM_HRESP_ERR_CLR BIT(1) + +/* BAM_IRQ_EN */ +#define BAM_TIMER_EN BIT(4) +#define BAM_EMPTY_EN BIT(3) +#define BAM_ERROR_EN BIT(2) +#define BAM_HRESP_ERR_EN BIT(1) + +/* BAM_P_IRQ_EN */ +#define P_PRCSD_DESC_EN BIT(0) +#define P_TIMER_EN BIT(1) +#define P_WAKE_EN BIT(2) +#define P_OUT_OF_DESC_EN BIT(3) +#define P_ERR_EN BIT(4) +#define P_TRNSFR_END_EN BIT(5) +#define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN) + +/* BAM_P_SW_OFSTS */ +#define P_SW_OFSTS_MASK 0xffff + +#define BAM_DESC_FIFO_SIZE SZ_32K +#define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1) +#define BAM_MAX_DATA_SIZE (SZ_32K - 8) + +struct bam_chan { + struct virt_dma_chan vc; + + struct bam_device *bdev; + + /* configuration from device tree */ + u32 id; + + struct bam_async_desc *curr_txd; /* current running dma */ + + /* runtime configuration */ + struct dma_slave_config slave; + + /* fifo storage */ + struct bam_desc_hw *fifo_virt; + dma_addr_t fifo_phys; + + /* fifo markers */ + unsigned short head; /* start of active descriptor entries */ + unsigned short tail; /* end of active descriptor entries */ + + unsigned int initialized; /* is the channel hw initialized? */ + unsigned int paused; /* is the channel paused? */ + unsigned int reconfigure; /* new slave config? */ + + struct list_head node; +}; + +static inline struct bam_chan *to_bam_chan(struct dma_chan *common) +{ + return container_of(common, struct bam_chan, vc.chan); +} + +struct bam_device { + void __iomem *regs; + struct device *dev; + struct dma_device common; + struct device_dma_parameters dma_parms; + struct bam_chan *channels; + u32 num_channels; + + /* execution environment ID, from DT */ + u32 ee; + + const struct reg_offset_data *layout; + + struct clk *bamclk; + int irq; + + /* dma start transaction tasklet */ + struct tasklet_struct task; +}; + +/** + * bam_addr - returns BAM register address + * @bdev: bam device + * @pipe: pipe instance (ignored when register doesn't have multiple instances) + * @reg: register enum + */ +static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe, + enum bam_reg reg) +{ + const struct reg_offset_data r = bdev->layout[reg]; + + return bdev->regs + r.base_offset + + r.pipe_mult * pipe + + r.evnt_mult * pipe + + r.ee_mult * bdev->ee; +} + +/** + * bam_reset_channel - Reset individual BAM DMA channel + * @bchan: bam channel + * + * This function resets a specific BAM channel + */ +static void bam_reset_channel(struct bam_chan *bchan) +{ + struct bam_device *bdev = bchan->bdev; + + lockdep_assert_held(&bchan->vc.lock); + + /* reset channel */ + writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST)); + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST)); + + /* don't allow cpu to reorder BAM register accesses done after this */ + wmb(); + + /* make sure hw is initialized when channel is used the first time */ + bchan->initialized = 0; +} + +/** + * bam_chan_init_hw - Initialize channel hardware + * @bchan: bam channel + * + * This function resets and initializes the BAM channel + */ +static void bam_chan_init_hw(struct bam_chan *bchan, + enum dma_transfer_direction dir) +{ + struct bam_device *bdev = bchan->bdev; + u32 val; + + /* Reset the channel to clear internal state of the FIFO */ + bam_reset_channel(bchan); + + /* + * write out 8 byte aligned address. We have enough space for this + * because we allocated 1 more descriptor (8 bytes) than we can use + */ + writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)), + bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR)); + writel_relaxed(BAM_DESC_FIFO_SIZE, + bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES)); + + /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */ + writel_relaxed(P_DEFAULT_IRQS_EN, + bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); + + /* unmask the specific pipe and EE combo */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + val |= BIT(bchan->id); + writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + + /* don't allow cpu to reorder the channel enable done below */ + wmb(); + + /* set fixed direction and mode, then enable channel */ + val = P_EN | P_SYS_MODE; + if (dir == DMA_DEV_TO_MEM) + val |= P_DIRECTION; + + writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL)); + + bchan->initialized = 1; + + /* init FIFO pointers */ + bchan->head = 0; + bchan->tail = 0; +} + +/** + * bam_alloc_chan - Allocate channel resources for DMA channel. + * @chan: specified channel + * + * This function allocates the FIFO descriptor memory + */ +static int bam_alloc_chan(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + + if (bchan->fifo_virt) + return 0; + + /* allocate FIFO descriptor space, but only if necessary */ + bchan->fifo_virt = dma_alloc_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, + &bchan->fifo_phys, GFP_KERNEL); + + if (!bchan->fifo_virt) { + dev_err(bdev->dev, "Failed to allocate desc fifo\n"); + return -ENOMEM; + } + + return 0; +} + +/** + * bam_free_chan - Frees dma resources associated with specific channel + * @chan: specified channel + * + * Free the allocated fifo descriptor memory and channel resources + * + */ +static void bam_free_chan(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + u32 val; + unsigned long flags; + + vchan_free_chan_resources(to_virt_chan(chan)); + + if (bchan->curr_txd) { + dev_err(bchan->bdev->dev, "Cannot free busy channel\n"); + return; + } + + spin_lock_irqsave(&bchan->vc.lock, flags); + bam_reset_channel(bchan); + spin_unlock_irqrestore(&bchan->vc.lock, flags); + + dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt, + bchan->fifo_phys); + bchan->fifo_virt = NULL; + + /* mask irq for pipe/channel */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + val &= ~BIT(bchan->id); + writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + + /* disable irq */ + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); +} + +/** + * bam_slave_config - set slave configuration for channel + * @chan: dma channel + * @cfg: slave configuration + * + * Sets slave configuration for channel + * + */ +static int bam_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct bam_chan *bchan = to_bam_chan(chan); + unsigned long flag; + + spin_lock_irqsave(&bchan->vc.lock, flag); + memcpy(&bchan->slave, cfg, sizeof(*cfg)); + bchan->reconfigure = 1; + spin_unlock_irqrestore(&bchan->vc.lock, flag); + + return 0; +} + +/** + * bam_prep_slave_sg - Prep slave sg transaction + * + * @chan: dma channel + * @sgl: scatter gather list + * @sg_len: length of sg + * @direction: DMA transfer direction + * @flags: DMA flags + * @context: transfer context (unused) + */ +static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction direction, unsigned long flags, + void *context) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + struct bam_async_desc *async_desc; + struct scatterlist *sg; + u32 i; + struct bam_desc_hw *desc; + unsigned int num_alloc = 0; + + + if (!is_slave_direction(direction)) { + dev_err(bdev->dev, "invalid dma direction\n"); + return NULL; + } + + /* calculate number of required entries */ + for_each_sg(sgl, sg, sg_len, i) + num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_MAX_DATA_SIZE); + + /* allocate enough room to accomodate the number of entries */ + async_desc = kzalloc(sizeof(*async_desc) + + (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT); + + if (!async_desc) + goto err_out; + + if (flags & DMA_PREP_FENCE) + async_desc->flags |= DESC_FLAG_NWD; + + if (flags & DMA_PREP_INTERRUPT) + async_desc->flags |= DESC_FLAG_EOT; + else + async_desc->flags |= DESC_FLAG_INT; + + async_desc->num_desc = num_alloc; + async_desc->curr_desc = async_desc->desc; + async_desc->dir = direction; + + /* fill in temporary descriptors */ + desc = async_desc->desc; + for_each_sg(sgl, sg, sg_len, i) { + unsigned int remainder = sg_dma_len(sg); + unsigned int curr_offset = 0; + + do { + desc->addr = cpu_to_le32(sg_dma_address(sg) + + curr_offset); + + if (remainder > BAM_MAX_DATA_SIZE) { + desc->size = cpu_to_le16(BAM_MAX_DATA_SIZE); + remainder -= BAM_MAX_DATA_SIZE; + curr_offset += BAM_MAX_DATA_SIZE; + } else { + desc->size = cpu_to_le16(remainder); + remainder = 0; + } + + async_desc->length += desc->size; + desc++; + } while (remainder > 0); + } + + return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags); + +err_out: + kfree(async_desc); + return NULL; +} + +/** + * bam_dma_terminate_all - terminate all transactions on a channel + * @bchan: bam dma channel + * + * Dequeues and frees all transactions + * No callbacks are done + * + */ +static int bam_dma_terminate_all(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + unsigned long flag; + LIST_HEAD(head); + + /* remove all transactions, including active transaction */ + spin_lock_irqsave(&bchan->vc.lock, flag); + if (bchan->curr_txd) { + list_add(&bchan->curr_txd->vd.node, &bchan->vc.desc_issued); + bchan->curr_txd = NULL; + } + + vchan_get_all_descriptors(&bchan->vc, &head); + spin_unlock_irqrestore(&bchan->vc.lock, flag); + + vchan_dma_desc_free_list(&bchan->vc, &head); + + return 0; +} + +/** + * bam_pause - Pause DMA channel + * @chan: dma channel + * + */ +static int bam_pause(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + unsigned long flag; + + spin_lock_irqsave(&bchan->vc.lock, flag); + writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT)); + bchan->paused = 1; + spin_unlock_irqrestore(&bchan->vc.lock, flag); + + return 0; +} + +/** + * bam_resume - Resume DMA channel operations + * @chan: dma channel + * + */ +static int bam_resume(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + unsigned long flag; + + spin_lock_irqsave(&bchan->vc.lock, flag); + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT)); + bchan->paused = 0; + spin_unlock_irqrestore(&bchan->vc.lock, flag); + + return 0; +} + +/** + * process_channel_irqs - processes the channel interrupts + * @bdev: bam controller + * + * This function processes the channel interrupts + * + */ +static u32 process_channel_irqs(struct bam_device *bdev) +{ + u32 i, srcs, pipe_stts; + unsigned long flags; + struct bam_async_desc *async_desc; + + srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE)); + + /* return early if no pipe/channel interrupts are present */ + if (!(srcs & P_IRQ)) + return srcs; + + for (i = 0; i < bdev->num_channels; i++) { + struct bam_chan *bchan = &bdev->channels[i]; + + if (!(srcs & BIT(i))) + continue; + + /* clear pipe irq */ + pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS)); + + writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR)); + + spin_lock_irqsave(&bchan->vc.lock, flags); + async_desc = bchan->curr_txd; + + if (async_desc) { + async_desc->num_desc -= async_desc->xfer_len; + async_desc->curr_desc += async_desc->xfer_len; + bchan->curr_txd = NULL; + + /* manage FIFO */ + bchan->head += async_desc->xfer_len; + bchan->head %= MAX_DESCRIPTORS; + + /* + * if complete, process cookie. Otherwise + * push back to front of desc_issued so that + * it gets restarted by the tasklet + */ + if (!async_desc->num_desc) + vchan_cookie_complete(&async_desc->vd); + else + list_add(&async_desc->vd.node, + &bchan->vc.desc_issued); + } + + spin_unlock_irqrestore(&bchan->vc.lock, flags); + } + + return srcs; +} + +/** + * bam_dma_irq - irq handler for bam controller + * @irq: IRQ of interrupt + * @data: callback data + * + * IRQ handler for the bam controller + */ +static irqreturn_t bam_dma_irq(int irq, void *data) +{ + struct bam_device *bdev = data; + u32 clr_mask = 0, srcs = 0; + + srcs |= process_channel_irqs(bdev); + + /* kick off tasklet to start next dma transfer */ + if (srcs & P_IRQ) + tasklet_schedule(&bdev->task); + + if (srcs & BAM_IRQ) + clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS)); + + /* don't allow reorder of the various accesses to the BAM registers */ + mb(); + + writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR)); + + return IRQ_HANDLED; +} + +/** + * bam_tx_status - returns status of transaction + * @chan: dma channel + * @cookie: transaction cookie + * @txstate: DMA transaction state + * + * Return status of dma transaction + */ +static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct virt_dma_desc *vd; + int ret; + size_t residue = 0; + unsigned int i; + unsigned long flags; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (!txstate) + return bchan->paused ? DMA_PAUSED : ret; + + spin_lock_irqsave(&bchan->vc.lock, flags); + vd = vchan_find_desc(&bchan->vc, cookie); + if (vd) + residue = container_of(vd, struct bam_async_desc, vd)->length; + else if (bchan->curr_txd && bchan->curr_txd->vd.tx.cookie == cookie) + for (i = 0; i < bchan->curr_txd->num_desc; i++) + residue += bchan->curr_txd->curr_desc[i].size; + + spin_unlock_irqrestore(&bchan->vc.lock, flags); + + dma_set_residue(txstate, residue); + + if (ret == DMA_IN_PROGRESS && bchan->paused) + ret = DMA_PAUSED; + + return ret; +} + +/** + * bam_apply_new_config + * @bchan: bam dma channel + * @dir: DMA direction + */ +static void bam_apply_new_config(struct bam_chan *bchan, + enum dma_transfer_direction dir) +{ + struct bam_device *bdev = bchan->bdev; + u32 maxburst; + + if (dir == DMA_DEV_TO_MEM) + maxburst = bchan->slave.src_maxburst; + else + maxburst = bchan->slave.dst_maxburst; + + writel_relaxed(maxburst, bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); + + bchan->reconfigure = 0; +} + +/** + * bam_start_dma - start next transaction + * @bchan - bam dma channel + */ +static void bam_start_dma(struct bam_chan *bchan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc); + struct bam_device *bdev = bchan->bdev; + struct bam_async_desc *async_desc; + struct bam_desc_hw *desc; + struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt, + sizeof(struct bam_desc_hw)); + + lockdep_assert_held(&bchan->vc.lock); + + if (!vd) + return; + + list_del(&vd->node); + + async_desc = container_of(vd, struct bam_async_desc, vd); + bchan->curr_txd = async_desc; + + /* on first use, initialize the channel hardware */ + if (!bchan->initialized) + bam_chan_init_hw(bchan, async_desc->dir); + + /* apply new slave config changes, if necessary */ + if (bchan->reconfigure) + bam_apply_new_config(bchan, async_desc->dir); + + desc = bchan->curr_txd->curr_desc; + + if (async_desc->num_desc > MAX_DESCRIPTORS) + async_desc->xfer_len = MAX_DESCRIPTORS; + else + async_desc->xfer_len = async_desc->num_desc; + + /* set any special flags on the last descriptor */ + if (async_desc->num_desc == async_desc->xfer_len) + desc[async_desc->xfer_len - 1].flags = + cpu_to_le16(async_desc->flags); + else + desc[async_desc->xfer_len - 1].flags |= + cpu_to_le16(DESC_FLAG_INT); + + if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) { + u32 partial = MAX_DESCRIPTORS - bchan->tail; + + memcpy(&fifo[bchan->tail], desc, + partial * sizeof(struct bam_desc_hw)); + memcpy(fifo, &desc[partial], (async_desc->xfer_len - partial) * + sizeof(struct bam_desc_hw)); + } else { + memcpy(&fifo[bchan->tail], desc, + async_desc->xfer_len * sizeof(struct bam_desc_hw)); + } + + bchan->tail += async_desc->xfer_len; + bchan->tail %= MAX_DESCRIPTORS; + + /* ensure descriptor writes and dma start not reordered */ + wmb(); + writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), + bam_addr(bdev, bchan->id, BAM_P_EVNT_REG)); +} + +/** + * dma_tasklet - DMA IRQ tasklet + * @data: tasklet argument (bam controller structure) + * + * Sets up next DMA operation and then processes all completed transactions + */ +static void dma_tasklet(unsigned long data) +{ + struct bam_device *bdev = (struct bam_device *)data; + struct bam_chan *bchan; + unsigned long flags; + unsigned int i; + + /* go through the channels and kick off transactions */ + for (i = 0; i < bdev->num_channels; i++) { + bchan = &bdev->channels[i]; + spin_lock_irqsave(&bchan->vc.lock, flags); + + if (!list_empty(&bchan->vc.desc_issued) && !bchan->curr_txd) + bam_start_dma(bchan); + spin_unlock_irqrestore(&bchan->vc.lock, flags); + } +} + +/** + * bam_issue_pending - starts pending transactions + * @chan: dma channel + * + * Calls tasklet directly which in turn starts any pending transactions + */ +static void bam_issue_pending(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&bchan->vc.lock, flags); + + /* if work pending and idle, start a transaction */ + if (vchan_issue_pending(&bchan->vc) && !bchan->curr_txd) + bam_start_dma(bchan); + + spin_unlock_irqrestore(&bchan->vc.lock, flags); +} + +/** + * bam_dma_free_desc - free descriptor memory + * @vd: virtual descriptor + * + */ +static void bam_dma_free_desc(struct virt_dma_desc *vd) +{ + struct bam_async_desc *async_desc = container_of(vd, + struct bam_async_desc, vd); + + kfree(async_desc); +} + +static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of) +{ + struct bam_device *bdev = container_of(of->of_dma_data, + struct bam_device, common); + unsigned int request; + + if (dma_spec->args_count != 1) + return NULL; + + request = dma_spec->args[0]; + if (request >= bdev->num_channels) + return NULL; + + return dma_get_slave_channel(&(bdev->channels[request].vc.chan)); +} + +/** + * bam_init + * @bdev: bam device + * + * Initialization helper for global bam registers + */ +static int bam_init(struct bam_device *bdev) +{ + u32 val; + + /* read revision and configuration information */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)) >> NUM_EES_SHIFT; + val &= NUM_EES_MASK; + + /* check that configured EE is within range */ + if (bdev->ee >= val) + return -EINVAL; + + val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES)); + bdev->num_channels = val & BAM_NUM_PIPES_MASK; + + /* s/w reset bam */ + /* after reset all pipes are disabled and idle */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); + val |= BAM_SW_RST; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + val &= ~BAM_SW_RST; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + + /* make sure previous stores are visible before enabling BAM */ + wmb(); + + /* enable bam */ + val |= BAM_EN; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + + /* set descriptor threshhold, start with 4 bytes */ + writel_relaxed(DEFAULT_CNT_THRSHLD, + bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); + + /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ + writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); + + /* enable irqs for errors */ + writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, + bam_addr(bdev, 0, BAM_IRQ_EN)); + + /* unmask global bam interrupt */ + writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + + return 0; +} + +static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, + u32 index) +{ + bchan->id = index; + bchan->bdev = bdev; + + vchan_init(&bchan->vc, &bdev->common); + bchan->vc.desc_free = bam_dma_free_desc; +} + +static const struct of_device_id bam_of_match[] = { + { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, + { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, + { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info }, + {} +}; + +MODULE_DEVICE_TABLE(of, bam_of_match); + +static int bam_dma_probe(struct platform_device *pdev) +{ + struct bam_device *bdev; + const struct of_device_id *match; + struct resource *iores; + int ret, i; + + bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL); + if (!bdev) + return -ENOMEM; + + bdev->dev = &pdev->dev; + + match = of_match_node(bam_of_match, pdev->dev.of_node); + if (!match) { + dev_err(&pdev->dev, "Unsupported BAM module\n"); + return -ENODEV; + } + + bdev->layout = match->data; + + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + bdev->regs = devm_ioremap_resource(&pdev->dev, iores); + if (IS_ERR(bdev->regs)) + return PTR_ERR(bdev->regs); + + bdev->irq = platform_get_irq(pdev, 0); + if (bdev->irq < 0) + return bdev->irq; + + ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee); + if (ret) { + dev_err(bdev->dev, "Execution environment unspecified\n"); + return ret; + } + + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); + + ret = clk_prepare_enable(bdev->bamclk); + if (ret) { + dev_err(bdev->dev, "failed to prepare/enable clock\n"); + return ret; + } + + ret = bam_init(bdev); + if (ret) + goto err_disable_clk; + + tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev); + + bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels, + sizeof(*bdev->channels), GFP_KERNEL); + + if (!bdev->channels) { + ret = -ENOMEM; + goto err_tasklet_kill; + } + + /* allocate and initialize channels */ + INIT_LIST_HEAD(&bdev->common.channels); + + for (i = 0; i < bdev->num_channels; i++) + bam_channel_init(bdev, &bdev->channels[i], i); + + ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, + IRQF_TRIGGER_HIGH, "bam_dma", bdev); + if (ret) + goto err_bam_channel_exit; + + /* set max dma segment size */ + bdev->common.dev = bdev->dev; + bdev->common.dev->dma_parms = &bdev->dma_parms; + ret = dma_set_max_seg_size(bdev->common.dev, BAM_MAX_DATA_SIZE); + if (ret) { + dev_err(bdev->dev, "cannot set maximum segment size\n"); + goto err_bam_channel_exit; + } + + platform_set_drvdata(pdev, bdev); + + /* set capabilities */ + dma_cap_zero(bdev->common.cap_mask); + dma_cap_set(DMA_SLAVE, bdev->common.cap_mask); + + /* initialize dmaengine apis */ + bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + bdev->common.device_alloc_chan_resources = bam_alloc_chan; + bdev->common.device_free_chan_resources = bam_free_chan; + bdev->common.device_prep_slave_sg = bam_prep_slave_sg; + bdev->common.device_config = bam_slave_config; + bdev->common.device_pause = bam_pause; + bdev->common.device_resume = bam_resume; + bdev->common.device_terminate_all = bam_dma_terminate_all; + bdev->common.device_issue_pending = bam_issue_pending; + bdev->common.device_tx_status = bam_tx_status; + bdev->common.dev = bdev->dev; + + ret = dma_async_device_register(&bdev->common); + if (ret) { + dev_err(bdev->dev, "failed to register dma async device\n"); + goto err_bam_channel_exit; + } + + ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, + &bdev->common); + if (ret) + goto err_unregister_dma; + + return 0; + +err_unregister_dma: + dma_async_device_unregister(&bdev->common); +err_bam_channel_exit: + for (i = 0; i < bdev->num_channels; i++) + tasklet_kill(&bdev->channels[i].vc.task); +err_tasklet_kill: + tasklet_kill(&bdev->task); +err_disable_clk: + clk_disable_unprepare(bdev->bamclk); + + return ret; +} + +static int bam_dma_remove(struct platform_device *pdev) +{ + struct bam_device *bdev = platform_get_drvdata(pdev); + u32 i; + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&bdev->common); + + /* mask all interrupts for this execution environment */ + writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + + devm_free_irq(bdev->dev, bdev->irq, bdev); + + for (i = 0; i < bdev->num_channels; i++) { + bam_dma_terminate_all(&bdev->channels[i].vc.chan); + tasklet_kill(&bdev->channels[i].vc.task); + + dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, + bdev->channels[i].fifo_virt, + bdev->channels[i].fifo_phys); + } + + tasklet_kill(&bdev->task); + + clk_disable_unprepare(bdev->bamclk); + + return 0; +} + +static struct platform_driver bam_dma_driver = { + .probe = bam_dma_probe, + .remove = bam_dma_remove, + .driver = { + .name = "bam-dma-engine", + .of_match_table = bam_of_match, + }, +}; + +module_platform_driver(bam_dma_driver); + +MODULE_AUTHOR("Andy Gross "); +MODULE_DESCRIPTION("QCOM BAM DMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom_bam_dma.c b/drivers/dma/qcom_bam_dma.c deleted file mode 100644 index 37f7aeca7fe1..000000000000 --- a/drivers/dma/qcom_bam_dma.c +++ /dev/null @@ -1,1262 +0,0 @@ -/* - * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ -/* - * QCOM BAM DMA engine driver - * - * QCOM BAM DMA blocks are distributed amongst a number of the on-chip - * peripherals on the MSM 8x74. The configuration of the channels are dependent - * on the way they are hard wired to that specific peripheral. The peripheral - * device tree entries specify the configuration of each channel. - * - * The DMA controller requires the use of external memory for storage of the - * hardware descriptors for each channel. The descriptor FIFO is accessed as a - * circular buffer and operations are managed according to the offset within the - * FIFO. After pipe/channel reset, all of the pipe registers and internal state - * are back to defaults. - * - * During DMA operations, we write descriptors to the FIFO, being careful to - * handle wrapping and then write the last FIFO offset to that channel's - * P_EVNT_REG register to kick off the transaction. The P_SW_OFSTS register - * indicates the current FIFO offset that is being processed, so there is some - * indication of where the hardware is currently working. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dmaengine.h" -#include "virt-dma.h" - -struct bam_desc_hw { - __le32 addr; /* Buffer physical address */ - __le16 size; /* Buffer size in bytes */ - __le16 flags; -}; - -#define DESC_FLAG_INT BIT(15) -#define DESC_FLAG_EOT BIT(14) -#define DESC_FLAG_EOB BIT(13) -#define DESC_FLAG_NWD BIT(12) - -struct bam_async_desc { - struct virt_dma_desc vd; - - u32 num_desc; - u32 xfer_len; - - /* transaction flags, EOT|EOB|NWD */ - u16 flags; - - struct bam_desc_hw *curr_desc; - - enum dma_transfer_direction dir; - size_t length; - struct bam_desc_hw desc[0]; -}; - -enum bam_reg { - BAM_CTRL, - BAM_REVISION, - BAM_NUM_PIPES, - BAM_DESC_CNT_TRSHLD, - BAM_IRQ_SRCS, - BAM_IRQ_SRCS_MSK, - BAM_IRQ_SRCS_UNMASKED, - BAM_IRQ_STTS, - BAM_IRQ_CLR, - BAM_IRQ_EN, - BAM_CNFG_BITS, - BAM_IRQ_SRCS_EE, - BAM_IRQ_SRCS_MSK_EE, - BAM_P_CTRL, - BAM_P_RST, - BAM_P_HALT, - BAM_P_IRQ_STTS, - BAM_P_IRQ_CLR, - BAM_P_IRQ_EN, - BAM_P_EVNT_DEST_ADDR, - BAM_P_EVNT_REG, - BAM_P_SW_OFSTS, - BAM_P_DATA_FIFO_ADDR, - BAM_P_DESC_FIFO_ADDR, - BAM_P_EVNT_GEN_TRSHLD, - BAM_P_FIFO_SIZES, -}; - -struct reg_offset_data { - u32 base_offset; - unsigned int pipe_mult, evnt_mult, ee_mult; -}; - -static const struct reg_offset_data bam_v1_3_reg_info[] = { - [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 }, - [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 }, - [BAM_NUM_PIPES] = { 0x0FBC, 0x00, 0x00, 0x00 }, - [BAM_DESC_CNT_TRSHLD] = { 0x0F88, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS] = { 0x0F8C, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_MSK] = { 0x0F90, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 }, - [BAM_IRQ_STTS] = { 0x0F94, 0x00, 0x00, 0x00 }, - [BAM_IRQ_CLR] = { 0x0F98, 0x00, 0x00, 0x00 }, - [BAM_IRQ_EN] = { 0x0F9C, 0x00, 0x00, 0x00 }, - [BAM_CNFG_BITS] = { 0x0FFC, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_EE] = { 0x1800, 0x00, 0x00, 0x80 }, - [BAM_IRQ_SRCS_MSK_EE] = { 0x1804, 0x00, 0x00, 0x80 }, - [BAM_P_CTRL] = { 0x0000, 0x80, 0x00, 0x00 }, - [BAM_P_RST] = { 0x0004, 0x80, 0x00, 0x00 }, - [BAM_P_HALT] = { 0x0008, 0x80, 0x00, 0x00 }, - [BAM_P_IRQ_STTS] = { 0x0010, 0x80, 0x00, 0x00 }, - [BAM_P_IRQ_CLR] = { 0x0014, 0x80, 0x00, 0x00 }, - [BAM_P_IRQ_EN] = { 0x0018, 0x80, 0x00, 0x00 }, - [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x40, 0x00 }, - [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x40, 0x00 }, - [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x40, 0x00 }, - [BAM_P_DATA_FIFO_ADDR] = { 0x1024, 0x00, 0x40, 0x00 }, - [BAM_P_DESC_FIFO_ADDR] = { 0x101C, 0x00, 0x40, 0x00 }, - [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 }, - [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 }, -}; - -static const struct reg_offset_data bam_v1_4_reg_info[] = { - [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 }, - [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 }, - [BAM_NUM_PIPES] = { 0x003C, 0x00, 0x00, 0x00 }, - [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS] = { 0x000C, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_MSK] = { 0x0010, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 }, - [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 }, - [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 }, - [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 }, - [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_EE] = { 0x0800, 0x00, 0x00, 0x80 }, - [BAM_IRQ_SRCS_MSK_EE] = { 0x0804, 0x00, 0x00, 0x80 }, - [BAM_P_CTRL] = { 0x1000, 0x1000, 0x00, 0x00 }, - [BAM_P_RST] = { 0x1004, 0x1000, 0x00, 0x00 }, - [BAM_P_HALT] = { 0x1008, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_STTS] = { 0x1010, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_CLR] = { 0x1014, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_EN] = { 0x1018, 0x1000, 0x00, 0x00 }, - [BAM_P_EVNT_DEST_ADDR] = { 0x182C, 0x00, 0x1000, 0x00 }, - [BAM_P_EVNT_REG] = { 0x1818, 0x00, 0x1000, 0x00 }, - [BAM_P_SW_OFSTS] = { 0x1800, 0x00, 0x1000, 0x00 }, - [BAM_P_DATA_FIFO_ADDR] = { 0x1824, 0x00, 0x1000, 0x00 }, - [BAM_P_DESC_FIFO_ADDR] = { 0x181C, 0x00, 0x1000, 0x00 }, - [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 }, - [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 }, -}; - -static const struct reg_offset_data bam_v1_7_reg_info[] = { - [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 }, - [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 }, - [BAM_NUM_PIPES] = { 0x01008, 0x00, 0x00, 0x00 }, - [BAM_DESC_CNT_TRSHLD] = { 0x00008, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS] = { 0x03010, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_MSK] = { 0x03014, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 }, - [BAM_IRQ_STTS] = { 0x00014, 0x00, 0x00, 0x00 }, - [BAM_IRQ_CLR] = { 0x00018, 0x00, 0x00, 0x00 }, - [BAM_IRQ_EN] = { 0x0001C, 0x00, 0x00, 0x00 }, - [BAM_CNFG_BITS] = { 0x0007C, 0x00, 0x00, 0x00 }, - [BAM_IRQ_SRCS_EE] = { 0x03000, 0x00, 0x00, 0x1000 }, - [BAM_IRQ_SRCS_MSK_EE] = { 0x03004, 0x00, 0x00, 0x1000 }, - [BAM_P_CTRL] = { 0x13000, 0x1000, 0x00, 0x00 }, - [BAM_P_RST] = { 0x13004, 0x1000, 0x00, 0x00 }, - [BAM_P_HALT] = { 0x13008, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_STTS] = { 0x13010, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_CLR] = { 0x13014, 0x1000, 0x00, 0x00 }, - [BAM_P_IRQ_EN] = { 0x13018, 0x1000, 0x00, 0x00 }, - [BAM_P_EVNT_DEST_ADDR] = { 0x1382C, 0x00, 0x1000, 0x00 }, - [BAM_P_EVNT_REG] = { 0x13818, 0x00, 0x1000, 0x00 }, - [BAM_P_SW_OFSTS] = { 0x13800, 0x00, 0x1000, 0x00 }, - [BAM_P_DATA_FIFO_ADDR] = { 0x13824, 0x00, 0x1000, 0x00 }, - [BAM_P_DESC_FIFO_ADDR] = { 0x1381C, 0x00, 0x1000, 0x00 }, - [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 }, - [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 }, -}; - -/* BAM CTRL */ -#define BAM_SW_RST BIT(0) -#define BAM_EN BIT(1) -#define BAM_EN_ACCUM BIT(4) -#define BAM_TESTBUS_SEL_SHIFT 5 -#define BAM_TESTBUS_SEL_MASK 0x3F -#define BAM_DESC_CACHE_SEL_SHIFT 13 -#define BAM_DESC_CACHE_SEL_MASK 0x3 -#define BAM_CACHED_DESC_STORE BIT(15) -#define IBC_DISABLE BIT(16) - -/* BAM REVISION */ -#define REVISION_SHIFT 0 -#define REVISION_MASK 0xFF -#define NUM_EES_SHIFT 8 -#define NUM_EES_MASK 0xF -#define CE_BUFFER_SIZE BIT(13) -#define AXI_ACTIVE BIT(14) -#define USE_VMIDMT BIT(15) -#define SECURED BIT(16) -#define BAM_HAS_NO_BYPASS BIT(17) -#define HIGH_FREQUENCY_BAM BIT(18) -#define INACTIV_TMRS_EXST BIT(19) -#define NUM_INACTIV_TMRS BIT(20) -#define DESC_CACHE_DEPTH_SHIFT 21 -#define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT) -#define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT) -#define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT) -#define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT) -#define CMD_DESC_EN BIT(23) -#define INACTIV_TMR_BASE_SHIFT 24 -#define INACTIV_TMR_BASE_MASK 0xFF - -/* BAM NUM PIPES */ -#define BAM_NUM_PIPES_SHIFT 0 -#define BAM_NUM_PIPES_MASK 0xFF -#define PERIPH_NON_PIPE_GRP_SHIFT 16 -#define PERIPH_NON_PIP_GRP_MASK 0xFF -#define BAM_NON_PIPE_GRP_SHIFT 24 -#define BAM_NON_PIPE_GRP_MASK 0xFF - -/* BAM CNFG BITS */ -#define BAM_PIPE_CNFG BIT(2) -#define BAM_FULL_PIPE BIT(11) -#define BAM_NO_EXT_P_RST BIT(12) -#define BAM_IBC_DISABLE BIT(13) -#define BAM_SB_CLK_REQ BIT(14) -#define BAM_PSM_CSW_REQ BIT(15) -#define BAM_PSM_P_RES BIT(16) -#define BAM_AU_P_RES BIT(17) -#define BAM_SI_P_RES BIT(18) -#define BAM_WB_P_RES BIT(19) -#define BAM_WB_BLK_CSW BIT(20) -#define BAM_WB_CSW_ACK_IDL BIT(21) -#define BAM_WB_RETR_SVPNT BIT(22) -#define BAM_WB_DSC_AVL_P_RST BIT(23) -#define BAM_REG_P_EN BIT(24) -#define BAM_PSM_P_HD_DATA BIT(25) -#define BAM_AU_ACCUMED BIT(26) -#define BAM_CMD_ENABLE BIT(27) - -#define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \ - BAM_NO_EXT_P_RST | \ - BAM_IBC_DISABLE | \ - BAM_SB_CLK_REQ | \ - BAM_PSM_CSW_REQ | \ - BAM_PSM_P_RES | \ - BAM_AU_P_RES | \ - BAM_SI_P_RES | \ - BAM_WB_P_RES | \ - BAM_WB_BLK_CSW | \ - BAM_WB_CSW_ACK_IDL | \ - BAM_WB_RETR_SVPNT | \ - BAM_WB_DSC_AVL_P_RST | \ - BAM_REG_P_EN | \ - BAM_PSM_P_HD_DATA | \ - BAM_AU_ACCUMED | \ - BAM_CMD_ENABLE) - -/* PIPE CTRL */ -#define P_EN BIT(1) -#define P_DIRECTION BIT(3) -#define P_SYS_STRM BIT(4) -#define P_SYS_MODE BIT(5) -#define P_AUTO_EOB BIT(6) -#define P_AUTO_EOB_SEL_SHIFT 7 -#define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT) -#define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT) -#define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT) -#define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT) -#define P_PREFETCH_LIMIT_SHIFT 9 -#define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT) -#define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT) -#define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT) -#define P_WRITE_NWD BIT(11) -#define P_LOCK_GROUP_SHIFT 16 -#define P_LOCK_GROUP_MASK 0x1F - -/* BAM_DESC_CNT_TRSHLD */ -#define CNT_TRSHLD 0xffff -#define DEFAULT_CNT_THRSHLD 0x4 - -/* BAM_IRQ_SRCS */ -#define BAM_IRQ BIT(31) -#define P_IRQ 0x7fffffff - -/* BAM_IRQ_SRCS_MSK */ -#define BAM_IRQ_MSK BAM_IRQ -#define P_IRQ_MSK P_IRQ - -/* BAM_IRQ_STTS */ -#define BAM_TIMER_IRQ BIT(4) -#define BAM_EMPTY_IRQ BIT(3) -#define BAM_ERROR_IRQ BIT(2) -#define BAM_HRESP_ERR_IRQ BIT(1) - -/* BAM_IRQ_CLR */ -#define BAM_TIMER_CLR BIT(4) -#define BAM_EMPTY_CLR BIT(3) -#define BAM_ERROR_CLR BIT(2) -#define BAM_HRESP_ERR_CLR BIT(1) - -/* BAM_IRQ_EN */ -#define BAM_TIMER_EN BIT(4) -#define BAM_EMPTY_EN BIT(3) -#define BAM_ERROR_EN BIT(2) -#define BAM_HRESP_ERR_EN BIT(1) - -/* BAM_P_IRQ_EN */ -#define P_PRCSD_DESC_EN BIT(0) -#define P_TIMER_EN BIT(1) -#define P_WAKE_EN BIT(2) -#define P_OUT_OF_DESC_EN BIT(3) -#define P_ERR_EN BIT(4) -#define P_TRNSFR_END_EN BIT(5) -#define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN) - -/* BAM_P_SW_OFSTS */ -#define P_SW_OFSTS_MASK 0xffff - -#define BAM_DESC_FIFO_SIZE SZ_32K -#define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1) -#define BAM_MAX_DATA_SIZE (SZ_32K - 8) - -struct bam_chan { - struct virt_dma_chan vc; - - struct bam_device *bdev; - - /* configuration from device tree */ - u32 id; - - struct bam_async_desc *curr_txd; /* current running dma */ - - /* runtime configuration */ - struct dma_slave_config slave; - - /* fifo storage */ - struct bam_desc_hw *fifo_virt; - dma_addr_t fifo_phys; - - /* fifo markers */ - unsigned short head; /* start of active descriptor entries */ - unsigned short tail; /* end of active descriptor entries */ - - unsigned int initialized; /* is the channel hw initialized? */ - unsigned int paused; /* is the channel paused? */ - unsigned int reconfigure; /* new slave config? */ - - struct list_head node; -}; - -static inline struct bam_chan *to_bam_chan(struct dma_chan *common) -{ - return container_of(common, struct bam_chan, vc.chan); -} - -struct bam_device { - void __iomem *regs; - struct device *dev; - struct dma_device common; - struct device_dma_parameters dma_parms; - struct bam_chan *channels; - u32 num_channels; - - /* execution environment ID, from DT */ - u32 ee; - - const struct reg_offset_data *layout; - - struct clk *bamclk; - int irq; - - /* dma start transaction tasklet */ - struct tasklet_struct task; -}; - -/** - * bam_addr - returns BAM register address - * @bdev: bam device - * @pipe: pipe instance (ignored when register doesn't have multiple instances) - * @reg: register enum - */ -static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe, - enum bam_reg reg) -{ - const struct reg_offset_data r = bdev->layout[reg]; - - return bdev->regs + r.base_offset + - r.pipe_mult * pipe + - r.evnt_mult * pipe + - r.ee_mult * bdev->ee; -} - -/** - * bam_reset_channel - Reset individual BAM DMA channel - * @bchan: bam channel - * - * This function resets a specific BAM channel - */ -static void bam_reset_channel(struct bam_chan *bchan) -{ - struct bam_device *bdev = bchan->bdev; - - lockdep_assert_held(&bchan->vc.lock); - - /* reset channel */ - writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST)); - writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST)); - - /* don't allow cpu to reorder BAM register accesses done after this */ - wmb(); - - /* make sure hw is initialized when channel is used the first time */ - bchan->initialized = 0; -} - -/** - * bam_chan_init_hw - Initialize channel hardware - * @bchan: bam channel - * - * This function resets and initializes the BAM channel - */ -static void bam_chan_init_hw(struct bam_chan *bchan, - enum dma_transfer_direction dir) -{ - struct bam_device *bdev = bchan->bdev; - u32 val; - - /* Reset the channel to clear internal state of the FIFO */ - bam_reset_channel(bchan); - - /* - * write out 8 byte aligned address. We have enough space for this - * because we allocated 1 more descriptor (8 bytes) than we can use - */ - writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)), - bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR)); - writel_relaxed(BAM_DESC_FIFO_SIZE, - bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES)); - - /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */ - writel_relaxed(P_DEFAULT_IRQS_EN, - bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); - - /* unmask the specific pipe and EE combo */ - val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - val |= BIT(bchan->id); - writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - - /* don't allow cpu to reorder the channel enable done below */ - wmb(); - - /* set fixed direction and mode, then enable channel */ - val = P_EN | P_SYS_MODE; - if (dir == DMA_DEV_TO_MEM) - val |= P_DIRECTION; - - writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL)); - - bchan->initialized = 1; - - /* init FIFO pointers */ - bchan->head = 0; - bchan->tail = 0; -} - -/** - * bam_alloc_chan - Allocate channel resources for DMA channel. - * @chan: specified channel - * - * This function allocates the FIFO descriptor memory - */ -static int bam_alloc_chan(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct bam_device *bdev = bchan->bdev; - - if (bchan->fifo_virt) - return 0; - - /* allocate FIFO descriptor space, but only if necessary */ - bchan->fifo_virt = dma_alloc_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, - &bchan->fifo_phys, GFP_KERNEL); - - if (!bchan->fifo_virt) { - dev_err(bdev->dev, "Failed to allocate desc fifo\n"); - return -ENOMEM; - } - - return 0; -} - -/** - * bam_free_chan - Frees dma resources associated with specific channel - * @chan: specified channel - * - * Free the allocated fifo descriptor memory and channel resources - * - */ -static void bam_free_chan(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct bam_device *bdev = bchan->bdev; - u32 val; - unsigned long flags; - - vchan_free_chan_resources(to_virt_chan(chan)); - - if (bchan->curr_txd) { - dev_err(bchan->bdev->dev, "Cannot free busy channel\n"); - return; - } - - spin_lock_irqsave(&bchan->vc.lock, flags); - bam_reset_channel(bchan); - spin_unlock_irqrestore(&bchan->vc.lock, flags); - - dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt, - bchan->fifo_phys); - bchan->fifo_virt = NULL; - - /* mask irq for pipe/channel */ - val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - val &= ~BIT(bchan->id); - writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - - /* disable irq */ - writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); -} - -/** - * bam_slave_config - set slave configuration for channel - * @chan: dma channel - * @cfg: slave configuration - * - * Sets slave configuration for channel - * - */ -static int bam_slave_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct bam_chan *bchan = to_bam_chan(chan); - unsigned long flag; - - spin_lock_irqsave(&bchan->vc.lock, flag); - memcpy(&bchan->slave, cfg, sizeof(*cfg)); - bchan->reconfigure = 1; - spin_unlock_irqrestore(&bchan->vc.lock, flag); - - return 0; -} - -/** - * bam_prep_slave_sg - Prep slave sg transaction - * - * @chan: dma channel - * @sgl: scatter gather list - * @sg_len: length of sg - * @direction: DMA transfer direction - * @flags: DMA flags - * @context: transfer context (unused) - */ -static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, - struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long flags, - void *context) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct bam_device *bdev = bchan->bdev; - struct bam_async_desc *async_desc; - struct scatterlist *sg; - u32 i; - struct bam_desc_hw *desc; - unsigned int num_alloc = 0; - - - if (!is_slave_direction(direction)) { - dev_err(bdev->dev, "invalid dma direction\n"); - return NULL; - } - - /* calculate number of required entries */ - for_each_sg(sgl, sg, sg_len, i) - num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_MAX_DATA_SIZE); - - /* allocate enough room to accomodate the number of entries */ - async_desc = kzalloc(sizeof(*async_desc) + - (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT); - - if (!async_desc) - goto err_out; - - if (flags & DMA_PREP_FENCE) - async_desc->flags |= DESC_FLAG_NWD; - - if (flags & DMA_PREP_INTERRUPT) - async_desc->flags |= DESC_FLAG_EOT; - else - async_desc->flags |= DESC_FLAG_INT; - - async_desc->num_desc = num_alloc; - async_desc->curr_desc = async_desc->desc; - async_desc->dir = direction; - - /* fill in temporary descriptors */ - desc = async_desc->desc; - for_each_sg(sgl, sg, sg_len, i) { - unsigned int remainder = sg_dma_len(sg); - unsigned int curr_offset = 0; - - do { - desc->addr = cpu_to_le32(sg_dma_address(sg) + - curr_offset); - - if (remainder > BAM_MAX_DATA_SIZE) { - desc->size = cpu_to_le16(BAM_MAX_DATA_SIZE); - remainder -= BAM_MAX_DATA_SIZE; - curr_offset += BAM_MAX_DATA_SIZE; - } else { - desc->size = cpu_to_le16(remainder); - remainder = 0; - } - - async_desc->length += desc->size; - desc++; - } while (remainder > 0); - } - - return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags); - -err_out: - kfree(async_desc); - return NULL; -} - -/** - * bam_dma_terminate_all - terminate all transactions on a channel - * @bchan: bam dma channel - * - * Dequeues and frees all transactions - * No callbacks are done - * - */ -static int bam_dma_terminate_all(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - unsigned long flag; - LIST_HEAD(head); - - /* remove all transactions, including active transaction */ - spin_lock_irqsave(&bchan->vc.lock, flag); - if (bchan->curr_txd) { - list_add(&bchan->curr_txd->vd.node, &bchan->vc.desc_issued); - bchan->curr_txd = NULL; - } - - vchan_get_all_descriptors(&bchan->vc, &head); - spin_unlock_irqrestore(&bchan->vc.lock, flag); - - vchan_dma_desc_free_list(&bchan->vc, &head); - - return 0; -} - -/** - * bam_pause - Pause DMA channel - * @chan: dma channel - * - */ -static int bam_pause(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct bam_device *bdev = bchan->bdev; - unsigned long flag; - - spin_lock_irqsave(&bchan->vc.lock, flag); - writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT)); - bchan->paused = 1; - spin_unlock_irqrestore(&bchan->vc.lock, flag); - - return 0; -} - -/** - * bam_resume - Resume DMA channel operations - * @chan: dma channel - * - */ -static int bam_resume(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct bam_device *bdev = bchan->bdev; - unsigned long flag; - - spin_lock_irqsave(&bchan->vc.lock, flag); - writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT)); - bchan->paused = 0; - spin_unlock_irqrestore(&bchan->vc.lock, flag); - - return 0; -} - -/** - * process_channel_irqs - processes the channel interrupts - * @bdev: bam controller - * - * This function processes the channel interrupts - * - */ -static u32 process_channel_irqs(struct bam_device *bdev) -{ - u32 i, srcs, pipe_stts; - unsigned long flags; - struct bam_async_desc *async_desc; - - srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE)); - - /* return early if no pipe/channel interrupts are present */ - if (!(srcs & P_IRQ)) - return srcs; - - for (i = 0; i < bdev->num_channels; i++) { - struct bam_chan *bchan = &bdev->channels[i]; - - if (!(srcs & BIT(i))) - continue; - - /* clear pipe irq */ - pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS)); - - writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR)); - - spin_lock_irqsave(&bchan->vc.lock, flags); - async_desc = bchan->curr_txd; - - if (async_desc) { - async_desc->num_desc -= async_desc->xfer_len; - async_desc->curr_desc += async_desc->xfer_len; - bchan->curr_txd = NULL; - - /* manage FIFO */ - bchan->head += async_desc->xfer_len; - bchan->head %= MAX_DESCRIPTORS; - - /* - * if complete, process cookie. Otherwise - * push back to front of desc_issued so that - * it gets restarted by the tasklet - */ - if (!async_desc->num_desc) - vchan_cookie_complete(&async_desc->vd); - else - list_add(&async_desc->vd.node, - &bchan->vc.desc_issued); - } - - spin_unlock_irqrestore(&bchan->vc.lock, flags); - } - - return srcs; -} - -/** - * bam_dma_irq - irq handler for bam controller - * @irq: IRQ of interrupt - * @data: callback data - * - * IRQ handler for the bam controller - */ -static irqreturn_t bam_dma_irq(int irq, void *data) -{ - struct bam_device *bdev = data; - u32 clr_mask = 0, srcs = 0; - - srcs |= process_channel_irqs(bdev); - - /* kick off tasklet to start next dma transfer */ - if (srcs & P_IRQ) - tasklet_schedule(&bdev->task); - - if (srcs & BAM_IRQ) - clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS)); - - /* don't allow reorder of the various accesses to the BAM registers */ - mb(); - - writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR)); - - return IRQ_HANDLED; -} - -/** - * bam_tx_status - returns status of transaction - * @chan: dma channel - * @cookie: transaction cookie - * @txstate: DMA transaction state - * - * Return status of dma transaction - */ -static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct bam_chan *bchan = to_bam_chan(chan); - struct virt_dma_desc *vd; - int ret; - size_t residue = 0; - unsigned int i; - unsigned long flags; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - if (!txstate) - return bchan->paused ? DMA_PAUSED : ret; - - spin_lock_irqsave(&bchan->vc.lock, flags); - vd = vchan_find_desc(&bchan->vc, cookie); - if (vd) - residue = container_of(vd, struct bam_async_desc, vd)->length; - else if (bchan->curr_txd && bchan->curr_txd->vd.tx.cookie == cookie) - for (i = 0; i < bchan->curr_txd->num_desc; i++) - residue += bchan->curr_txd->curr_desc[i].size; - - spin_unlock_irqrestore(&bchan->vc.lock, flags); - - dma_set_residue(txstate, residue); - - if (ret == DMA_IN_PROGRESS && bchan->paused) - ret = DMA_PAUSED; - - return ret; -} - -/** - * bam_apply_new_config - * @bchan: bam dma channel - * @dir: DMA direction - */ -static void bam_apply_new_config(struct bam_chan *bchan, - enum dma_transfer_direction dir) -{ - struct bam_device *bdev = bchan->bdev; - u32 maxburst; - - if (dir == DMA_DEV_TO_MEM) - maxburst = bchan->slave.src_maxburst; - else - maxburst = bchan->slave.dst_maxburst; - - writel_relaxed(maxburst, bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); - - bchan->reconfigure = 0; -} - -/** - * bam_start_dma - start next transaction - * @bchan - bam dma channel - */ -static void bam_start_dma(struct bam_chan *bchan) -{ - struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc); - struct bam_device *bdev = bchan->bdev; - struct bam_async_desc *async_desc; - struct bam_desc_hw *desc; - struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt, - sizeof(struct bam_desc_hw)); - - lockdep_assert_held(&bchan->vc.lock); - - if (!vd) - return; - - list_del(&vd->node); - - async_desc = container_of(vd, struct bam_async_desc, vd); - bchan->curr_txd = async_desc; - - /* on first use, initialize the channel hardware */ - if (!bchan->initialized) - bam_chan_init_hw(bchan, async_desc->dir); - - /* apply new slave config changes, if necessary */ - if (bchan->reconfigure) - bam_apply_new_config(bchan, async_desc->dir); - - desc = bchan->curr_txd->curr_desc; - - if (async_desc->num_desc > MAX_DESCRIPTORS) - async_desc->xfer_len = MAX_DESCRIPTORS; - else - async_desc->xfer_len = async_desc->num_desc; - - /* set any special flags on the last descriptor */ - if (async_desc->num_desc == async_desc->xfer_len) - desc[async_desc->xfer_len - 1].flags = - cpu_to_le16(async_desc->flags); - else - desc[async_desc->xfer_len - 1].flags |= - cpu_to_le16(DESC_FLAG_INT); - - if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) { - u32 partial = MAX_DESCRIPTORS - bchan->tail; - - memcpy(&fifo[bchan->tail], desc, - partial * sizeof(struct bam_desc_hw)); - memcpy(fifo, &desc[partial], (async_desc->xfer_len - partial) * - sizeof(struct bam_desc_hw)); - } else { - memcpy(&fifo[bchan->tail], desc, - async_desc->xfer_len * sizeof(struct bam_desc_hw)); - } - - bchan->tail += async_desc->xfer_len; - bchan->tail %= MAX_DESCRIPTORS; - - /* ensure descriptor writes and dma start not reordered */ - wmb(); - writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), - bam_addr(bdev, bchan->id, BAM_P_EVNT_REG)); -} - -/** - * dma_tasklet - DMA IRQ tasklet - * @data: tasklet argument (bam controller structure) - * - * Sets up next DMA operation and then processes all completed transactions - */ -static void dma_tasklet(unsigned long data) -{ - struct bam_device *bdev = (struct bam_device *)data; - struct bam_chan *bchan; - unsigned long flags; - unsigned int i; - - /* go through the channels and kick off transactions */ - for (i = 0; i < bdev->num_channels; i++) { - bchan = &bdev->channels[i]; - spin_lock_irqsave(&bchan->vc.lock, flags); - - if (!list_empty(&bchan->vc.desc_issued) && !bchan->curr_txd) - bam_start_dma(bchan); - spin_unlock_irqrestore(&bchan->vc.lock, flags); - } -} - -/** - * bam_issue_pending - starts pending transactions - * @chan: dma channel - * - * Calls tasklet directly which in turn starts any pending transactions - */ -static void bam_issue_pending(struct dma_chan *chan) -{ - struct bam_chan *bchan = to_bam_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&bchan->vc.lock, flags); - - /* if work pending and idle, start a transaction */ - if (vchan_issue_pending(&bchan->vc) && !bchan->curr_txd) - bam_start_dma(bchan); - - spin_unlock_irqrestore(&bchan->vc.lock, flags); -} - -/** - * bam_dma_free_desc - free descriptor memory - * @vd: virtual descriptor - * - */ -static void bam_dma_free_desc(struct virt_dma_desc *vd) -{ - struct bam_async_desc *async_desc = container_of(vd, - struct bam_async_desc, vd); - - kfree(async_desc); -} - -static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec, - struct of_dma *of) -{ - struct bam_device *bdev = container_of(of->of_dma_data, - struct bam_device, common); - unsigned int request; - - if (dma_spec->args_count != 1) - return NULL; - - request = dma_spec->args[0]; - if (request >= bdev->num_channels) - return NULL; - - return dma_get_slave_channel(&(bdev->channels[request].vc.chan)); -} - -/** - * bam_init - * @bdev: bam device - * - * Initialization helper for global bam registers - */ -static int bam_init(struct bam_device *bdev) -{ - u32 val; - - /* read revision and configuration information */ - val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)) >> NUM_EES_SHIFT; - val &= NUM_EES_MASK; - - /* check that configured EE is within range */ - if (bdev->ee >= val) - return -EINVAL; - - val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES)); - bdev->num_channels = val & BAM_NUM_PIPES_MASK; - - /* s/w reset bam */ - /* after reset all pipes are disabled and idle */ - val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); - val |= BAM_SW_RST; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - val &= ~BAM_SW_RST; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - - /* make sure previous stores are visible before enabling BAM */ - wmb(); - - /* enable bam */ - val |= BAM_EN; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - - /* set descriptor threshhold, start with 4 bytes */ - writel_relaxed(DEFAULT_CNT_THRSHLD, - bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); - - /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ - writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); - - /* enable irqs for errors */ - writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, - bam_addr(bdev, 0, BAM_IRQ_EN)); - - /* unmask global bam interrupt */ - writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - - return 0; -} - -static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, - u32 index) -{ - bchan->id = index; - bchan->bdev = bdev; - - vchan_init(&bchan->vc, &bdev->common); - bchan->vc.desc_free = bam_dma_free_desc; -} - -static const struct of_device_id bam_of_match[] = { - { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, - { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, - { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info }, - {} -}; - -MODULE_DEVICE_TABLE(of, bam_of_match); - -static int bam_dma_probe(struct platform_device *pdev) -{ - struct bam_device *bdev; - const struct of_device_id *match; - struct resource *iores; - int ret, i; - - bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL); - if (!bdev) - return -ENOMEM; - - bdev->dev = &pdev->dev; - - match = of_match_node(bam_of_match, pdev->dev.of_node); - if (!match) { - dev_err(&pdev->dev, "Unsupported BAM module\n"); - return -ENODEV; - } - - bdev->layout = match->data; - - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - bdev->regs = devm_ioremap_resource(&pdev->dev, iores); - if (IS_ERR(bdev->regs)) - return PTR_ERR(bdev->regs); - - bdev->irq = platform_get_irq(pdev, 0); - if (bdev->irq < 0) - return bdev->irq; - - ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee); - if (ret) { - dev_err(bdev->dev, "Execution environment unspecified\n"); - return ret; - } - - bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - if (IS_ERR(bdev->bamclk)) - return PTR_ERR(bdev->bamclk); - - ret = clk_prepare_enable(bdev->bamclk); - if (ret) { - dev_err(bdev->dev, "failed to prepare/enable clock\n"); - return ret; - } - - ret = bam_init(bdev); - if (ret) - goto err_disable_clk; - - tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev); - - bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels, - sizeof(*bdev->channels), GFP_KERNEL); - - if (!bdev->channels) { - ret = -ENOMEM; - goto err_tasklet_kill; - } - - /* allocate and initialize channels */ - INIT_LIST_HEAD(&bdev->common.channels); - - for (i = 0; i < bdev->num_channels; i++) - bam_channel_init(bdev, &bdev->channels[i], i); - - ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, - IRQF_TRIGGER_HIGH, "bam_dma", bdev); - if (ret) - goto err_bam_channel_exit; - - /* set max dma segment size */ - bdev->common.dev = bdev->dev; - bdev->common.dev->dma_parms = &bdev->dma_parms; - ret = dma_set_max_seg_size(bdev->common.dev, BAM_MAX_DATA_SIZE); - if (ret) { - dev_err(bdev->dev, "cannot set maximum segment size\n"); - goto err_bam_channel_exit; - } - - platform_set_drvdata(pdev, bdev); - - /* set capabilities */ - dma_cap_zero(bdev->common.cap_mask); - dma_cap_set(DMA_SLAVE, bdev->common.cap_mask); - - /* initialize dmaengine apis */ - bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; - bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; - bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; - bdev->common.device_alloc_chan_resources = bam_alloc_chan; - bdev->common.device_free_chan_resources = bam_free_chan; - bdev->common.device_prep_slave_sg = bam_prep_slave_sg; - bdev->common.device_config = bam_slave_config; - bdev->common.device_pause = bam_pause; - bdev->common.device_resume = bam_resume; - bdev->common.device_terminate_all = bam_dma_terminate_all; - bdev->common.device_issue_pending = bam_issue_pending; - bdev->common.device_tx_status = bam_tx_status; - bdev->common.dev = bdev->dev; - - ret = dma_async_device_register(&bdev->common); - if (ret) { - dev_err(bdev->dev, "failed to register dma async device\n"); - goto err_bam_channel_exit; - } - - ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, - &bdev->common); - if (ret) - goto err_unregister_dma; - - return 0; - -err_unregister_dma: - dma_async_device_unregister(&bdev->common); -err_bam_channel_exit: - for (i = 0; i < bdev->num_channels; i++) - tasklet_kill(&bdev->channels[i].vc.task); -err_tasklet_kill: - tasklet_kill(&bdev->task); -err_disable_clk: - clk_disable_unprepare(bdev->bamclk); - - return ret; -} - -static int bam_dma_remove(struct platform_device *pdev) -{ - struct bam_device *bdev = platform_get_drvdata(pdev); - u32 i; - - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&bdev->common); - - /* mask all interrupts for this execution environment */ - writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - - devm_free_irq(bdev->dev, bdev->irq, bdev); - - for (i = 0; i < bdev->num_channels; i++) { - bam_dma_terminate_all(&bdev->channels[i].vc.chan); - tasklet_kill(&bdev->channels[i].vc.task); - - dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, - bdev->channels[i].fifo_virt, - bdev->channels[i].fifo_phys); - } - - tasklet_kill(&bdev->task); - - clk_disable_unprepare(bdev->bamclk); - - return 0; -} - -static struct platform_driver bam_dma_driver = { - .probe = bam_dma_probe, - .remove = bam_dma_remove, - .driver = { - .name = "bam-dma-engine", - .of_match_table = bam_of_match, - }, -}; - -module_platform_driver(bam_dma_driver); - -MODULE_AUTHOR("Andy Gross "); -MODULE_DESCRIPTION("QCOM BAM DMA engine driver"); -MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 7f8f209fd6e09a07fd1820144452caba419cf2b4 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 4 Feb 2016 23:34:34 -0500 Subject: dmaengine: add Qualcomm Technologies HIDMA management driver The Qualcomm Technologies HIDMA device has been designed to support virtualization technology. The driver has been divided into two to follow the hardware design. 1. HIDMA Management driver 2. HIDMA Channel driver Each HIDMA HW consists of multiple channels. These channels share some set of common parameters. These parameters are initialized by the management driver during power up. Same management driver is used for monitoring the execution of the channels. Management driver can change the performance behavior dynamically such as bandwidth allocation and prioritization. The management driver is executed in host context and is the main management entity for all channels provided by the device. Signed-off-by: Sinan Kaya Reviewed-by: Andy Shevchenko Signed-off-by: Vinod Koul --- drivers/dma/qcom/Kconfig | 11 ++ drivers/dma/qcom/Makefile | 2 + drivers/dma/qcom/hidma_mgmt.c | 302 ++++++++++++++++++++++++++++++++++++++ drivers/dma/qcom/hidma_mgmt.h | 39 +++++ drivers/dma/qcom/hidma_mgmt_sys.c | 295 +++++++++++++++++++++++++++++++++++++ 5 files changed, 649 insertions(+) create mode 100644 drivers/dma/qcom/hidma_mgmt.c create mode 100644 drivers/dma/qcom/hidma_mgmt.h create mode 100644 drivers/dma/qcom/hidma_mgmt_sys.c (limited to 'drivers/dma') diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index f17c27288808..c975b1167acf 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -6,3 +6,14 @@ config QCOM_BAM_DMA ---help--- Enable support for the QCOM BAM DMA controller. This controller provides DMA capabilities for a variety of on-chip devices. + +config QCOM_HIDMA_MGMT + tristate "Qualcomm Technologies HIDMA Management support" + select DMA_ENGINE + help + Enable support for the Qualcomm Technologies HIDMA Management. + Each DMA device requires one management interface driver + for basic initialization before QCOM_HIDMA channel driver can + start managing the channels. In a virtualized environment, + the guest OS would run QCOM_HIDMA channel driver and the + host would run the QCOM_HIDMA_MGMT management driver. diff --git a/drivers/dma/qcom/Makefile b/drivers/dma/qcom/Makefile index f612ae303ee0..bfea6990229f 100644 --- a/drivers/dma/qcom/Makefile +++ b/drivers/dma/qcom/Makefile @@ -1 +1,3 @@ obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o +obj-$(CONFIG_QCOM_HIDMA_MGMT) += hdma_mgmt.o +hdma_mgmt-objs := hidma_mgmt.o hidma_mgmt_sys.o diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c new file mode 100644 index 000000000000..ef491b893f40 --- /dev/null +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -0,0 +1,302 @@ +/* + * Qualcomm Technologies HIDMA DMA engine Management interface + * + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hidma_mgmt.h" + +#define HIDMA_QOS_N_OFFSET 0x300 +#define HIDMA_CFG_OFFSET 0x400 +#define HIDMA_MAX_BUS_REQ_LEN_OFFSET 0x41C +#define HIDMA_MAX_XACTIONS_OFFSET 0x420 +#define HIDMA_HW_VERSION_OFFSET 0x424 +#define HIDMA_CHRESET_TIMEOUT_OFFSET 0x418 + +#define HIDMA_MAX_WR_XACTIONS_MASK GENMASK(4, 0) +#define HIDMA_MAX_RD_XACTIONS_MASK GENMASK(4, 0) +#define HIDMA_WEIGHT_MASK GENMASK(6, 0) +#define HIDMA_MAX_BUS_REQ_LEN_MASK GENMASK(15, 0) +#define HIDMA_CHRESET_TIMEOUT_MASK GENMASK(19, 0) + +#define HIDMA_MAX_WR_XACTIONS_BIT_POS 16 +#define HIDMA_MAX_BUS_WR_REQ_BIT_POS 16 +#define HIDMA_WRR_BIT_POS 8 +#define HIDMA_PRIORITY_BIT_POS 15 + +#define HIDMA_AUTOSUSPEND_TIMEOUT 2000 +#define HIDMA_MAX_CHANNEL_WEIGHT 15 + +int hidma_mgmt_setup(struct hidma_mgmt_dev *mgmtdev) +{ + unsigned int i; + u32 val; + + if (!is_power_of_2(mgmtdev->max_write_request) || + (mgmtdev->max_write_request < 128) || + (mgmtdev->max_write_request > 1024)) { + dev_err(&mgmtdev->pdev->dev, "invalid write request %d\n", + mgmtdev->max_write_request); + return -EINVAL; + } + + if (!is_power_of_2(mgmtdev->max_read_request) || + (mgmtdev->max_read_request < 128) || + (mgmtdev->max_read_request > 1024)) { + dev_err(&mgmtdev->pdev->dev, "invalid read request %d\n", + mgmtdev->max_read_request); + return -EINVAL; + } + + if (mgmtdev->max_wr_xactions > HIDMA_MAX_WR_XACTIONS_MASK) { + dev_err(&mgmtdev->pdev->dev, + "max_wr_xactions cannot be bigger than %ld\n", + HIDMA_MAX_WR_XACTIONS_MASK); + return -EINVAL; + } + + if (mgmtdev->max_rd_xactions > HIDMA_MAX_RD_XACTIONS_MASK) { + dev_err(&mgmtdev->pdev->dev, + "max_rd_xactions cannot be bigger than %ld\n", + HIDMA_MAX_RD_XACTIONS_MASK); + return -EINVAL; + } + + for (i = 0; i < mgmtdev->dma_channels; i++) { + if (mgmtdev->priority[i] > 1) { + dev_err(&mgmtdev->pdev->dev, + "priority can be 0 or 1\n"); + return -EINVAL; + } + + if (mgmtdev->weight[i] > HIDMA_MAX_CHANNEL_WEIGHT) { + dev_err(&mgmtdev->pdev->dev, + "max value of weight can be %d.\n", + HIDMA_MAX_CHANNEL_WEIGHT); + return -EINVAL; + } + + /* weight needs to be at least one */ + if (mgmtdev->weight[i] == 0) + mgmtdev->weight[i] = 1; + } + + pm_runtime_get_sync(&mgmtdev->pdev->dev); + val = readl(mgmtdev->virtaddr + HIDMA_MAX_BUS_REQ_LEN_OFFSET); + val &= ~(HIDMA_MAX_BUS_REQ_LEN_MASK << HIDMA_MAX_BUS_WR_REQ_BIT_POS); + val |= mgmtdev->max_write_request << HIDMA_MAX_BUS_WR_REQ_BIT_POS; + val &= ~HIDMA_MAX_BUS_REQ_LEN_MASK; + val |= mgmtdev->max_read_request; + writel(val, mgmtdev->virtaddr + HIDMA_MAX_BUS_REQ_LEN_OFFSET); + + val = readl(mgmtdev->virtaddr + HIDMA_MAX_XACTIONS_OFFSET); + val &= ~(HIDMA_MAX_WR_XACTIONS_MASK << HIDMA_MAX_WR_XACTIONS_BIT_POS); + val |= mgmtdev->max_wr_xactions << HIDMA_MAX_WR_XACTIONS_BIT_POS; + val &= ~HIDMA_MAX_RD_XACTIONS_MASK; + val |= mgmtdev->max_rd_xactions; + writel(val, mgmtdev->virtaddr + HIDMA_MAX_XACTIONS_OFFSET); + + mgmtdev->hw_version = + readl(mgmtdev->virtaddr + HIDMA_HW_VERSION_OFFSET); + mgmtdev->hw_version_major = (mgmtdev->hw_version >> 28) & 0xF; + mgmtdev->hw_version_minor = (mgmtdev->hw_version >> 16) & 0xF; + + for (i = 0; i < mgmtdev->dma_channels; i++) { + u32 weight = mgmtdev->weight[i]; + u32 priority = mgmtdev->priority[i]; + + val = readl(mgmtdev->virtaddr + HIDMA_QOS_N_OFFSET + (4 * i)); + val &= ~(1 << HIDMA_PRIORITY_BIT_POS); + val |= (priority & 0x1) << HIDMA_PRIORITY_BIT_POS; + val &= ~(HIDMA_WEIGHT_MASK << HIDMA_WRR_BIT_POS); + val |= (weight & HIDMA_WEIGHT_MASK) << HIDMA_WRR_BIT_POS; + writel(val, mgmtdev->virtaddr + HIDMA_QOS_N_OFFSET + (4 * i)); + } + + val = readl(mgmtdev->virtaddr + HIDMA_CHRESET_TIMEOUT_OFFSET); + val &= ~HIDMA_CHRESET_TIMEOUT_MASK; + val |= mgmtdev->chreset_timeout_cycles & HIDMA_CHRESET_TIMEOUT_MASK; + writel(val, mgmtdev->virtaddr + HIDMA_CHRESET_TIMEOUT_OFFSET); + + pm_runtime_mark_last_busy(&mgmtdev->pdev->dev); + pm_runtime_put_autosuspend(&mgmtdev->pdev->dev); + return 0; +} +EXPORT_SYMBOL_GPL(hidma_mgmt_setup); + +static int hidma_mgmt_probe(struct platform_device *pdev) +{ + struct hidma_mgmt_dev *mgmtdev; + struct resource *res; + void __iomem *virtaddr; + int irq; + int rc; + u32 val; + + pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_sync(&pdev->dev); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + virtaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(virtaddr)) { + rc = -ENOMEM; + goto out; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(&pdev->dev, "irq resources not found\n"); + rc = irq; + goto out; + } + + mgmtdev = devm_kzalloc(&pdev->dev, sizeof(*mgmtdev), GFP_KERNEL); + if (!mgmtdev) { + rc = -ENOMEM; + goto out; + } + + mgmtdev->pdev = pdev; + mgmtdev->addrsize = resource_size(res); + mgmtdev->virtaddr = virtaddr; + + rc = device_property_read_u32(&pdev->dev, "dma-channels", + &mgmtdev->dma_channels); + if (rc) { + dev_err(&pdev->dev, "number of channels missing\n"); + goto out; + } + + rc = device_property_read_u32(&pdev->dev, + "channel-reset-timeout-cycles", + &mgmtdev->chreset_timeout_cycles); + if (rc) { + dev_err(&pdev->dev, "channel reset timeout missing\n"); + goto out; + } + + rc = device_property_read_u32(&pdev->dev, "max-write-burst-bytes", + &mgmtdev->max_write_request); + if (rc) { + dev_err(&pdev->dev, "max-write-burst-bytes missing\n"); + goto out; + } + + rc = device_property_read_u32(&pdev->dev, "max-read-burst-bytes", + &mgmtdev->max_read_request); + if (rc) { + dev_err(&pdev->dev, "max-read-burst-bytes missing\n"); + goto out; + } + + rc = device_property_read_u32(&pdev->dev, "max-write-transactions", + &mgmtdev->max_wr_xactions); + if (rc) { + dev_err(&pdev->dev, "max-write-transactions missing\n"); + goto out; + } + + rc = device_property_read_u32(&pdev->dev, "max-read-transactions", + &mgmtdev->max_rd_xactions); + if (rc) { + dev_err(&pdev->dev, "max-read-transactions missing\n"); + goto out; + } + + mgmtdev->priority = devm_kcalloc(&pdev->dev, + mgmtdev->dma_channels, + sizeof(*mgmtdev->priority), + GFP_KERNEL); + if (!mgmtdev->priority) { + rc = -ENOMEM; + goto out; + } + + mgmtdev->weight = devm_kcalloc(&pdev->dev, + mgmtdev->dma_channels, + sizeof(*mgmtdev->weight), GFP_KERNEL); + if (!mgmtdev->weight) { + rc = -ENOMEM; + goto out; + } + + rc = hidma_mgmt_setup(mgmtdev); + if (rc) { + dev_err(&pdev->dev, "setup failed\n"); + goto out; + } + + /* start the HW */ + val = readl(mgmtdev->virtaddr + HIDMA_CFG_OFFSET); + val |= 1; + writel(val, mgmtdev->virtaddr + HIDMA_CFG_OFFSET); + + rc = hidma_mgmt_init_sys(mgmtdev); + if (rc) { + dev_err(&pdev->dev, "sysfs setup failed\n"); + goto out; + } + + dev_info(&pdev->dev, + "HW rev: %d.%d @ %pa with %d physical channels\n", + mgmtdev->hw_version_major, mgmtdev->hw_version_minor, + &res->start, mgmtdev->dma_channels); + + platform_set_drvdata(pdev, mgmtdev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + return 0; +out: + pm_runtime_put_sync_suspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); + return rc; +} + +#if IS_ENABLED(CONFIG_ACPI) +static const struct acpi_device_id hidma_mgmt_acpi_ids[] = { + {"QCOM8060"}, + {}, +}; +#endif + +static const struct of_device_id hidma_mgmt_match[] = { + {.compatible = "qcom,hidma-mgmt-1.0",}, + {}, +}; +MODULE_DEVICE_TABLE(of, hidma_mgmt_match); + +static struct platform_driver hidma_mgmt_driver = { + .probe = hidma_mgmt_probe, + .driver = { + .name = "hidma-mgmt", + .of_match_table = hidma_mgmt_match, + .acpi_match_table = ACPI_PTR(hidma_mgmt_acpi_ids), + }, +}; + +module_platform_driver(hidma_mgmt_driver); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/hidma_mgmt.h b/drivers/dma/qcom/hidma_mgmt.h new file mode 100644 index 000000000000..f7daf33769f4 --- /dev/null +++ b/drivers/dma/qcom/hidma_mgmt.h @@ -0,0 +1,39 @@ +/* + * Qualcomm Technologies HIDMA Management common header + * + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +struct hidma_mgmt_dev { + u8 hw_version_major; + u8 hw_version_minor; + + u32 max_wr_xactions; + u32 max_rd_xactions; + u32 max_write_request; + u32 max_read_request; + u32 dma_channels; + u32 chreset_timeout_cycles; + u32 hw_version; + u32 *priority; + u32 *weight; + + /* Hardware device constants */ + void __iomem *virtaddr; + resource_size_t addrsize; + + struct kobject **chroots; + struct platform_device *pdev; +}; + +int hidma_mgmt_init_sys(struct hidma_mgmt_dev *dev); +int hidma_mgmt_setup(struct hidma_mgmt_dev *mgmtdev); diff --git a/drivers/dma/qcom/hidma_mgmt_sys.c b/drivers/dma/qcom/hidma_mgmt_sys.c new file mode 100644 index 000000000000..d61f1068a34b --- /dev/null +++ b/drivers/dma/qcom/hidma_mgmt_sys.c @@ -0,0 +1,295 @@ +/* + * Qualcomm Technologies HIDMA Management SYS interface + * + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include + +#include "hidma_mgmt.h" + +struct hidma_chan_attr { + struct hidma_mgmt_dev *mdev; + int index; + struct kobj_attribute attr; +}; + +struct hidma_mgmt_fileinfo { + char *name; + int mode; + int (*get)(struct hidma_mgmt_dev *mdev); + int (*set)(struct hidma_mgmt_dev *mdev, u64 val); +}; + +#define IMPLEMENT_GETSET(name) \ +static int get_##name(struct hidma_mgmt_dev *mdev) \ +{ \ + return mdev->name; \ +} \ +static int set_##name(struct hidma_mgmt_dev *mdev, u64 val) \ +{ \ + u64 tmp; \ + int rc; \ + \ + tmp = mdev->name; \ + mdev->name = val; \ + rc = hidma_mgmt_setup(mdev); \ + if (rc) \ + mdev->name = tmp; \ + return rc; \ +} + +#define DECLARE_ATTRIBUTE(name, mode) \ + {#name, mode, get_##name, set_##name} + +IMPLEMENT_GETSET(hw_version_major) +IMPLEMENT_GETSET(hw_version_minor) +IMPLEMENT_GETSET(max_wr_xactions) +IMPLEMENT_GETSET(max_rd_xactions) +IMPLEMENT_GETSET(max_write_request) +IMPLEMENT_GETSET(max_read_request) +IMPLEMENT_GETSET(dma_channels) +IMPLEMENT_GETSET(chreset_timeout_cycles) + +static int set_priority(struct hidma_mgmt_dev *mdev, unsigned int i, u64 val) +{ + u64 tmp; + int rc; + + if (i >= mdev->dma_channels) + return -EINVAL; + + tmp = mdev->priority[i]; + mdev->priority[i] = val; + rc = hidma_mgmt_setup(mdev); + if (rc) + mdev->priority[i] = tmp; + return rc; +} + +static int set_weight(struct hidma_mgmt_dev *mdev, unsigned int i, u64 val) +{ + u64 tmp; + int rc; + + if (i >= mdev->dma_channels) + return -EINVAL; + + tmp = mdev->weight[i]; + mdev->weight[i] = val; + rc = hidma_mgmt_setup(mdev); + if (rc) + mdev->weight[i] = tmp; + return rc; +} + +static struct hidma_mgmt_fileinfo hidma_mgmt_files[] = { + DECLARE_ATTRIBUTE(hw_version_major, S_IRUGO), + DECLARE_ATTRIBUTE(hw_version_minor, S_IRUGO), + DECLARE_ATTRIBUTE(dma_channels, S_IRUGO), + DECLARE_ATTRIBUTE(chreset_timeout_cycles, S_IRUGO), + DECLARE_ATTRIBUTE(max_wr_xactions, S_IRUGO), + DECLARE_ATTRIBUTE(max_rd_xactions, S_IRUGO), + DECLARE_ATTRIBUTE(max_write_request, S_IRUGO), + DECLARE_ATTRIBUTE(max_read_request, S_IRUGO), +}; + +static ssize_t show_values(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct hidma_mgmt_dev *mdev = platform_get_drvdata(pdev); + unsigned int i; + + buf[0] = 0; + + for (i = 0; i < ARRAY_SIZE(hidma_mgmt_files); i++) { + if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0) { + sprintf(buf, "%d\n", hidma_mgmt_files[i].get(mdev)); + break; + } + } + return strlen(buf); +} + +static ssize_t set_values(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + struct hidma_mgmt_dev *mdev = platform_get_drvdata(pdev); + unsigned long tmp; + unsigned int i; + int rc; + + rc = kstrtoul(buf, 0, &tmp); + if (rc) + return rc; + + for (i = 0; i < ARRAY_SIZE(hidma_mgmt_files); i++) { + if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0) { + rc = hidma_mgmt_files[i].set(mdev, tmp); + if (rc) + return rc; + + break; + } + } + return count; +} + +static ssize_t show_values_channel(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct hidma_chan_attr *chattr; + struct hidma_mgmt_dev *mdev; + + buf[0] = 0; + chattr = container_of(attr, struct hidma_chan_attr, attr); + mdev = chattr->mdev; + if (strcmp(attr->attr.name, "priority") == 0) + sprintf(buf, "%d\n", mdev->priority[chattr->index]); + else if (strcmp(attr->attr.name, "weight") == 0) + sprintf(buf, "%d\n", mdev->weight[chattr->index]); + + return strlen(buf); +} + +static ssize_t set_values_channel(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + struct hidma_chan_attr *chattr; + struct hidma_mgmt_dev *mdev; + unsigned long tmp; + int rc; + + chattr = container_of(attr, struct hidma_chan_attr, attr); + mdev = chattr->mdev; + + rc = kstrtoul(buf, 0, &tmp); + if (rc) + return rc; + + if (strcmp(attr->attr.name, "priority") == 0) { + rc = set_priority(mdev, chattr->index, tmp); + if (rc) + return rc; + } else if (strcmp(attr->attr.name, "weight") == 0) { + rc = set_weight(mdev, chattr->index, tmp); + if (rc) + return rc; + } + return count; +} + +static int create_sysfs_entry(struct hidma_mgmt_dev *dev, char *name, int mode) +{ + struct device_attribute *attrs; + char *name_copy; + + attrs = devm_kmalloc(&dev->pdev->dev, + sizeof(struct device_attribute), GFP_KERNEL); + if (!attrs) + return -ENOMEM; + + name_copy = devm_kstrdup(&dev->pdev->dev, name, GFP_KERNEL); + if (!name_copy) + return -ENOMEM; + + attrs->attr.name = name_copy; + attrs->attr.mode = mode; + attrs->show = show_values; + attrs->store = set_values; + sysfs_attr_init(&attrs->attr); + + return device_create_file(&dev->pdev->dev, attrs); +} + +static int create_sysfs_entry_channel(struct hidma_mgmt_dev *mdev, char *name, + int mode, int index, + struct kobject *parent) +{ + struct hidma_chan_attr *chattr; + char *name_copy; + + chattr = devm_kmalloc(&mdev->pdev->dev, sizeof(*chattr), GFP_KERNEL); + if (!chattr) + return -ENOMEM; + + name_copy = devm_kstrdup(&mdev->pdev->dev, name, GFP_KERNEL); + if (!name_copy) + return -ENOMEM; + + chattr->mdev = mdev; + chattr->index = index; + chattr->attr.attr.name = name_copy; + chattr->attr.attr.mode = mode; + chattr->attr.show = show_values_channel; + chattr->attr.store = set_values_channel; + sysfs_attr_init(&chattr->attr.attr); + + return sysfs_create_file(parent, &chattr->attr.attr); +} + +int hidma_mgmt_init_sys(struct hidma_mgmt_dev *mdev) +{ + unsigned int i; + int rc; + int required; + struct kobject *chanops; + + required = sizeof(*mdev->chroots) * mdev->dma_channels; + mdev->chroots = devm_kmalloc(&mdev->pdev->dev, required, GFP_KERNEL); + if (!mdev->chroots) + return -ENOMEM; + + chanops = kobject_create_and_add("chanops", &mdev->pdev->dev.kobj); + if (!chanops) + return -ENOMEM; + + /* create each channel directory here */ + for (i = 0; i < mdev->dma_channels; i++) { + char name[20]; + + snprintf(name, sizeof(name), "chan%d", i); + mdev->chroots[i] = kobject_create_and_add(name, chanops); + if (!mdev->chroots[i]) + return -ENOMEM; + } + + /* populate common parameters */ + for (i = 0; i < ARRAY_SIZE(hidma_mgmt_files); i++) { + rc = create_sysfs_entry(mdev, hidma_mgmt_files[i].name, + hidma_mgmt_files[i].mode); + if (rc) + return rc; + } + + /* populate parameters that are per channel */ + for (i = 0; i < mdev->dma_channels; i++) { + rc = create_sysfs_entry_channel(mdev, "priority", + (S_IRUGO | S_IWUGO), i, + mdev->chroots[i]); + if (rc) + return rc; + + rc = create_sysfs_entry_channel(mdev, "weight", + (S_IRUGO | S_IWUGO), i, + mdev->chroots[i]); + if (rc) + return rc; + } + + return 0; +} +EXPORT_SYMBOL_GPL(hidma_mgmt_init_sys); -- cgit v1.2.3 From 67a2003e060739747cfa3ea9b0d88b3d321ebf24 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 4 Feb 2016 23:34:35 -0500 Subject: dmaengine: add Qualcomm Technologies HIDMA channel driver This patch adds support for hidma engine. The driver consists of two logical blocks. The DMA engine interface and the low-level interface. The hardware only supports memcpy/memset and this driver only support memcpy interface. HW and driver doesn't support slave interface. Signed-off-by: Sinan Kaya Reviewed-by: Andy Shevchenko Signed-off-by: Vinod Koul --- drivers/dma/qcom/Kconfig | 10 + drivers/dma/qcom/hidma.c | 706 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/qcom/hidma.h | 160 +++++++++++ 3 files changed, 876 insertions(+) create mode 100644 drivers/dma/qcom/hidma.c create mode 100644 drivers/dma/qcom/hidma.h (limited to 'drivers/dma') diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index c975b1167acf..a7761c4025f4 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -17,3 +17,13 @@ config QCOM_HIDMA_MGMT start managing the channels. In a virtualized environment, the guest OS would run QCOM_HIDMA channel driver and the host would run the QCOM_HIDMA_MGMT management driver. + +config QCOM_HIDMA + tristate "Qualcomm Technologies HIDMA Channel support" + select DMA_ENGINE + help + Enable support for the Qualcomm Technologies HIDMA controller. + The HIDMA controller supports optimized buffer copies + (user to kernel, kernel to kernel, etc.). It only supports + memcpy interface. The core is not intended for general + purpose slave DMA. diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c new file mode 100644 index 000000000000..cccc78efbca9 --- /dev/null +++ b/drivers/dma/qcom/hidma.c @@ -0,0 +1,706 @@ +/* + * Qualcomm Technologies HIDMA DMA engine interface + * + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* + * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. + * Copyright (C) Semihalf 2009 + * Copyright (C) Ilya Yanok, Emcraft Systems 2010 + * Copyright (C) Alexander Popov, Promcontroller 2014 + * + * Written by Piotr Ziecik . Hardware description + * (defines, structures and comments) was taken from MPC5121 DMA driver + * written by Hongjun Chen . + * + * Approved as OSADL project by a majority of OSADL members and funded + * by OSADL membership fees in 2009; for details see www.osadl.org. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* Linux Foundation elects GPLv2 license only. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "hidma.h" + +/* + * Default idle time is 2 seconds. This parameter can + * be overridden by changing the following + * /sys/bus/platform/devices/QCOM8061:/power/autosuspend_delay_ms + * during kernel boot. + */ +#define HIDMA_AUTOSUSPEND_TIMEOUT 2000 +#define HIDMA_ERR_INFO_SW 0xFF +#define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0 +#define HIDMA_NR_DEFAULT_DESC 10 + +static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev) +{ + return container_of(dmadev, struct hidma_dev, ddev); +} + +static inline +struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp) +{ + return container_of(_lldevp, struct hidma_dev, lldev); +} + +static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach) +{ + return container_of(dmach, struct hidma_chan, chan); +} + +static inline +struct hidma_desc *to_hidma_desc(struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct hidma_desc, desc); +} + +static void hidma_free(struct hidma_dev *dmadev) +{ + INIT_LIST_HEAD(&dmadev->ddev.channels); +} + +static unsigned int nr_desc_prm; +module_param(nr_desc_prm, uint, 0644); +MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)"); + + +/* process completed descriptors */ +static void hidma_process_completed(struct hidma_chan *mchan) +{ + struct dma_device *ddev = mchan->chan.device; + struct hidma_dev *mdma = to_hidma_dev(ddev); + struct dma_async_tx_descriptor *desc; + dma_cookie_t last_cookie; + struct hidma_desc *mdesc; + unsigned long irqflags; + struct list_head list; + + INIT_LIST_HEAD(&list); + + /* Get all completed descriptors */ + spin_lock_irqsave(&mchan->lock, irqflags); + list_splice_tail_init(&mchan->completed, &list); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + /* Execute callbacks and run dependencies */ + list_for_each_entry(mdesc, &list, node) { + enum dma_status llstat; + + desc = &mdesc->desc; + + spin_lock_irqsave(&mchan->lock, irqflags); + dma_cookie_complete(desc); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); + if (desc->callback && (llstat == DMA_COMPLETE)) + desc->callback(desc->callback_param); + + last_cookie = desc->cookie; + dma_run_dependencies(desc); + } + + /* Free descriptors */ + spin_lock_irqsave(&mchan->lock, irqflags); + list_splice_tail_init(&list, &mchan->free); + spin_unlock_irqrestore(&mchan->lock, irqflags); + +} + +/* + * Called once for each submitted descriptor. + * PM is locked once for each descriptor that is currently + * in execution. + */ +static void hidma_callback(void *data) +{ + struct hidma_desc *mdesc = data; + struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); + struct dma_device *ddev = mchan->chan.device; + struct hidma_dev *dmadev = to_hidma_dev(ddev); + unsigned long irqflags; + bool queued = false; + + spin_lock_irqsave(&mchan->lock, irqflags); + if (mdesc->node.next) { + /* Delete from the active list, add to completed list */ + list_move_tail(&mdesc->node, &mchan->completed); + queued = true; + + /* calculate the next running descriptor */ + mchan->running = list_first_entry(&mchan->active, + struct hidma_desc, node); + } + spin_unlock_irqrestore(&mchan->lock, irqflags); + + hidma_process_completed(mchan); + + if (queued) { + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + } +} + +static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig) +{ + struct hidma_chan *mchan; + struct dma_device *ddev; + + mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); + if (!mchan) + return -ENOMEM; + + ddev = &dmadev->ddev; + mchan->dma_sig = dma_sig; + mchan->dmadev = dmadev; + mchan->chan.device = ddev; + dma_cookie_init(&mchan->chan); + + INIT_LIST_HEAD(&mchan->free); + INIT_LIST_HEAD(&mchan->prepared); + INIT_LIST_HEAD(&mchan->active); + INIT_LIST_HEAD(&mchan->completed); + + spin_lock_init(&mchan->lock); + list_add_tail(&mchan->chan.device_node, &ddev->channels); + dmadev->ddev.chancnt++; + return 0; +} + +static void hidma_issue_task(unsigned long arg) +{ + struct hidma_dev *dmadev = (struct hidma_dev *)arg; + + pm_runtime_get_sync(dmadev->ddev.dev); + hidma_ll_start(dmadev->lldev); +} + +static void hidma_issue_pending(struct dma_chan *dmach) +{ + struct hidma_chan *mchan = to_hidma_chan(dmach); + struct hidma_dev *dmadev = mchan->dmadev; + unsigned long flags; + int status; + + spin_lock_irqsave(&mchan->lock, flags); + if (!mchan->running) { + struct hidma_desc *desc = list_first_entry(&mchan->active, + struct hidma_desc, + node); + mchan->running = desc; + } + spin_unlock_irqrestore(&mchan->lock, flags); + + /* PM will be released in hidma_callback function. */ + status = pm_runtime_get(dmadev->ddev.dev); + if (status < 0) + tasklet_schedule(&dmadev->task); + else + hidma_ll_start(dmadev->lldev); +} + +static enum dma_status hidma_tx_status(struct dma_chan *dmach, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct hidma_chan *mchan = to_hidma_chan(dmach); + enum dma_status ret; + + ret = dma_cookie_status(dmach, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (mchan->paused && (ret == DMA_IN_PROGRESS)) { + unsigned long flags; + dma_cookie_t runcookie; + + spin_lock_irqsave(&mchan->lock, flags); + if (mchan->running) + runcookie = mchan->running->desc.cookie; + else + runcookie = -EINVAL; + + if (runcookie == cookie) + ret = DMA_PAUSED; + + spin_unlock_irqrestore(&mchan->lock, flags); + } + + return ret; +} + +/* + * Submit descriptor to hardware. + * Lock the PM for each descriptor we are sending. + */ +static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd) +{ + struct hidma_chan *mchan = to_hidma_chan(txd->chan); + struct hidma_dev *dmadev = mchan->dmadev; + struct hidma_desc *mdesc; + unsigned long irqflags; + dma_cookie_t cookie; + + pm_runtime_get_sync(dmadev->ddev.dev); + if (!hidma_ll_isenabled(dmadev->lldev)) { + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + return -ENODEV; + } + + mdesc = container_of(txd, struct hidma_desc, desc); + spin_lock_irqsave(&mchan->lock, irqflags); + + /* Move descriptor to active */ + list_move_tail(&mdesc->node, &mchan->active); + + /* Update cookie */ + cookie = dma_cookie_assign(txd); + + hidma_ll_queue_request(dmadev->lldev, mdesc->tre_ch); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + return cookie; +} + +static int hidma_alloc_chan_resources(struct dma_chan *dmach) +{ + struct hidma_chan *mchan = to_hidma_chan(dmach); + struct hidma_dev *dmadev = mchan->dmadev; + struct hidma_desc *mdesc, *tmp; + unsigned long irqflags; + LIST_HEAD(descs); + unsigned int i; + int rc = 0; + + if (mchan->allocated) + return 0; + + /* Alloc descriptors for this channel */ + for (i = 0; i < dmadev->nr_descriptors; i++) { + mdesc = kzalloc(sizeof(struct hidma_desc), GFP_NOWAIT); + if (!mdesc) { + rc = -ENOMEM; + break; + } + dma_async_tx_descriptor_init(&mdesc->desc, dmach); + mdesc->desc.tx_submit = hidma_tx_submit; + + rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, + "DMA engine", hidma_callback, mdesc, + &mdesc->tre_ch); + if (rc) { + dev_err(dmach->device->dev, + "channel alloc failed at %u\n", i); + kfree(mdesc); + break; + } + list_add_tail(&mdesc->node, &descs); + } + + if (rc) { + /* return the allocated descriptors */ + list_for_each_entry_safe(mdesc, tmp, &descs, node) { + hidma_ll_free(dmadev->lldev, mdesc->tre_ch); + kfree(mdesc); + } + return rc; + } + + spin_lock_irqsave(&mchan->lock, irqflags); + list_splice_tail_init(&descs, &mchan->free); + mchan->allocated = true; + spin_unlock_irqrestore(&mchan->lock, irqflags); + return 1; +} + +static struct dma_async_tx_descriptor * +hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct hidma_chan *mchan = to_hidma_chan(dmach); + struct hidma_desc *mdesc = NULL; + struct hidma_dev *mdma = mchan->dmadev; + unsigned long irqflags; + + /* Get free descriptor */ + spin_lock_irqsave(&mchan->lock, irqflags); + if (!list_empty(&mchan->free)) { + mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); + list_del(&mdesc->node); + } + spin_unlock_irqrestore(&mchan->lock, irqflags); + + if (!mdesc) + return NULL; + + hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, + src, dest, len, flags); + + /* Place descriptor in prepared list */ + spin_lock_irqsave(&mchan->lock, irqflags); + list_add_tail(&mdesc->node, &mchan->prepared); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + return &mdesc->desc; +} + +static int hidma_terminate_channel(struct dma_chan *chan) +{ + struct hidma_chan *mchan = to_hidma_chan(chan); + struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); + struct hidma_desc *tmp, *mdesc; + unsigned long irqflags; + LIST_HEAD(list); + int rc; + + pm_runtime_get_sync(dmadev->ddev.dev); + /* give completed requests a chance to finish */ + hidma_process_completed(mchan); + + spin_lock_irqsave(&mchan->lock, irqflags); + list_splice_init(&mchan->active, &list); + list_splice_init(&mchan->prepared, &list); + list_splice_init(&mchan->completed, &list); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + /* this suspends the existing transfer */ + rc = hidma_ll_pause(dmadev->lldev); + if (rc) { + dev_err(dmadev->ddev.dev, "channel did not pause\n"); + goto out; + } + + /* return all user requests */ + list_for_each_entry_safe(mdesc, tmp, &list, node) { + struct dma_async_tx_descriptor *txd = &mdesc->desc; + dma_async_tx_callback callback = mdesc->desc.callback; + void *param = mdesc->desc.callback_param; + + dma_descriptor_unmap(txd); + + if (callback) + callback(param); + + dma_run_dependencies(txd); + + /* move myself to free_list */ + list_move(&mdesc->node, &mchan->free); + } + + rc = hidma_ll_resume(dmadev->lldev); +out: + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + return rc; +} + +static int hidma_terminate_all(struct dma_chan *chan) +{ + struct hidma_chan *mchan = to_hidma_chan(chan); + struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); + int rc; + + rc = hidma_terminate_channel(chan); + if (rc) + return rc; + + /* reinitialize the hardware */ + pm_runtime_get_sync(dmadev->ddev.dev); + rc = hidma_ll_setup(dmadev->lldev); + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + return rc; +} + +static void hidma_free_chan_resources(struct dma_chan *dmach) +{ + struct hidma_chan *mchan = to_hidma_chan(dmach); + struct hidma_dev *mdma = mchan->dmadev; + struct hidma_desc *mdesc, *tmp; + unsigned long irqflags; + LIST_HEAD(descs); + + /* terminate running transactions and free descriptors */ + hidma_terminate_channel(dmach); + + spin_lock_irqsave(&mchan->lock, irqflags); + + /* Move data */ + list_splice_tail_init(&mchan->free, &descs); + + /* Free descriptors */ + list_for_each_entry_safe(mdesc, tmp, &descs, node) { + hidma_ll_free(mdma->lldev, mdesc->tre_ch); + list_del(&mdesc->node); + kfree(mdesc); + } + + mchan->allocated = 0; + spin_unlock_irqrestore(&mchan->lock, irqflags); +} + +static int hidma_pause(struct dma_chan *chan) +{ + struct hidma_chan *mchan; + struct hidma_dev *dmadev; + + mchan = to_hidma_chan(chan); + dmadev = to_hidma_dev(mchan->chan.device); + if (!mchan->paused) { + pm_runtime_get_sync(dmadev->ddev.dev); + if (hidma_ll_pause(dmadev->lldev)) + dev_warn(dmadev->ddev.dev, "channel did not stop\n"); + mchan->paused = true; + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + } + return 0; +} + +static int hidma_resume(struct dma_chan *chan) +{ + struct hidma_chan *mchan; + struct hidma_dev *dmadev; + int rc = 0; + + mchan = to_hidma_chan(chan); + dmadev = to_hidma_dev(mchan->chan.device); + if (mchan->paused) { + pm_runtime_get_sync(dmadev->ddev.dev); + rc = hidma_ll_resume(dmadev->lldev); + if (!rc) + mchan->paused = false; + else + dev_err(dmadev->ddev.dev, + "failed to resume the channel"); + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + } + return rc; +} + +static irqreturn_t hidma_chirq_handler(int chirq, void *arg) +{ + struct hidma_lldev *lldev = arg; + + /* + * All interrupts are request driven. + * HW doesn't send an interrupt by itself. + */ + return hidma_ll_inthandler(chirq, lldev); +} + +static int hidma_probe(struct platform_device *pdev) +{ + struct hidma_dev *dmadev; + struct resource *trca_resource; + struct resource *evca_resource; + int chirq; + void __iomem *evca; + void __iomem *trca; + int rc; + + pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); + trca = devm_ioremap_resource(&pdev->dev, trca_resource); + if (IS_ERR(trca)) { + rc = -ENOMEM; + goto bailout; + } + + evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1); + evca = devm_ioremap_resource(&pdev->dev, evca_resource); + if (IS_ERR(evca)) { + rc = -ENOMEM; + goto bailout; + } + + /* + * This driver only handles the channel IRQs. + * Common IRQ is handled by the management driver. + */ + chirq = platform_get_irq(pdev, 0); + if (chirq < 0) { + rc = -ENODEV; + goto bailout; + } + + dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); + if (!dmadev) { + rc = -ENOMEM; + goto bailout; + } + + INIT_LIST_HEAD(&dmadev->ddev.channels); + spin_lock_init(&dmadev->lock); + dmadev->ddev.dev = &pdev->dev; + pm_runtime_get_sync(dmadev->ddev.dev); + + dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); + if (WARN_ON(!pdev->dev.dma_mask)) { + rc = -ENXIO; + goto dmafree; + } + + dmadev->dev_evca = evca; + dmadev->evca_resource = evca_resource; + dmadev->dev_trca = trca; + dmadev->trca_resource = trca_resource; + dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; + dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; + dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; + dmadev->ddev.device_tx_status = hidma_tx_status; + dmadev->ddev.device_issue_pending = hidma_issue_pending; + dmadev->ddev.device_pause = hidma_pause; + dmadev->ddev.device_resume = hidma_resume; + dmadev->ddev.device_terminate_all = hidma_terminate_all; + dmadev->ddev.copy_align = 8; + + device_property_read_u32(&pdev->dev, "desc-count", + &dmadev->nr_descriptors); + + if (!dmadev->nr_descriptors && nr_desc_prm) + dmadev->nr_descriptors = nr_desc_prm; + + if (!dmadev->nr_descriptors) + dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; + + dmadev->chidx = readl(dmadev->dev_trca + 0x28); + + /* Set DMA mask to 64 bits. */ + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) { + dev_warn(&pdev->dev, "unable to set coherent mask to 64"); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) + goto dmafree; + } + + dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, + dmadev->nr_descriptors, dmadev->dev_trca, + dmadev->dev_evca, dmadev->chidx); + if (!dmadev->lldev) { + rc = -EPROBE_DEFER; + goto dmafree; + } + + rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 0, + "qcom-hidma", dmadev->lldev); + if (rc) + goto uninit; + + INIT_LIST_HEAD(&dmadev->ddev.channels); + rc = hidma_chan_init(dmadev, 0); + if (rc) + goto uninit; + + rc = dma_async_device_register(&dmadev->ddev); + if (rc) + goto uninit; + + dmadev->irq = chirq; + tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev); + dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); + platform_set_drvdata(pdev, dmadev); + pm_runtime_mark_last_busy(dmadev->ddev.dev); + pm_runtime_put_autosuspend(dmadev->ddev.dev); + return 0; + +uninit: + hidma_ll_uninit(dmadev->lldev); +dmafree: + if (dmadev) + hidma_free(dmadev); +bailout: + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + return rc; +} + +static int hidma_remove(struct platform_device *pdev) +{ + struct hidma_dev *dmadev = platform_get_drvdata(pdev); + + pm_runtime_get_sync(dmadev->ddev.dev); + dma_async_device_unregister(&dmadev->ddev); + devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); + hidma_ll_uninit(dmadev->lldev); + hidma_free(dmadev); + + dev_info(&pdev->dev, "HI-DMA engine removed\n"); + pm_runtime_put_sync_suspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + return 0; +} + +#if IS_ENABLED(CONFIG_ACPI) +static const struct acpi_device_id hidma_acpi_ids[] = { + {"QCOM8061"}, + {}, +}; +#endif + +static const struct of_device_id hidma_match[] = { + {.compatible = "qcom,hidma-1.0",}, + {}, +}; + +MODULE_DEVICE_TABLE(of, hidma_match); + +static struct platform_driver hidma_driver = { + .probe = hidma_probe, + .remove = hidma_remove, + .driver = { + .name = "hidma", + .of_match_table = hidma_match, + .acpi_match_table = ACPI_PTR(hidma_acpi_ids), + }, +}; + +module_platform_driver(hidma_driver); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h new file mode 100644 index 000000000000..231e306f6d87 --- /dev/null +++ b/drivers/dma/qcom/hidma.h @@ -0,0 +1,160 @@ +/* + * Qualcomm Technologies HIDMA data structures + * + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef QCOM_HIDMA_H +#define QCOM_HIDMA_H + +#include +#include +#include + +#define TRE_SIZE 32 /* each TRE is 32 bytes */ +#define TRE_CFG_IDX 0 +#define TRE_LEN_IDX 1 +#define TRE_SRC_LOW_IDX 2 +#define TRE_SRC_HI_IDX 3 +#define TRE_DEST_LOW_IDX 4 +#define TRE_DEST_HI_IDX 5 + +struct hidma_tx_status { + u8 err_info; /* error record in this transfer */ + u8 err_code; /* completion code */ +}; + +struct hidma_tre { + atomic_t allocated; /* if this channel is allocated */ + bool queued; /* flag whether this is pending */ + u16 status; /* status */ + u32 chidx; /* index of the tre */ + u32 dma_sig; /* signature of the tre */ + const char *dev_name; /* name of the device */ + void (*callback)(void *data); /* requester callback */ + void *data; /* Data associated with this channel*/ + struct hidma_lldev *lldev; /* lldma device pointer */ + u32 tre_local[TRE_SIZE / sizeof(u32) + 1]; /* TRE local copy */ + u32 tre_index; /* the offset where this was written*/ + u32 int_flags; /* interrupt flags */ +}; + +struct hidma_lldev { + bool initialized; /* initialized flag */ + u8 trch_state; /* trch_state of the device */ + u8 evch_state; /* evch_state of the device */ + u8 chidx; /* channel index in the core */ + u32 nr_tres; /* max number of configs */ + spinlock_t lock; /* reentrancy */ + struct hidma_tre *trepool; /* trepool of user configs */ + struct device *dev; /* device */ + void __iomem *trca; /* Transfer Channel address */ + void __iomem *evca; /* Event Channel address */ + struct hidma_tre + **pending_tre_list; /* Pointers to pending TREs */ + struct hidma_tx_status + *tx_status_list; /* Pointers to pending TREs status*/ + s32 pending_tre_count; /* Number of TREs pending */ + + void *tre_ring; /* TRE ring */ + dma_addr_t tre_ring_handle; /* TRE ring to be shared with HW */ + u32 tre_ring_size; /* Byte size of the ring */ + u32 tre_processed_off; /* last processed TRE */ + + void *evre_ring; /* EVRE ring */ + dma_addr_t evre_ring_handle; /* EVRE ring to be shared with HW */ + u32 evre_ring_size; /* Byte size of the ring */ + u32 evre_processed_off; /* last processed EVRE */ + + u32 tre_write_offset; /* TRE write location */ + struct tasklet_struct task; /* task delivering notifications */ + DECLARE_KFIFO_PTR(handoff_fifo, + struct hidma_tre *); /* pending TREs FIFO */ +}; + +struct hidma_desc { + struct dma_async_tx_descriptor desc; + /* link list node for this channel*/ + struct list_head node; + u32 tre_ch; +}; + +struct hidma_chan { + bool paused; + bool allocated; + char dbg_name[16]; + u32 dma_sig; + + /* + * active descriptor on this channel + * It is used by the DMA complete notification to + * locate the descriptor that initiated the transfer. + */ + struct dentry *debugfs; + struct dentry *stats; + struct hidma_dev *dmadev; + struct hidma_desc *running; + + struct dma_chan chan; + struct list_head free; + struct list_head prepared; + struct list_head active; + struct list_head completed; + + /* Lock for this structure */ + spinlock_t lock; +}; + +struct hidma_dev { + int irq; + int chidx; + u32 nr_descriptors; + + struct hidma_lldev *lldev; + void __iomem *dev_trca; + struct resource *trca_resource; + void __iomem *dev_evca; + struct resource *evca_resource; + + /* used to protect the pending channel list*/ + spinlock_t lock; + struct dma_device ddev; + + struct dentry *debugfs; + struct dentry *stats; + + /* Task delivering issue_pending */ + struct tasklet_struct task; +}; + +int hidma_ll_request(struct hidma_lldev *llhndl, u32 dev_id, + const char *dev_name, + void (*callback)(void *data), void *data, u32 *tre_ch); + +void hidma_ll_free(struct hidma_lldev *llhndl, u32 tre_ch); +enum dma_status hidma_ll_status(struct hidma_lldev *llhndl, u32 tre_ch); +bool hidma_ll_isenabled(struct hidma_lldev *llhndl); +void hidma_ll_queue_request(struct hidma_lldev *llhndl, u32 tre_ch); +void hidma_ll_start(struct hidma_lldev *llhndl); +int hidma_ll_pause(struct hidma_lldev *llhndl); +int hidma_ll_resume(struct hidma_lldev *llhndl); +void hidma_ll_set_transfer_params(struct hidma_lldev *llhndl, u32 tre_ch, + dma_addr_t src, dma_addr_t dest, u32 len, u32 flags); +int hidma_ll_setup(struct hidma_lldev *lldev); +struct hidma_lldev *hidma_ll_init(struct device *dev, u32 max_channels, + void __iomem *trca, void __iomem *evca, + u8 chidx); +int hidma_ll_uninit(struct hidma_lldev *llhndl); +irqreturn_t hidma_ll_inthandler(int irq, void *arg); +void hidma_cleanup_pending_tre(struct hidma_lldev *llhndl, u8 err_info, + u8 err_code); +#endif -- cgit v1.2.3 From c997e30e7f65f00832abc5d92f7fd3d6ca325402 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 10 Mar 2016 16:18:40 -0700 Subject: dmaengine: IOATDMA: revise channel reset workaround on CB3.3 platforms Previously we unloaded the interrupts and reloaded in order to work around a channel reset bug that cleared the MSIX table. This approach just isn't practical when a reset needs to happen in the error handler that just happens to be running in interrupt context (bottom half). It looks like we can work around the hardware issue by just storing a shadow copy of the MSIX table and restore it after reset. Signed-off-by: Dave Jiang Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 50 ++++++++++++++------------------------------------ drivers/dma/ioat/dma.h | 5 +++++ 2 files changed, 19 insertions(+), 36 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 14ae9a0994fd..5428746f03fb 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -804,40 +804,6 @@ ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, return dma_cookie_status(c, cookie, txstate); } -static int ioat_irq_reinit(struct ioatdma_device *ioat_dma) -{ - struct pci_dev *pdev = ioat_dma->pdev; - int irq = pdev->irq, i; - - if (!is_bwd_ioat(pdev)) - return 0; - - switch (ioat_dma->irq_mode) { - case IOAT_MSIX: - for (i = 0; i < ioat_dma->dma_dev.chancnt; i++) { - struct msix_entry *msix = &ioat_dma->msix_entries[i]; - struct ioatdma_chan *ioat_chan; - - ioat_chan = ioat_chan_by_index(ioat_dma, i); - devm_free_irq(&pdev->dev, msix->vector, ioat_chan); - } - - pci_disable_msix(pdev); - break; - case IOAT_MSI: - pci_disable_msi(pdev); - /* fall through */ - case IOAT_INTX: - devm_free_irq(&pdev->dev, irq, ioat_dma); - break; - default: - return 0; - } - ioat_dma->irq_mode = IOAT_NOIRQ; - - return ioat_dma_setup_interrupts(ioat_dma); -} - int ioat_reset_hw(struct ioatdma_chan *ioat_chan) { /* throw away whatever the channel was doing and get it @@ -877,9 +843,21 @@ int ioat_reset_hw(struct ioatdma_chan *ioat_chan) } } + if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { + ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000); + ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008); + ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800); + } + + err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200)); - if (!err) - err = ioat_irq_reinit(ioat_dma); + if (!err) { + if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { + writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000); + writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008); + writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800); + } + } if (err) dev_err(&pdev->dev, "Failed to reset: %d\n", err); diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 57a9b83db455..a9bc1a15b0d1 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -86,6 +86,11 @@ struct ioatdma_device { struct dca_provider *dca; enum ioat_irq_mode irq_mode; u32 cap; + + /* shadow version for CB3.3 chan reset errata workaround */ + u64 msixtba0; + u64 msixdata0; + u32 msixpba; }; struct ioat_descs { -- cgit v1.2.3