diff options
| author | Linus Torvalds | 2011-01-17 10:54:41 -0800 |
|---|---|---|
| committer | Linus Torvalds | 2011-01-17 10:54:41 -0800 |
| commit | e1288cd72f54e7fc16ae9ebb4d0647537ef848d4 (patch) | |
| tree | b4fd87b9307d8041fb680cb9b8fbf787ec968df7 /drivers/dma | |
| parent | e78bf5e6cbe837daa6ab628a5f679548742994d3 (diff) | |
| parent | 94ae85220a07d357d4937086c490854f63344de4 (diff) | |
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: (63 commits)
ARM: PL08x: cleanup comments
Update CONFIG_MD_RAID6_PQ to CONFIG_RAID6_PQ in drivers/dma/iop-adma.c
ARM: PL08x: fix a warning
Fix dmaengine_submit() return type
dmaengine: at_hdmac: fix race while monitoring channel status
dmaengine: at_hdmac: flags located in first descriptor
dmaengine: at_hdmac: use subsys_initcall instead of module_init
dmaengine: at_hdmac: no need set ACK in new descriptor
dmaengine: at_hdmac: trivial add precision to unmapping comment
dmaengine: at_hdmac: use dma_address to program DMA hardware
pch_dma: support new device ML7213 IOH
ARM: PL08x: prevent dma_set_runtime_config() reconfiguring memcpy channels
ARM: PL08x: allow dma_set_runtime_config() to return errors
ARM: PL08x: fix locking between prepare function and submit function
ARM: PL08x: introduce 'phychan_hold' to hold on to physical channels
ARM: PL08x: put txd's on the pending list in pl08x_tx_submit()
ARM: PL08x: rename 'desc_list' as 'pend_list'
ARM: PL08x: implement unmapping of memcpy buffers
ARM: PL08x: store prep_* flags in async_tx structure
ARM: PL08x: shrink srcbus/dstbus in txd structure
...
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/Kconfig | 9 | ||||
| -rw-r--r-- | drivers/dma/amba-pl08x.c | 1168 | ||||
| -rw-r--r-- | drivers/dma/at_hdmac.c | 19 | ||||
| -rw-r--r-- | drivers/dma/fsldma.c | 4 | ||||
| -rw-r--r-- | drivers/dma/intel_mid_dma.c | 33 | ||||
| -rw-r--r-- | drivers/dma/iop-adma.c | 4 | ||||
| -rw-r--r-- | drivers/dma/pch_dma.c | 19 | ||||
| -rw-r--r-- | drivers/dma/ste_dma40.c | 191 | ||||
| -rw-r--r-- | drivers/dma/ste_dma40_ll.c | 246 | ||||
| -rw-r--r-- | drivers/dma/ste_dma40_ll.h | 36 |
10 files changed, 935 insertions, 794 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index ef138731c0ea..1c28816152fa 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -200,11 +200,16 @@ config PL330_DMA platform_data for a dma-pl330 device. config PCH_DMA - tristate "Topcliff (Intel EG20T) PCH DMA support" + tristate "Intel EG20T PCH / OKI SEMICONDUCTOR ML7213 IOH DMA support" depends on PCI && X86 select DMA_ENGINE help - Enable support for the Topcliff (Intel EG20T) PCH DMA engine. + Enable support for Intel EG20T PCH DMA engine. + + This driver also can be used for OKI SEMICONDUCTOR ML7213 IOH(Input/ + Output Hub) which is for IVI(In-Vehicle Infotainment) use. + ML7213 is companion chip for Intel Atom E6xx series. + ML7213 is completely compatible for Intel EG20T PCH. config IMX_SDMA tristate "i.MX SDMA support" diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index b605cc9ac3a2..297f48b0cba9 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -19,14 +19,14 @@ * this program; if not, write to the Free Software Foundation, Inc., 59 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. * - * The full GNU General Public License is iin this distribution in the - * file called COPYING. + * The full GNU General Public License is in this distribution in the file + * called COPYING. * * Documentation: ARM DDI 0196G == PL080 - * Documentation: ARM DDI 0218E == PL081 + * Documentation: ARM DDI 0218E == PL081 * - * PL080 & PL081 both have 16 sets of DMA signals that can be routed to - * any channel. + * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any + * channel. * * The PL080 has 8 channels available for simultaneous use, and the PL081 * has only two channels. So on these DMA controllers the number of channels @@ -53,7 +53,23 @@ * * ASSUMES default (little) endianness for DMA transfers * - * Only DMAC flow control is implemented + * The PL08x has two flow control settings: + * - DMAC flow control: the transfer size defines the number of transfers + * which occur for the current LLI entry, and the DMAC raises TC at the + * end of every LLI entry. Observed behaviour shows the DMAC listening + * to both the BREQ and SREQ signals (contrary to documented), + * transferring data if either is active. The LBREQ and LSREQ signals + * are ignored. + * + * - Peripheral flow control: the transfer size is ignored (and should be + * zero). The data is transferred from the current LLI entry, until + * after the final transfer signalled by LBREQ or LSREQ. The DMAC + * will then move to the next LLI entry. + * + * Only the former works sanely with scatter lists, so we only implement + * the DMAC flow control method. However, peripherals which use the LBREQ + * and LSREQ signals (eg, MMCI) are unable to use this mode, which through + * these hardware restrictions prevents them from using scatter DMA. * * Global TODO: * - Break out common code from arch/arm/mach-s3c64xx and share @@ -61,50 +77,39 @@ #include <linux/device.h> #include <linux/init.h> #include <linux/module.h> -#include <linux/pci.h> #include <linux/interrupt.h> #include <linux/slab.h> #include <linux/dmapool.h> -#include <linux/amba/bus.h> #include <linux/dmaengine.h> +#include <linux/amba/bus.h> #include <linux/amba/pl08x.h> #include <linux/debugfs.h> #include <linux/seq_file.h> #include <asm/hardware/pl080.h> -#include <asm/dma.h> -#include <asm/mach/dma.h> -#include <asm/atomic.h> -#include <asm/processor.h> -#include <asm/cacheflush.h> #define DRIVER_NAME "pl08xdmac" /** - * struct vendor_data - vendor-specific config parameters - * for PL08x derivates - * @name: the name of this specific variant + * struct vendor_data - vendor-specific config parameters for PL08x derivatives * @channels: the number of channels available in this variant - * @dualmaster: whether this version supports dual AHB masters - * or not. + * @dualmaster: whether this version supports dual AHB masters or not. */ struct vendor_data { - char *name; u8 channels; bool dualmaster; }; /* * PL08X private data structures - * An LLI struct - see pl08x TRM - * Note that next uses bit[0] as a bus bit, - * start & end do not - their bus bit info - * is in cctl + * An LLI struct - see PL08x TRM. Note that next uses bit[0] as a bus bit, + * start & end do not - their bus bit info is in cctl. Also note that these + * are fixed 32-bit quantities. */ -struct lli { - dma_addr_t src; - dma_addr_t dst; - dma_addr_t next; +struct pl08x_lli { + u32 src; + u32 dst; + u32 lli; u32 cctl; }; @@ -119,6 +124,8 @@ struct lli { * @phy_chans: array of data for the physical channels * @pool: a pool for the LLI descriptors * @pool_ctr: counter of LLIs in the pool + * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI fetches + * @mem_buses: set to indicate memory transfers on AHB2. * @lock: a spinlock for this struct */ struct pl08x_driver_data { @@ -126,11 +133,13 @@ struct pl08x_driver_data { struct dma_device memcpy; void __iomem *base; struct amba_device *adev; - struct vendor_data *vd; + const struct vendor_data *vd; struct pl08x_platform_data *pd; struct pl08x_phy_chan *phy_chans; struct dma_pool *pool; int pool_ctr; + u8 lli_buses; + u8 mem_buses; spinlock_t lock; }; @@ -152,9 +161,9 @@ struct pl08x_driver_data { /* Size (bytes) of each LLI buffer allocated for one transfer */ # define PL08X_LLI_TSFR_SIZE 0x2000 -/* Maximimum times we call dma_pool_alloc on this pool without freeing */ +/* Maximum times we call dma_pool_alloc on this pool without freeing */ #define PL08X_MAX_ALLOCS 0x40 -#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct lli)) +#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli)) #define PL08X_ALIGN 8 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) @@ -162,6 +171,11 @@ static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) return container_of(chan, struct pl08x_dma_chan, chan); } +static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct pl08x_txd, tx); +} + /* * Physical channel handling */ @@ -177,88 +191,47 @@ static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) /* * Set the initial DMA register values i.e. those for the first LLI - * The next lli pointer and the configuration interrupt bit have - * been set when the LLIs were constructed + * The next LLI pointer and the configuration interrupt bit have + * been set when the LLIs were constructed. Poke them into the hardware + * and start the transfer. */ -static void pl08x_set_cregs(struct pl08x_driver_data *pl08x, - struct pl08x_phy_chan *ch) -{ - /* Wait for channel inactive */ - while (pl08x_phy_channel_busy(ch)) - ; - - dev_vdbg(&pl08x->adev->dev, - "WRITE channel %d: csrc=%08x, cdst=%08x, " - "cctl=%08x, clli=%08x, ccfg=%08x\n", - ch->id, - ch->csrc, - ch->cdst, - ch->cctl, - ch->clli, - ch->ccfg); - - writel(ch->csrc, ch->base + PL080_CH_SRC_ADDR); - writel(ch->cdst, ch->base + PL080_CH_DST_ADDR); - writel(ch->clli, ch->base + PL080_CH_LLI); - writel(ch->cctl, ch->base + PL080_CH_CONTROL); - writel(ch->ccfg, ch->base + PL080_CH_CONFIG); -} - -static inline void pl08x_config_phychan_for_txd(struct pl08x_dma_chan *plchan) +static void pl08x_start_txd(struct pl08x_dma_chan *plchan, + struct pl08x_txd *txd) { - struct pl08x_channel_data *cd = plchan->cd; + struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_phy_chan *phychan = plchan->phychan; - struct pl08x_txd *txd = plchan->at; - - /* Copy the basic control register calculated at transfer config */ - phychan->csrc = txd->csrc; - phychan->cdst = txd->cdst; - phychan->clli = txd->clli; - phychan->cctl = txd->cctl; - - /* Assign the signal to the proper control registers */ - phychan->ccfg = cd->ccfg; - phychan->ccfg &= ~PL080_CONFIG_SRC_SEL_MASK; - phychan->ccfg &= ~PL080_CONFIG_DST_SEL_MASK; - /* If it wasn't set from AMBA, ignore it */ - if (txd->direction == DMA_TO_DEVICE) - /* Select signal as destination */ - phychan->ccfg |= - (phychan->signal << PL080_CONFIG_DST_SEL_SHIFT); - else if (txd->direction == DMA_FROM_DEVICE) - /* Select signal as source */ - phychan->ccfg |= - (phychan->signal << PL080_CONFIG_SRC_SEL_SHIFT); - /* Always enable error interrupts */ - phychan->ccfg |= PL080_CONFIG_ERR_IRQ_MASK; - /* Always enable terminal interrupts */ - phychan->ccfg |= PL080_CONFIG_TC_IRQ_MASK; -} - -/* - * Enable the DMA channel - * Assumes all other configuration bits have been set - * as desired before this code is called - */ -static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x, - struct pl08x_phy_chan *ch) -{ + struct pl08x_lli *lli = &txd->llis_va[0]; u32 val; - /* - * Do not access config register until channel shows as disabled - */ - while (readl(pl08x->base + PL080_EN_CHAN) & (1 << ch->id)) - ; + plchan->at = txd; - /* - * Do not access config register until channel shows as inactive - */ - val = readl(ch->base + PL080_CH_CONFIG); + /* Wait for channel inactive */ + while (pl08x_phy_channel_busy(phychan)) + cpu_relax(); + + dev_vdbg(&pl08x->adev->dev, + "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " + "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", + phychan->id, lli->src, lli->dst, lli->lli, lli->cctl, + txd->ccfg); + + writel(lli->src, phychan->base + PL080_CH_SRC_ADDR); + writel(lli->dst, phychan->base + PL080_CH_DST_ADDR); + writel(lli->lli, phychan->base + PL080_CH_LLI); + writel(lli->cctl, phychan->base + PL080_CH_CONTROL); + writel(txd->ccfg, phychan->base + PL080_CH_CONFIG); + + /* Enable the DMA channel */ + /* Do not access config register until channel shows as disabled */ + while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id)) + cpu_relax(); + + /* Do not access config register until channel shows as inactive */ + val = readl(phychan->base + PL080_CH_CONFIG); while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE)) - val = readl(ch->base + PL080_CH_CONFIG); + val = readl(phychan->base + PL080_CH_CONFIG); - writel(val | PL080_CONFIG_ENABLE, ch->base + PL080_CH_CONFIG); + writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG); } /* @@ -266,10 +239,8 @@ static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x, * * Disabling individual channels could lose data. * - * Disable the peripheral DMA after disabling the DMAC - * in order to allow the DMAC FIFO to drain, and - * hence allow the channel to show inactive - * + * Disable the peripheral DMA after disabling the DMAC in order to allow + * the DMAC FIFO to drain, and hence allow the channel to show inactive */ static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) { @@ -282,7 +253,7 @@ static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) /* Wait for channel inactive */ while (pl08x_phy_channel_busy(ch)) - ; + cpu_relax(); } static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) @@ -333,54 +304,56 @@ static inline u32 get_bytes_in_cctl(u32 cctl) static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) { struct pl08x_phy_chan *ch; - struct pl08x_txd *txdi = NULL; struct pl08x_txd *txd; unsigned long flags; - u32 bytes = 0; + size_t bytes = 0; spin_lock_irqsave(&plchan->lock, flags); - ch = plchan->phychan; txd = plchan->at; /* - * Next follow the LLIs to get the number of pending bytes in the - * currently active transaction. + * Follow the LLIs to get the number of remaining + * bytes in the currently active transaction. */ if (ch && txd) { - struct lli *llis_va = txd->llis_va; - struct lli *llis_bus = (struct lli *) txd->llis_bus; - u32 clli = readl(ch->base + PL080_CH_LLI); + u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2; - /* First get the bytes in the current active LLI */ + /* First get the remaining bytes in the active transfer */ bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL)); if (clli) { - int i = 0; + struct pl08x_lli *llis_va = txd->llis_va; + dma_addr_t llis_bus = txd->llis_bus; + int index; + + BUG_ON(clli < llis_bus || clli >= llis_bus + + sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS); + + /* + * Locate the next LLI - as this is an array, + * it's simple maths to find. + */ + index = (clli - llis_bus) / sizeof(struct pl08x_lli); - /* Forward to the LLI pointed to by clli */ - while ((clli != (u32) &(llis_bus[i])) && - (i < MAX_NUM_TSFR_LLIS)) - i++; + for (; index < MAX_NUM_TSFR_LLIS; index++) { + bytes += get_bytes_in_cctl(llis_va[index].cctl); - while (clli) { - bytes += get_bytes_in_cctl(llis_va[i].cctl); /* - * A clli of 0x00000000 will terminate the - * LLI list + * A LLI pointer of 0 terminates the LLI list */ - clli = llis_va[i].next; - i++; + if (!llis_va[index].lli) + break; } } } /* Sum up all queued transactions */ - if (!list_empty(&plchan->desc_list)) { - list_for_each_entry(txdi, &plchan->desc_list, node) { + if (!list_empty(&plchan->pend_list)) { + struct pl08x_txd *txdi; + list_for_each_entry(txdi, &plchan->pend_list, node) { bytes += txdi->len; } - } spin_unlock_irqrestore(&plchan->lock, flags); @@ -390,6 +363,10 @@ static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) /* * Allocate a physical channel for a virtual channel + * + * Try to locate a physical channel to be used for this transfer. If all + * are taken return NULL and the requester will have to cope by using + * some fallback PIO mode or retrying later. */ static struct pl08x_phy_chan * pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, @@ -399,12 +376,6 @@ pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, unsigned long flags; int i; - /* - * Try to locate a physical channel to be used for - * this transfer. If all are taken return NULL and - * the requester will have to cope by using some fallback - * PIO mode or retrying later. - */ for (i = 0; i < pl08x->vd->channels; i++) { ch = &pl08x->phy_chans[i]; @@ -465,11 +436,11 @@ static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded) } static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, - u32 tsize) + size_t tsize) { u32 retbits = cctl; - /* Remove all src, dst and transfersize bits */ + /* Remove all src, dst and transfer size bits */ retbits &= ~PL080_CONTROL_DWIDTH_MASK; retbits &= ~PL080_CONTROL_SWIDTH_MASK; retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; @@ -509,95 +480,87 @@ static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, return retbits; } +struct pl08x_lli_build_data { + struct pl08x_txd *txd; + struct pl08x_driver_data *pl08x; + struct pl08x_bus_data srcbus; + struct pl08x_bus_data dstbus; + size_t remainder; +}; + /* - * Autoselect a master bus to use for the transfer - * this prefers the destination bus if both available - * if fixed address on one bus the other will be chosen + * Autoselect a master bus to use for the transfer this prefers the + * destination bus if both available if fixed address on one bus the + * other will be chosen */ -void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus, - struct pl08x_bus_data *dst_bus, struct pl08x_bus_data **mbus, - struct pl08x_bus_data **sbus, u32 cctl) +static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd, + struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl) { if (!(cctl & PL080_CONTROL_DST_INCR)) { - *mbus = src_bus; - *sbus = dst_bus; + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { - *mbus = dst_bus; - *sbus = src_bus; + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; } else { - if (dst_bus->buswidth == 4) { - *mbus = dst_bus; - *sbus = src_bus; - } else if (src_bus->buswidth == 4) { - *mbus = src_bus; - *sbus = dst_bus; - } else if (dst_bus->buswidth == 2) { - *mbus = dst_bus; - *sbus = src_bus; - } else if (src_bus->buswidth == 2) { - *mbus = src_bus; - *sbus = dst_bus; + if (bd->dstbus.buswidth == 4) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else if (bd->srcbus.buswidth == 4) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; + } else if (bd->dstbus.buswidth == 2) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else if (bd->srcbus.buswidth == 2) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; } else { - /* src_bus->buswidth == 1 */ - *mbus = dst_bus; - *sbus = src_bus; + /* bd->srcbus.buswidth == 1 */ + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; } } } /* - * Fills in one LLI for a certain transfer descriptor - * and advance the counter + * Fills in one LLI for a certain transfer descriptor and advance the counter */ -int pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, - struct pl08x_txd *txd, int num_llis, int len, - u32 cctl, u32 *remainder) +static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd, + int num_llis, int len, u32 cctl) { - struct lli *llis_va = txd->llis_va; - struct lli *llis_bus = (struct lli *) txd->llis_bus; + struct pl08x_lli *llis_va = bd->txd->llis_va; + dma_addr_t llis_bus = bd->txd->llis_bus; BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); - llis_va[num_llis].cctl = cctl; - llis_va[num_llis].src = txd->srcbus.addr; - llis_va[num_llis].dst = txd->dstbus.addr; - - /* - * On versions with dual masters, you can optionally AND on - * PL080_LLI_LM_AHB2 to the LLI to tell the hardware to read - * in new LLIs with that controller, but we always try to - * choose AHB1 to point into memory. The idea is to have AHB2 - * fixed on the peripheral and AHB1 messing around in the - * memory. So we don't manipulate this bit currently. - */ - - llis_va[num_llis].next = - (dma_addr_t)((u32) &(llis_bus[num_llis + 1])); + llis_va[num_llis].cctl = cctl; + llis_va[num_llis].src = bd->srcbus.addr; + llis_va[num_llis].dst = bd->dstbus.addr; + llis_va[num_llis].lli = llis_bus + (num_llis + 1) * sizeof(struct pl08x_lli); + if (bd->pl08x->lli_buses & PL08X_AHB2) + llis_va[num_llis].lli |= PL080_LLI_LM_AHB2; if (cctl & PL080_CONTROL_SRC_INCR) - txd->srcbus.addr += len; + bd->srcbus.addr += len; if (cctl & PL080_CONTROL_DST_INCR) - txd->dstbus.addr += len; + bd->dstbus.addr += len; - *remainder -= len; + BUG_ON(bd->remainder < len); - return num_llis + 1; + bd->remainder -= len; } /* - * Return number of bytes to fill to boundary, or len + * Return number of bytes to fill to boundary, or len. + * This calculation works for any value of addr. */ -static inline u32 pl08x_pre_boundary(u32 addr, u32 len) +static inline size_t pl08x_pre_boundary(u32 addr, size_t len) { - u32 boundary; - - boundary = ((addr >> PL08X_BOUNDARY_SHIFT) + 1) - << PL08X_BOUNDARY_SHIFT; + size_t boundary_len = PL08X_BOUNDARY_SIZE - + (addr & (PL08X_BOUNDARY_SIZE - 1)); - if (boundary < addr + len) - return boundary - addr; - else - return len; + return min(boundary_len, len); } /* @@ -608,20 +571,13 @@ static inline u32 pl08x_pre_boundary(u32 addr, u32 len) static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, struct pl08x_txd *txd) { - struct pl08x_channel_data *cd = txd->cd; struct pl08x_bus_data *mbus, *sbus; - u32 remainder; + struct pl08x_lli_build_data bd; int num_llis = 0; u32 cctl; - int max_bytes_per_lli; - int total_bytes = 0; - struct lli *llis_va; - struct lli *llis_bus; - - if (!txd) { - dev_err(&pl08x->adev->dev, "%s no descriptor\n", __func__); - return 0; - } + size_t max_bytes_per_lli; + size_t total_bytes = 0; + struct pl08x_lli *llis_va; txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); @@ -632,121 +588,79 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, pl08x->pool_ctr++; - /* - * Initialize bus values for this transfer - * from the passed optimal values - */ - if (!cd) { - dev_err(&pl08x->adev->dev, "%s no channel data\n", __func__); - return 0; - } + /* Get the default CCTL */ + cctl = txd->cctl; - /* Get the default CCTL from the platform data */ - cctl = cd->cctl; - - /* - * On the PL080 we have two bus masters and we - * should select one for source and one for - * destination. We try to use AHB2 for the - * bus which does not increment (typically the - * peripheral) else we just choose something. - */ - cctl &= ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); - if (pl08x->vd->dualmaster) { - if (cctl & PL080_CONTROL_SRC_INCR) - /* Source increments, use AHB2 for destination */ - cctl |= PL080_CONTROL_DST_AHB2; - else if (cctl & PL080_CONTROL_DST_INCR) - /* Destination increments, use AHB2 for source */ - cctl |= PL080_CONTROL_SRC_AHB2; - else - /* Just pick something, source AHB1 dest AHB2 */ - cctl |= PL080_CONTROL_DST_AHB2; - } + bd.txd = txd; + bd.pl08x = pl08x; + bd.srcbus.addr = txd->src_addr; + bd.dstbus.addr = txd->dst_addr; /* Find maximum width of the source bus */ - txd->srcbus.maxwidth = + bd.srcbus.maxwidth = pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >> PL080_CONTROL_SWIDTH_SHIFT); /* Find maximum width of the destination bus */ - txd->dstbus.maxwidth = + bd.dstbus.maxwidth = pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> PL080_CONTROL_DWIDTH_SHIFT); /* Set up the bus widths to the maximum */ - txd->srcbus.buswidth = txd->srcbus.maxwidth; - txd->dstbus.buswidth = txd->dstbus.maxwidth; + bd.srcbus.buswidth = bd.srcbus.maxwidth; + bd.dstbus.buswidth = bd.dstbus.maxwidth; dev_vdbg(&pl08x->adev->dev, "%s source bus is %d bytes wide, dest bus is %d bytes wide\n", - __func__, txd->srcbus.buswidth, txd->dstbus.buswidth); + __func__, bd.srcbus.buswidth, bd.dstbus.buswidth); /* * Bytes transferred == tsize * MIN(buswidths), not max(buswidths) */ - max_bytes_per_lli = min(txd->srcbus.buswidth, txd->dstbus.buswidth) * + max_bytes_per_lli = min(bd.srcbus.buswidth, bd.dstbus.buswidth) * PL080_CONTROL_TRANSFER_SIZE_MASK; dev_vdbg(&pl08x->adev->dev, - "%s max bytes per lli = %d\n", + "%s max bytes per lli = %zu\n", __func__, max_bytes_per_lli); /* We need to count this down to zero */ - remainder = txd->len; + bd.remainder = txd->len; dev_vdbg(&pl08x->adev->dev, - "%s remainder = %d\n", - __func__, remainder); + "%s remainder = %zu\n", + __func__, bd.remainder); /* * Choose bus to align to * - prefers destination bus if both available * - if fixed address on one bus chooses other - * - modifies cctl to choose an apropriate master - */ - pl08x_choose_master_bus(&txd->srcbus, &txd->dstbus, - &mbus, &sbus, cctl); - - - /* - * The lowest bit of the LLI register - * is also used to indicate which master to - * use for reading the LLIs. */ + pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); if (txd->len < mbus->buswidth) { - /* - * Less than a bus width available - * - send as single bytes - */ - while (remainder) { + /* Less than a bus width available - send as single bytes */ + while (bd.remainder) { dev_vdbg(&pl08x->adev->dev, "%s single byte LLIs for a transfer of " - "less than a bus width (remain %08x)\n", - __func__, remainder); + "less than a bus width (remain 0x%08x)\n", + __func__, bd.remainder); cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - num_llis = - pl08x_fill_lli_for_desc(pl08x, txd, num_llis, 1, - cctl, &remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } } else { - /* - * Make one byte LLIs until master bus is aligned - * - slave will then be aligned also - */ + /* Make one byte LLIs until master bus is aligned */ while ((mbus->addr) % (mbus->buswidth)) { dev_vdbg(&pl08x->adev->dev, "%s adjustment lli for less than bus width " - "(remain %08x)\n", - __func__, remainder); + "(remain 0x%08x)\n", + __func__, bd.remainder); cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - num_llis = pl08x_fill_lli_for_desc - (pl08x, txd, num_llis, 1, cctl, &remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } /* - * Master now aligned + * Master now aligned * - if slave is not then we must set its width down */ if (sbus->addr % sbus->buswidth) { @@ -761,63 +675,51 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, * Make largest possible LLIs until less than one bus * width left */ - while (remainder > (mbus->buswidth - 1)) { - int lli_len, target_len; - int tsize; - int odd_bytes; + while (bd.remainder > (mbus->buswidth - 1)) { + size_t lli_len, target_len, tsize, odd_bytes; /* * If enough left try to send max possible, * otherwise try to send the remainder */ - target_len = remainder; - if (remainder > max_bytes_per_lli) - target_len = max_bytes_per_lli; + target_len = min(bd.remainder, max_bytes_per_lli); /* - * Set bus lengths for incrementing busses - * to number of bytes which fill to next memory - * boundary + * Set bus lengths for incrementing buses to the + * number of bytes which fill to next memory boundary, + * limiting on the target length calculated above. */ if (cctl & PL080_CONTROL_SRC_INCR) - txd->srcbus.fill_bytes = - pl08x_pre_boundary( - txd->srcbus.addr, - remainder); + bd.srcbus.fill_bytes = + pl08x_pre_boundary(bd.srcbus.addr, + target_len); else - txd->srcbus.fill_bytes = - max_bytes_per_lli; + bd.srcbus.fill_bytes = target_len; if (cctl & PL080_CONTROL_DST_INCR) - txd->dstbus.fill_bytes = - pl08x_pre_boundary( - txd->dstbus.addr, - remainder); + bd.dstbus.fill_bytes = + pl08x_pre_boundary(bd.dstbus.addr, + target_len); else - txd->dstbus.fill_bytes = - max_bytes_per_lli; + bd.dstbus.fill_bytes = target_len; - /* - * Find the nearest - */ - lli_len = min(txd->srcbus.fill_bytes, - txd->dstbus.fill_bytes); + /* Find the nearest */ + lli_len = min(bd.srcbus.fill_bytes, + bd.dstbus.fill_bytes); - BUG_ON(lli_len > remainder); + BUG_ON(lli_len > bd.remainder); if (lli_len <= 0) { dev_err(&pl08x->adev->dev, - "%s lli_len is %d, <= 0\n", + "%s lli_len is %zu, <= 0\n", __func__, lli_len); return 0; } if (lli_len == target_len) { /* - * Can send what we wanted - */ - /* - * Maintain alignment + * Can send what we wanted. + * Maintain alignment */ lli_len = (lli_len/mbus->buswidth) * mbus->buswidth; @@ -825,17 +727,14 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, } else { /* * So now we know how many bytes to transfer - * to get to the nearest boundary - * The next lli will past the boundary - * - however we may be working to a boundary - * on the slave bus - * We need to ensure the master stays aligned + * to get to the nearest boundary. The next + * LLI will past the boundary. However, we + * may be working to a boundary on the slave + * bus. We need to ensure the master stays + * aligned, and that we are working in + * multiples of the bus widths. */ odd_bytes = lli_len % mbus->buswidth; - /* - * - and that we are working in multiples - * of the bus widths - */ lli_len -= odd_bytes; } @@ -855,41 +754,38 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, if (target_len != lli_len) { dev_vdbg(&pl08x->adev->dev, - "%s can't send what we want. Desired %08x, lli of %08x bytes in txd of %08x\n", + "%s can't send what we want. Desired 0x%08zx, lli of 0x%08zx bytes in txd of 0x%08zx\n", __func__, target_len, lli_len, txd->len); } cctl = pl08x_cctl_bits(cctl, - txd->srcbus.buswidth, - txd->dstbus.buswidth, + bd.srcbus.buswidth, + bd.dstbus.buswidth, tsize); dev_vdbg(&pl08x->adev->dev, - "%s fill lli with single lli chunk of size %08x (remainder %08x)\n", - __func__, lli_len, remainder); - num_llis = pl08x_fill_lli_for_desc(pl08x, txd, - num_llis, lli_len, cctl, - &remainder); + "%s fill lli with single lli chunk of size 0x%08zx (remainder 0x%08zx)\n", + __func__, lli_len, bd.remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, + lli_len, cctl); total_bytes += lli_len; } if (odd_bytes) { /* - * Creep past the boundary, - * maintaining master alignment + * Creep past the boundary, maintaining + * master alignment */ int j; for (j = 0; (j < mbus->buswidth) - && (remainder); j++) { + && (bd.remainder); j++) { cctl = pl08x_cctl_bits(cctl, 1, 1, 1); dev_vdbg(&pl08x->adev->dev, - "%s align with boundardy, single byte (remain %08x)\n", - __func__, remainder); - num_llis = - pl08x_fill_lli_for_desc(pl08x, - txd, num_llis, 1, - cctl, &remainder); + "%s align with boundary, single byte (remain 0x%08zx)\n", + __func__, bd.remainder); + pl08x_fill_lli_for_desc(&bd, + num_llis++, 1, cctl); total_bytes++; } } @@ -898,25 +794,18 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, /* * Send any odd bytes */ - if (remainder < 0) { - dev_err(&pl08x->adev->dev, "%s remainder not fitted 0x%08x bytes\n", - __func__, remainder); - return 0; - } - - while (remainder) { + while (bd.remainder) { cctl = pl08x_cctl_bits(cctl, 1, 1, 1); dev_vdbg(&pl08x->adev->dev, - "%s align with boundardy, single odd byte (remain %d)\n", - __func__, remainder); - num_llis = pl08x_fill_lli_for_desc(pl08x, txd, num_llis, - 1, cctl, &remainder); + "%s align with boundary, single odd byte (remain %zu)\n", + __func__, bd.remainder); + pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); total_bytes++; } } if (total_bytes != txd->len) { dev_err(&pl08x->adev->dev, - "%s size of encoded lli:s don't match total txd, transferred 0x%08x from size 0x%08x\n", + "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", __func__, total_bytes, txd->len); return 0; } @@ -927,41 +816,12 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, __func__, (u32) MAX_NUM_TSFR_LLIS); return 0; } - /* - * Decide whether this is a loop or a terminated transfer - */ - llis_va = txd->llis_va; - llis_bus = (struct lli *) txd->llis_bus; - if (cd->circular_buffer) { - /* - * Loop the circular buffer so that the next element - * points back to the beginning of the LLI. - */ - llis_va[num_llis - 1].next = - (dma_addr_t)((unsigned int)&(llis_bus[0])); - } else { - /* - * On non-circular buffers, the final LLI terminates - * the LLI. - */ - llis_va[num_llis - 1].next = 0; - /* - * The final LLI element shall also fire an interrupt - */ - llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN; - } - - /* Now store the channel register values */ - txd->csrc = llis_va[0].src; - txd->cdst = llis_va[0].dst; - if (num_llis > 1) - txd->clli = llis_va[0].next; - else - txd->clli = 0; - - txd->cctl = llis_va[0].cctl; - /* ccfg will be set at physical channel allocation time */ + llis_va = txd->llis_va; + /* The final LLI terminates the LLI. */ + llis_va[num_llis - 1].lli = 0; + /* The final LLI element shall also fire an interrupt. */ + llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN; #ifdef VERBOSE_DEBUG { @@ -969,13 +829,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, for (i = 0; i < num_llis; i++) { dev_vdbg(&pl08x->adev->dev, - "lli %d @%p: csrc=%08x, cdst=%08x, cctl=%08x, clli=%08x\n", + "lli %d @%p: csrc=0x%08x, cdst=0x%08x, cctl=0x%08x, clli=0x%08x\n", i, &llis_va[i], llis_va[i].src, llis_va[i].dst, llis_va[i].cctl, - llis_va[i].next + llis_va[i].lli ); } } @@ -988,14 +848,8 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, static void pl08x_free_txd(struct pl08x_driver_data *pl08x, struct pl08x_txd *txd) { - if (!txd) - dev_err(&pl08x->adev->dev, - "%s no descriptor to free\n", - __func__); - /* Free the LLI */ - dma_pool_free(pl08x->pool, txd->llis_va, - txd->llis_bus); + dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); pl08x->pool_ctr--; @@ -1008,13 +862,12 @@ static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, struct pl08x_txd *txdi = NULL; struct pl08x_txd *next; - if (!list_empty(&plchan->desc_list)) { + if (!list_empty(&plchan->pend_list)) { list_for_each_entry_safe(txdi, - next, &plchan->desc_list, node) { + next, &plchan->pend_list, node) { list_del(&txdi->node); pl08x_free_txd(pl08x, txdi); } - } } @@ -1069,6 +922,12 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan, return -EBUSY; } ch->signal = ret; + + /* Assign the flow control signal to this channel */ + if (txd->direction == DMA_TO_DEVICE) + txd->ccfg |= ch->signal << PL080_CONFIG_DST_SEL_SHIFT; + else if (txd->direction == DMA_FROM_DEVICE) + txd->ccfg |= ch->signal << PL080_CONFIG_SRC_SEL_SHIFT; } dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n", @@ -1076,19 +935,54 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan, ch->signal, plchan->name); + plchan->phychan_hold++; plchan->phychan = ch; return 0; } +static void release_phy_channel(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + + if ((plchan->phychan->signal >= 0) && pl08x->pd->put_signal) { + pl08x->pd->put_signal(plchan); + plchan->phychan->signal = -1; + } + pl08x_put_phy_channel(pl08x, plchan->phychan); + plchan->phychan = NULL; +} + static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx) { struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan); + struct pl08x_txd *txd = to_pl08x_txd(tx); + unsigned long flags; - atomic_inc(&plchan->last_issued); - tx->cookie = atomic_read(&plchan->last_issued); - /* This unlock follows the lock in the prep() function */ - spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); + spin_lock_irqsave(&plchan->lock, flags); + + plchan->chan.cookie += 1; + if (plchan->chan.cookie < 0) + plchan->chan.cookie = 1; + tx->cookie = plchan->chan.cookie; + + /* Put this onto the pending list */ + list_add_tail(&txd->node, &plchan->pend_list); + + /* + * If there was no physical channel available for this memcpy, + * stack the request up and indicate that the channel is waiting + * for a free physical channel. + */ + if (!plchan->slave && !plchan->phychan) { + /* Do this memcpy whenever there is a channel ready */ + plchan->state = PL08X_CHAN_WAITING; + plchan->waiting = txd; + } else { + plchan->phychan_hold--; + } + + spin_unlock_irqrestore(&plchan->lock, flags); return tx->cookie; } @@ -1102,10 +996,9 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( } /* - * Code accessing dma_async_is_complete() in a tight loop - * may give problems - could schedule where indicated. - * If slaves are relying on interrupts to signal completion this - * function must not be called with interrupts disabled + * Code accessing dma_async_is_complete() in a tight loop may give problems. + * If slaves are relying on interrupts to signal completion this function + * must not be called with interrupts disabled. */ static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, @@ -1118,7 +1011,7 @@ pl08x_dma_tx_status(struct dma_chan *chan, enum dma_status ret; u32 bytesleft = 0; - last_used = atomic_read(&plchan->last_issued); + last_used = plchan->chan.cookie; last_complete = plchan->lc; ret = dma_async_is_complete(cookie, last_complete, last_used); @@ -1128,13 +1021,9 @@ pl08x_dma_tx_status(struct dma_chan *chan, } /* - * schedule(); could be inserted here - */ - - /* * This cookie not complete yet */ - last_used = atomic_read(&plchan->last_issued); + last_used = plchan->chan.cookie; last_complete = plchan->lc; /* Get number of bytes left in the active transactions and queue */ @@ -1199,37 +1088,35 @@ static const struct burst_table burst_sizes[] = { }, }; -static void dma_set_runtime_config(struct dma_chan *chan, - struct dma_slave_config *config) +static int dma_set_runtime_config(struct dma_chan *chan, + struct dma_slave_config *config) { struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_channel_data *cd = plchan->cd; enum dma_slave_buswidth addr_width; + dma_addr_t addr; u32 maxburst; u32 cctl = 0; - /* Mask out all except src and dst channel */ - u32 ccfg = cd->ccfg & 0x000003DEU; - int i = 0; + int i; + + if (!plchan->slave) + return -EINVAL; /* Transfer direction */ plchan->runtime_direction = config->direction; if (config->direction == DMA_TO_DEVICE) { - plchan->runtime_addr = config->dst_addr; - cctl |= PL080_CONTROL_SRC_INCR; - ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT; + addr = config->dst_addr; addr_width = config->dst_addr_width; maxburst = config->dst_maxburst; } else if (config->direction == DMA_FROM_DEVICE) { - plchan->runtime_addr = config->src_addr; - cctl |= PL080_CONTROL_DST_INCR; - ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + addr = config->src_addr; addr_width = config->src_addr_width; maxburst = config->src_maxburst; } else { dev_err(&pl08x->adev->dev, "bad runtime_config: alien transfer direction\n"); - return; + return -EINVAL; } switch (addr_width) { @@ -1248,42 +1135,40 @@ static void dma_set_runtime_config(struct dma_chan *chan, default: dev_err(&pl08x->adev->dev, "bad runtime_config: alien address width\n"); - return; + return -EINVAL; } /* * Now decide on a maxburst: - * If this channel will only request single transfers, set - * this down to ONE element. + * If this channel will only request single transfers, set this + * down to ONE element. Also select one element if no maxburst + * is specified. */ - if (plchan->cd->single) { + if (plchan->cd->single || maxburst == 0) { cctl |= (PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT) | (PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT); } else { - while (i < ARRAY_SIZE(burst_sizes)) { + for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) if (burst_sizes[i].burstwords <= maxburst) break; - i++; - } cctl |= burst_sizes[i].reg; } - /* Access the cell in privileged mode, non-bufferable, non-cacheable */ - cctl &= ~PL080_CONTROL_PROT_MASK; - cctl |= PL080_CONTROL_PROT_SYS; + plchan->runtime_addr = addr; /* Modify the default channel data to fit PrimeCell request */ cd->cctl = cctl; - cd->ccfg = ccfg; dev_dbg(&pl08x->adev->dev, "configured channel %s (%s) for %s, data width %d, " - "maxburst %d words, LE, CCTL=%08x, CCFG=%08x\n", + "maxburst %d words, LE, CCTL=0x%08x\n", dma_chan_name(chan), plchan->name, (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", addr_width, maxburst, - cctl, ccfg); + cctl); + + return 0; } /* @@ -1293,35 +1178,26 @@ static void dma_set_runtime_config(struct dma_chan *chan, static void pl08x_issue_pending(struct dma_chan *chan) { struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); - struct pl08x_driver_data *pl08x = plchan->host; unsigned long flags; spin_lock_irqsave(&plchan->lock, flags); - /* Something is already active */ - if (plchan->at) { - spin_unlock_irqrestore(&plchan->lock, flags); - return; - } - - /* Didn't get a physical channel so waiting for it ... */ - if (plchan->state == PL08X_CHAN_WAITING) + /* Something is already active, or we're waiting for a channel... */ + if (plchan->at || plchan->state == PL08X_CHAN_WAITING) { + spin_unlock_irqrestore(&plchan->lock, flags); return; + } /* Take the first element in the queue and execute it */ - if (!list_empty(&plchan->desc_list)) { + if (!list_empty(&plchan->pend_list)) { struct pl08x_txd *next; - next = list_first_entry(&plchan->desc_list, + next = list_first_entry(&plchan->pend_list, struct pl08x_txd, node); list_del(&next->node); - plchan->at = next; plchan->state = PL08X_CHAN_RUNNING; - /* Configure the physical channel for the active txd */ - pl08x_config_phychan_for_txd(plchan); - pl08x_set_cregs(pl08x, plchan->phychan); - pl08x_enable_phy_chan(pl08x, plchan->phychan); + pl08x_start_txd(plchan, next); } spin_unlock_irqrestore(&plchan->lock, flags); @@ -1330,30 +1206,17 @@ static void pl08x_issue_pending(struct dma_chan *chan) static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, struct pl08x_txd *txd) { - int num_llis; struct pl08x_driver_data *pl08x = plchan->host; - int ret; + unsigned long flags; + int num_llis, ret; num_llis = pl08x_fill_llis_for_desc(pl08x, txd); - - if (!num_llis) + if (!num_llis) { + kfree(txd); return -EINVAL; + } - spin_lock_irqsave(&plchan->lock, plchan->lockflags); - - /* - * If this device is not using a circular buffer then - * queue this new descriptor for transfer. - * The descriptor for a circular buffer continues - * to be used until the channel is freed. - */ - if (txd->cd->circular_buffer) - dev_err(&pl08x->adev->dev, - "%s attempting to queue a circular buffer\n", - __func__); - else - list_add_tail(&txd->node, - &plchan->desc_list); + spin_lock_irqsave(&plchan->lock, flags); /* * See if we already have a physical channel allocated, @@ -1362,45 +1225,74 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, ret = prep_phy_channel(plchan, txd); if (ret) { /* - * No physical channel available, we will - * stack up the memcpy channels until there is a channel - * available to handle it whereas slave transfers may - * have been denied due to platform channel muxing restrictions - * and since there is no guarantee that this will ever be - * resolved, and since the signal must be aquired AFTER - * aquiring the physical channel, we will let them be NACK:ed - * with -EBUSY here. The drivers can alway retry the prep() - * call if they are eager on doing this using DMA. + * No physical channel was available. + * + * memcpy transfers can be sorted out at submission time. + * + * Slave transfers may have been denied due to platform + * channel muxing restrictions. Since there is no guarantee + * that this will ever be resolved, and the signal must be + * acquired AFTER acquiring the physical channel, we will let + * them be NACK:ed with -EBUSY here. The drivers can retry + * the prep() call if they are eager on doing this using DMA. */ if (plchan->slave) { pl08x_free_txd_list(pl08x, plchan); - spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); + pl08x_free_txd(pl08x, txd); + spin_unlock_irqrestore(&plchan->lock, flags); return -EBUSY; } - /* Do this memcpy whenever there is a channel ready */ - plchan->state = PL08X_CHAN_WAITING; - plchan->waiting = txd; } else /* - * Else we're all set, paused and ready to roll, - * status will switch to PL08X_CHAN_RUNNING when - * we call issue_pending(). If there is something - * running on the channel already we don't change - * its state. + * Else we're all set, paused and ready to roll, status + * will switch to PL08X_CHAN_RUNNING when we call + * issue_pending(). If there is something running on the + * channel already we don't change its state. */ if (plchan->state == PL08X_CHAN_IDLE) plchan->state = PL08X_CHAN_PAUSED; - /* - * Notice that we leave plchan->lock locked on purpose: - * it will be unlocked in the subsequent tx_submit() - * call. This is a consequence of the current API. - */ + spin_unlock_irqrestore(&plchan->lock, flags); return 0; } /* + * Given the source and destination available bus masks, select which + * will be routed to each port. We try to have source and destination + * on separate ports, but always respect the allowable settings. + */ +static u32 pl08x_select_bus(struct pl08x_driver_data *pl08x, u8 src, u8 dst) +{ + u32 cctl = 0; + + if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) + cctl |= PL080_CONTROL_DST_AHB2; + if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) + cctl |= PL080_CONTROL_SRC_AHB2; + + return cctl; +} + +static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan, + unsigned long flags) +{ + struct pl08x_txd *txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + + if (txd) { + dma_async_tx_descriptor_init(&txd->tx, &plchan->chan); + txd->tx.flags = flags; + txd->tx.tx_submit = pl08x_tx_submit; + INIT_LIST_HEAD(&txd->node); + + /* Always enable error and terminal interrupts */ + txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | + PL080_CONFIG_TC_IRQ_MASK; + } + return txd; +} + +/* * Initialize a descriptor to be used by memcpy submit */ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( @@ -1412,40 +1304,38 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( struct pl08x_txd *txd; int ret; - txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + txd = pl08x_get_txd(plchan, flags); if (!txd) { dev_err(&pl08x->adev->dev, "%s no memory for descriptor\n", __func__); return NULL; } - dma_async_tx_descriptor_init(&txd->tx, chan); txd->direction = DMA_NONE; - txd->srcbus.addr = src; - txd->dstbus.addr = dest; + txd->src_addr = src; + txd->dst_addr = dest; + txd->len = len; /* Set platform data for m2m */ - txd->cd = &pl08x->pd->memcpy_channel; + txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl = pl08x->pd->memcpy_channel.cctl & + ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); + /* Both to be incremented or the code will break */ - txd->cd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; - txd->tx.tx_submit = pl08x_tx_submit; - txd->tx.callback = NULL; - txd->tx.callback_param = NULL; - txd->len = len; + txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; + + if (pl08x->vd->dualmaster) + txd->cctl |= pl08x_select_bus(pl08x, + pl08x->mem_buses, pl08x->mem_buses); - INIT_LIST_HEAD(&txd->node); ret = pl08x_prep_channel_resources(plchan, txd); if (ret) return NULL; - /* - * NB: the channel lock is held at this point so tx_submit() - * must be called in direct succession. - */ return &txd->tx; } -struct dma_async_tx_descriptor *pl08x_prep_slave_sg( +static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, unsigned long flags) @@ -1453,6 +1343,7 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_txd *txd; + u8 src_buses, dst_buses; int ret; /* @@ -1467,14 +1358,12 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", __func__, sgl->length, plchan->name); - txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + txd = pl08x_get_txd(plchan, flags); if (!txd) { dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); return NULL; } - dma_async_tx_descriptor_init(&txd->tx, chan); - if (direction != plchan->runtime_direction) dev_err(&pl08x->adev->dev, "%s DMA setup does not match " "the direction configured for the PrimeCell\n", @@ -1486,37 +1375,47 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg( * channel target address dynamically at runtime. */ txd->direction = direction; + txd->len = sgl->length; + + txd->cctl = plchan->cd->cctl & + ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | + PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | + PL080_CONTROL_PROT_MASK); + + /* Access the cell in privileged mode, non-bufferable, non-cacheable */ + txd->cctl |= PL080_CONTROL_PROT_SYS; + if (direction == DMA_TO_DEVICE) { - txd->srcbus.addr = sgl->dma_address; + txd->ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl |= PL080_CONTROL_SRC_INCR; + txd->src_addr = sgl->dma_address; if (plchan->runtime_addr) - txd->dstbus.addr = plchan->runtime_addr; + txd->dst_addr = plchan->runtime_addr; else - txd->dstbus.addr = plchan->cd->addr; + txd->dst_addr = plchan->cd->addr; + src_buses = pl08x->mem_buses; + dst_buses = plchan->cd->periph_buses; } else if (direction == DMA_FROM_DEVICE) { + txd->ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl |= PL080_CONTROL_DST_INCR; if (plchan->runtime_addr) - txd->srcbus.addr = plchan->runtime_addr; + txd->src_addr = plchan->runtime_addr; else - txd->srcbus.addr = plchan->cd->addr; - txd->dstbus.addr = sgl->dma_address; + txd->src_addr = plchan->cd->addr; + txd->dst_addr = sgl->dma_address; + src_buses = plchan->cd->periph_buses; + dst_buses = pl08x->mem_buses; } else { dev_err(&pl08x->adev->dev, "%s direction unsupported\n", __func__); return NULL; } - txd->cd = plchan->cd; - txd->tx.tx_submit = pl08x_tx_submit; - txd->tx.callback = NULL; - txd->tx.callback_param = NULL; - txd->len = sgl->length; - INIT_LIST_HEAD(&txd->node); + + txd->cctl |= pl08x_select_bus(pl08x, src_buses, dst_buses); ret = pl08x_prep_channel_resources(plchan, txd); if (ret) return NULL; - /* - * NB: the channel lock is held at this point so tx_submit() - * must be called in direct succession. - */ return &txd->tx; } @@ -1531,10 +1430,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, /* Controls applicable to inactive channels */ if (cmd == DMA_SLAVE_CONFIG) { - dma_set_runtime_config(chan, - (struct dma_slave_config *) - arg); - return 0; + return dma_set_runtime_config(chan, + (struct dma_slave_config *)arg); } /* @@ -1558,16 +1455,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, * Mark physical channel as free and free any slave * signal */ - if ((plchan->phychan->signal >= 0) && - pl08x->pd->put_signal) { - pl08x->pd->put_signal(plchan); - plchan->phychan->signal = -1; - } - pl08x_put_phy_channel(pl08x, plchan->phychan); - plchan->phychan = NULL; + release_phy_channel(plchan); } - /* Stop any pending tasklet */ - tasklet_disable(&plchan->tasklet); /* Dequeue jobs and free LLIs */ if (plchan->at) { pl08x_free_txd(pl08x, plchan->at); @@ -1609,10 +1498,9 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) /* * Just check that the device is there and active - * TODO: turn this bit on/off depending on the number of - * physical channels actually used, if it is zero... well - * shut it off. That will save some power. Cut the clock - * at the same time. + * TODO: turn this bit on/off depending on the number of physical channels + * actually used, if it is zero... well shut it off. That will save some + * power. Cut the clock at the same time. */ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) { @@ -1620,78 +1508,66 @@ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) val = readl(pl08x->base + PL080_CONFIG); val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE); - /* We implictly clear bit 1 and that means little-endian mode */ + /* We implicitly clear bit 1 and that means little-endian mode */ val |= PL080_CONFIG_ENABLE; writel(val, pl08x->base + PL080_CONFIG); } +static void pl08x_unmap_buffers(struct pl08x_txd *txd) +{ + struct device *dev = txd->tx.chan->device->dev; + + if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE) + dma_unmap_single(dev, txd->src_addr, txd->len, + DMA_TO_DEVICE); + else + dma_unmap_page(dev, txd->src_addr, txd->len, + DMA_TO_DEVICE); + } + if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE) + dma_unmap_single(dev, txd->dst_addr, txd->len, + DMA_FROM_DEVICE); + else + dma_unmap_page(dev, txd->dst_addr, txd->len, + DMA_FROM_DEVICE); + } +} + static void pl08x_tasklet(unsigned long data) { struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data; - struct pl08x_phy_chan *phychan = plchan->phychan; struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + unsigned long flags; - if (!plchan) - BUG(); - - spin_lock(&plchan->lock); - - if (plchan->at) { - dma_async_tx_callback callback = - plchan->at->tx.callback; - void *callback_param = - plchan->at->tx.callback_param; - - /* - * Update last completed - */ - plchan->lc = - (plchan->at->tx.cookie); - - /* - * Callback to signal completion - */ - if (callback) - callback(callback_param); + spin_lock_irqsave(&plchan->lock, flags); - /* - * Device callbacks should NOT clear - * the current transaction on the channel - * Linus: sometimes they should? - */ - if (!plchan->at) - BUG(); + txd = plchan->at; + plchan->at = NULL; - /* - * Free the descriptor if it's not for a device - * using a circular buffer - */ - if (!plchan->at->cd->circular_buffer) { - pl08x_free_txd(pl08x, plchan->at); - plchan->at = NULL; - } - /* - * else descriptor for circular - * buffers only freed when - * client has disabled dma - */ + if (txd) { + /* Update last completed */ + plchan->lc = txd->tx.cookie; } - /* - * If a new descriptor is queued, set it up - * plchan->at is NULL here - */ - if (!list_empty(&plchan->desc_list)) { + + /* If a new descriptor is queued, set it up plchan->at is NULL here */ + if (!list_empty(&plchan->pend_list)) { struct pl08x_txd *next; - next = list_first_entry(&plchan->desc_list, + next = list_first_entry(&plchan->pend_list, struct pl08x_txd, node); list_del(&next->node); - plchan->at = next; - /* Configure the physical channel for the next txd */ - pl08x_config_phychan_for_txd(plchan); - pl08x_set_cregs(pl08x, plchan->phychan); - pl08x_enable_phy_chan(pl08x, plchan->phychan); + + pl08x_start_txd(plchan, next); + } else if (plchan->phychan_hold) { + /* + * This channel is still in use - we have a new txd being + * prepared and will soon be queued. Don't give up the + * physical channel. + */ } else { struct pl08x_dma_chan *waiting = NULL; @@ -1699,20 +1575,14 @@ static void pl08x_tasklet(unsigned long data) * No more jobs, so free up the physical channel * Free any allocated signal on slave transfers too */ - if ((phychan->signal >= 0) && pl08x->pd->put_signal) { - pl08x->pd->put_signal(plchan); - phychan->signal = -1; - } - pl08x_put_phy_channel(pl08x, phychan); - plchan->phychan = NULL; + release_phy_channel(plchan); plchan->state = PL08X_CHAN_IDLE; /* - * And NOW before anyone else can grab that free:d - * up physical channel, see if there is some memcpy - * pending that seriously needs to start because of - * being stacked up while we were choking the - * physical channels with data. + * And NOW before anyone else can grab that free:d up + * physical channel, see if there is some memcpy pending + * that seriously needs to start because of being stacked + * up while we were choking the physical channels with data. */ list_for_each_entry(waiting, &pl08x->memcpy.channels, chan.device_node) { @@ -1724,6 +1594,7 @@ static void pl08x_tasklet(unsigned long data) ret = prep_phy_channel(waiting, waiting->waiting); BUG_ON(ret); + waiting->phychan_hold--; waiting->state = PL08X_CHAN_RUNNING; waiting->waiting = NULL; pl08x_issue_pending(&waiting->chan); @@ -1732,7 +1603,25 @@ static void pl08x_tasklet(unsigned long data) } } - spin_unlock(&plchan->lock); + spin_unlock_irqrestore(&plchan->lock, flags); + + if (txd) { + dma_async_tx_callback callback = txd->tx.callback; + void *callback_param = txd->tx.callback_param; + + /* Don't try to unmap buffers on slave channels */ + if (!plchan->slave) + pl08x_unmap_buffers(txd); + + /* Free the descriptor */ + spin_lock_irqsave(&plchan->lock, flags); + pl08x_free_txd(pl08x, txd); + spin_unlock_irqrestore(&plchan->lock, flags); + + /* Callback to signal completion */ + if (callback) + callback(callback_param); + } } static irqreturn_t pl08x_irq(int irq, void *dev) @@ -1744,9 +1633,7 @@ static irqreturn_t pl08x_irq(int irq, void *dev) val = readl(pl08x->base + PL080_ERR_STATUS); if (val) { - /* - * An error interrupt (on one or more channels) - */ + /* An error interrupt (on one or more channels) */ dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", __func__, val); @@ -1770,9 +1657,7 @@ static irqreturn_t pl08x_irq(int irq, void *dev) mask |= (1 << i); } } - /* - * Clear only the terminal interrupts on channels we processed - */ + /* Clear only the terminal interrupts on channels we processed */ writel(mask, pl08x->base + PL080_TC_CLEAR); return mask ? IRQ_HANDLED : IRQ_NONE; @@ -1791,6 +1676,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, int i; INIT_LIST_HEAD(&dmadev->channels); + /* * Register as many many memcpy as we have physical channels, * we won't always be able to use all but the code will have @@ -1819,16 +1705,23 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, return -ENOMEM; } } + if (chan->cd->circular_buffer) { + dev_err(&pl08x->adev->dev, + "channel %s: circular buffers not supported\n", + chan->name); + kfree(chan); + continue; + } dev_info(&pl08x->adev->dev, "initialize virtual channel \"%s\"\n", chan->name); chan->chan.device = dmadev; - atomic_set(&chan->last_issued, 0); - chan->lc = atomic_read(&chan->last_issued); + chan->chan.cookie = 0; + chan->lc = 0; spin_lock_init(&chan->lock); - INIT_LIST_HEAD(&chan->desc_list); + INIT_LIST_HEAD(&chan->pend_list); tasklet_init(&chan->tasklet, pl08x_tasklet, (unsigned long) chan); @@ -1898,7 +1791,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data) seq_printf(s, "CHANNEL:\tSTATE:\n"); seq_printf(s, "--------\t------\n"); list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) { - seq_printf(s, "%s\t\t\%s\n", chan->name, + seq_printf(s, "%s\t\t%s\n", chan->name, pl08x_state_str(chan->state)); } @@ -1906,7 +1799,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data) seq_printf(s, "CHANNEL:\tSTATE:\n"); seq_printf(s, "--------\t------\n"); list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) { - seq_printf(s, "%s\t\t\%s\n", chan->name, + seq_printf(s, "%s\t\t%s\n", chan->name, pl08x_state_str(chan->state)); } @@ -1942,7 +1835,7 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) static int pl08x_probe(struct amba_device *adev, struct amba_id *id) { struct pl08x_driver_data *pl08x; - struct vendor_data *vd = id->data; + const struct vendor_data *vd = id->data; int ret = 0; int i; @@ -1990,6 +1883,14 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) pl08x->adev = adev; pl08x->vd = vd; + /* By default, AHB1 only. If dualmaster, from platform */ + pl08x->lli_buses = PL08X_AHB1; + pl08x->mem_buses = PL08X_AHB1; + if (pl08x->vd->dualmaster) { + pl08x->lli_buses = pl08x->pd->lli_buses; + pl08x->mem_buses = pl08x->pd->mem_buses; + } + /* A DMA memory pool for LLIs, align on 1-byte boundary */ pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0); @@ -2009,14 +1910,12 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) /* Turn on the PL08x */ pl08x_ensure_on(pl08x); - /* - * Attach the interrupt handler - */ + /* Attach the interrupt handler */ writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED, - vd->name, pl08x); + DRIVER_NAME, pl08x); if (ret) { dev_err(&adev->dev, "%s failed to request interrupt %d\n", __func__, adev->irq[0]); @@ -2087,8 +1986,9 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id) amba_set_drvdata(adev, pl08x); init_pl08x_debugfs(pl08x); - dev_info(&pl08x->adev->dev, "ARM(R) %s DMA block initialized @%08x\n", - vd->name, adev->res.start); + dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n", + amba_part(adev), amba_rev(adev), + (unsigned long long)adev->res.start, adev->irq[0]); return 0; out_no_slave_reg: @@ -2115,13 +2015,11 @@ out_no_pl08x: /* PL080 has 8 channels and the PL080 have just 2 */ static struct vendor_data vendor_pl080 = { - .name = "PL080", .channels = 8, .dualmaster = true, }; static struct vendor_data vendor_pl081 = { - .name = "PL081", .channels = 2, .dualmaster = false, }; @@ -2160,7 +2058,7 @@ static int __init pl08x_init(void) retval = amba_driver_register(&pl08x_amba_driver); if (retval) printk(KERN_WARNING DRIVER_NAME - "failed to register as an amba device (%d)\n", + "failed to register as an AMBA device (%d)\n", retval); return retval; } diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index ea0ee81cff53..3d7d705f026f 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -253,7 +253,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); - /* unmap dma addresses */ + /* unmap dma addresses (not on slave channels) */ if (!atchan->chan_common.private) { struct device *parent = chan2parent(&atchan->chan_common); if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { @@ -583,7 +583,6 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, desc->lli.ctrlb = ctrlb; desc->txd.cookie = 0; - async_tx_ack(&desc->txd); if (!first) { first = desc; @@ -604,7 +603,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, /* set end-of-link to the last link descriptor of list*/ set_desc_eol(desc); - desc->txd.flags = flags; /* client is in control of this ack */ + first->txd.flags = flags; /* client is in control of this ack */ return &first->txd; @@ -670,7 +669,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!desc) goto err_desc_get; - mem = sg_phys(sg); + mem = sg_dma_address(sg); len = sg_dma_len(sg); mem_width = 2; if (unlikely(mem & 3 || len & 3)) @@ -712,7 +711,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!desc) goto err_desc_get; - mem = sg_phys(sg); + mem = sg_dma_address(sg); len = sg_dma_len(sg); mem_width = 2; if (unlikely(mem & 3 || len & 3)) @@ -749,8 +748,8 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, first->txd.cookie = -EBUSY; first->len = total_len; - /* last link descriptor of list is responsible of flags */ - prev->txd.flags = flags; /* client is in control of this ack */ + /* first link descriptor of list is responsible of flags */ + first->txd.flags = flags; /* client is in control of this ack */ return &first->txd; @@ -854,11 +853,11 @@ static void atc_issue_pending(struct dma_chan *chan) dev_vdbg(chan2dev(chan), "issue_pending\n"); + spin_lock_bh(&atchan->lock); if (!atc_chan_is_enabled(atchan)) { - spin_lock_bh(&atchan->lock); atc_advance_work(atchan); - spin_unlock_bh(&atchan->lock); } + spin_unlock_bh(&atchan->lock); } /** @@ -1210,7 +1209,7 @@ static int __init at_dma_init(void) { return platform_driver_probe(&at_dma_driver, at_dma_probe); } -module_init(at_dma_init); +subsys_initcall(at_dma_init); static void __exit at_dma_exit(void) { diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index e5e172d21692..4de947a450fc 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1,7 +1,7 @@ /* * Freescale MPC85xx, MPC83xx DMA Engine support * - * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved. * * Author: * Zhang Wei <wei.zhang@freescale.com>, Jul 2007 @@ -1324,6 +1324,8 @@ static int __devinit fsldma_of_probe(struct platform_device *op, fdev->common.device_control = fsl_dma_device_control; fdev->common.dev = &op->dev; + dma_set_mask(&(op->dev), DMA_BIT_MASK(36)); + dev_set_drvdata(&op->dev, fdev); /* diff --git a/drivers/dma/intel_mid_dma.c b/drivers/dma/intel_mid_dma.c index 78266382797e..798f46a4590d 100644 --- a/drivers/dma/intel_mid_dma.c +++ b/drivers/dma/intel_mid_dma.c @@ -664,11 +664,20 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_memcpy( /*calculate CTL_LO*/ ctl_lo.ctl_lo = 0; ctl_lo.ctlx.int_en = 1; - ctl_lo.ctlx.dst_tr_width = mids->dma_slave.dst_addr_width; - ctl_lo.ctlx.src_tr_width = mids->dma_slave.src_addr_width; ctl_lo.ctlx.dst_msize = mids->dma_slave.src_maxburst; ctl_lo.ctlx.src_msize = mids->dma_slave.dst_maxburst; + /* + * Here we need some translation from "enum dma_slave_buswidth" + * to the format for our dma controller + * standard intel_mid_dmac's format + * 1 Byte 0b000 + * 2 Bytes 0b001 + * 4 Bytes 0b010 + */ + ctl_lo.ctlx.dst_tr_width = mids->dma_slave.dst_addr_width / 2; + ctl_lo.ctlx.src_tr_width = mids->dma_slave.src_addr_width / 2; + if (mids->cfg_mode == LNW_DMA_MEM_TO_MEM) { ctl_lo.ctlx.tt_fc = 0; ctl_lo.ctlx.sinc = 0; @@ -746,8 +755,18 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_slave_sg( BUG_ON(!mids); if (!midc->dma->pimr_mask) { - pr_debug("MDMA: SG list is not supported by this controller\n"); - return NULL; + /* We can still handle sg list with only one item */ + if (sg_len == 1) { + txd = intel_mid_dma_prep_memcpy(chan, + mids->dma_slave.dst_addr, + mids->dma_slave.src_addr, + sgl->length, + flags); + return txd; + } else { + pr_warn("MDMA: SG list is not supported by this controller\n"); + return NULL; + } } pr_debug("MDMA: SG Length = %d, direction = %d, Flags = %#lx\n", @@ -758,6 +777,7 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_slave_sg( pr_err("MDMA: Prep memcpy failed\n"); return NULL; } + desc = to_intel_mid_dma_desc(txd); desc->dirn = direction; ctl_lo.ctl_lo = desc->ctl_lo; @@ -1021,11 +1041,6 @@ static irqreturn_t intel_mid_dma_interrupt(int irq, void *data) /*DMA Interrupt*/ pr_debug("MDMA:Got an interrupt on irq %d\n", irq); - if (!mid) { - pr_err("ERR_MDMA:null pointer mid\n"); - return -EINVAL; - } - pr_debug("MDMA: Status %x, Mask %x\n", tfr_status, mid->intr_mask); tfr_status &= mid->intr_mask; if (tfr_status) { diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c index 161c452923b8..c6b01f535b29 100644 --- a/drivers/dma/iop-adma.c +++ b/drivers/dma/iop-adma.c @@ -1261,7 +1261,7 @@ out: return err; } -#ifdef CONFIG_MD_RAID6_PQ +#ifdef CONFIG_RAID6_PQ static int __devinit iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) { @@ -1584,7 +1584,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev) if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { - #ifdef CONFIG_MD_RAID6_PQ + #ifdef CONFIG_RAID6_PQ ret = iop_adma_pq_zero_sum_self_test(adev); dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); #else diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index c064c89420d0..1c38418ae61f 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -1,6 +1,7 @@ /* * Topcliff PCH DMA controller driver * Copyright (c) 2010 Intel Corporation + * Copyright (C) 2011 OKI SEMICONDUCTOR CO., LTD. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -921,12 +922,19 @@ static void __devexit pch_dma_remove(struct pci_dev *pdev) } /* PCI Device ID of DMA device */ -#define PCI_DEVICE_ID_PCH_DMA_8CH 0x8810 -#define PCI_DEVICE_ID_PCH_DMA_4CH 0x8815 +#define PCI_VENDOR_ID_ROHM 0x10DB +#define PCI_DEVICE_ID_EG20T_PCH_DMA_8CH 0x8810 +#define PCI_DEVICE_ID_EG20T_PCH_DMA_4CH 0x8815 +#define PCI_DEVICE_ID_ML7213_DMA1_8CH 0x8026 +#define PCI_DEVICE_ID_ML7213_DMA2_8CH 0x802B +#define PCI_DEVICE_ID_ML7213_DMA3_4CH 0x8034 static const struct pci_device_id pch_dma_id_table[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_8CH), 8 }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_4CH), 4 }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_8CH), 8 }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_4CH), 4 }, + { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA1_8CH), 8}, /* UART Video */ + { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA2_8CH), 8}, /* PCMIF SPI */ + { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA3_4CH), 4}, /* FPGA */ { 0, }, }; @@ -954,6 +962,7 @@ static void __exit pch_dma_exit(void) module_init(pch_dma_init); module_exit(pch_dma_exit); -MODULE_DESCRIPTION("Topcliff PCH DMA controller driver"); +MODULE_DESCRIPTION("Intel EG20T PCH / OKI SEMICONDUCTOR ML7213 IOH " + "DMA controller driver"); MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index fab68a553205..6e1d46a65d0e 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -1,5 +1,6 @@ /* - * Copyright (C) ST-Ericsson SA 2007-2010 + * Copyright (C) Ericsson AB 2007-2008 + * Copyright (C) ST-Ericsson SA 2008-2010 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson * License terms: GNU General Public License (GPL) version 2 @@ -554,8 +555,66 @@ static struct d40_desc *d40_last_queued(struct d40_chan *d40c) return d; } -/* Support functions for logical channels */ +static int d40_psize_2_burst_size(bool is_log, int psize) +{ + if (is_log) { + if (psize == STEDMA40_PSIZE_LOG_1) + return 1; + } else { + if (psize == STEDMA40_PSIZE_PHY_1) + return 1; + } + + return 2 << psize; +} + +/* + * The dma only supports transmitting packages up to + * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of + * dma elements required to send the entire sg list + */ +static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2) +{ + int dmalen; + u32 max_w = max(data_width1, data_width2); + u32 min_w = min(data_width1, data_width2); + u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); + + if (seg_max > STEDMA40_MAX_SEG_SIZE) + seg_max -= (1 << max_w); + + if (!IS_ALIGNED(size, 1 << max_w)) + return -EINVAL; + + if (size <= seg_max) + dmalen = 1; + else { + dmalen = size / seg_max; + if (dmalen * seg_max < size) + dmalen++; + } + return dmalen; +} + +static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, + u32 data_width1, u32 data_width2) +{ + struct scatterlist *sg; + int i; + int len = 0; + int ret; + + for_each_sg(sgl, sg, sg_len, i) { + ret = d40_size_2_dmalen(sg_dma_len(sg), + data_width1, data_width2); + if (ret < 0) + return ret; + len += ret; + } + return len; +} +/* Support functions for logical channels */ static int d40_channel_execute_command(struct d40_chan *d40c, enum d40_command command) @@ -1241,6 +1300,21 @@ static int d40_validate_conf(struct d40_chan *d40c, res = -EINVAL; } + if (d40_psize_2_burst_size(is_log, conf->src_info.psize) * + (1 << conf->src_info.data_width) != + d40_psize_2_burst_size(is_log, conf->dst_info.psize) * + (1 << conf->dst_info.data_width)) { + /* + * The DMAC hardware only supports + * src (burst x width) == dst (burst x width) + */ + + dev_err(&d40c->chan.dev->device, + "[%s] src (burst x width) != dst (burst x width)\n", + __func__); + res = -EINVAL; + } + return res; } @@ -1638,13 +1712,21 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, if (d40d == NULL) goto err; - d40d->lli_len = sgl_len; + d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + goto err; + } + d40d->lli_current = 0; d40d->txd.flags = dma_flags; if (d40c->log_num != D40_PHY_CHAN) { - if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; @@ -1654,15 +1736,17 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, sgl_len, d40d->lli_log.src, d40c->log_def.lcsp1, - d40c->dma_cfg.src_info.data_width); + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); (void) d40_log_sg_to_lli(sgl_dst, sgl_len, d40d->lli_log.dst, d40c->log_def.lcsp3, - d40c->dma_cfg.dst_info.data_width); + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width); } else { - if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; @@ -1675,6 +1759,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, virt_to_phys(d40d->lli_phy.src), d40c->src_def_cfg, d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, d40c->dma_cfg.src_info.psize); if (res < 0) @@ -1687,6 +1772,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, virt_to_phys(d40d->lli_phy.dst), d40c->dst_def_cfg, d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, d40c->dma_cfg.dst_info.psize); if (res < 0) @@ -1826,7 +1912,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); unsigned long flags; - int err = 0; if (d40c->phy_chan == NULL) { dev_err(&d40c->chan.dev->device, @@ -1844,6 +1929,15 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, } d40d->txd.flags = dma_flags; + d40d->lli_len = d40_size_2_dmalen(size, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + goto err; + } + dma_async_tx_descriptor_init(&d40d->txd, chan); @@ -1851,37 +1945,40 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, if (d40c->log_num != D40_PHY_CHAN) { - if (d40_pool_lli_alloc(d40d, 1, true) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; } - d40d->lli_len = 1; d40d->lli_current = 0; - d40_log_fill_lli(d40d->lli_log.src, - src, - size, - d40c->log_def.lcsp1, - d40c->dma_cfg.src_info.data_width, - true); + if (d40_log_buf_to_lli(d40d->lli_log.src, + src, + size, + d40c->log_def.lcsp1, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, + true) == NULL) + goto err; - d40_log_fill_lli(d40d->lli_log.dst, - dst, - size, - d40c->log_def.lcsp3, - d40c->dma_cfg.dst_info.data_width, - true); + if (d40_log_buf_to_lli(d40d->lli_log.dst, + dst, + size, + d40c->log_def.lcsp3, + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, + true) == NULL) + goto err; } else { - if (d40_pool_lli_alloc(d40d, 1, false) < 0) { + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); goto err; } - err = d40_phy_fill_lli(d40d->lli_phy.src, + if (d40_phy_buf_to_lli(d40d->lli_phy.src, src, size, d40c->dma_cfg.src_info.psize, @@ -1889,11 +1986,11 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, d40c->src_def_cfg, true, d40c->dma_cfg.src_info.data_width, - false); - if (err) - goto err_fill_lli; + d40c->dma_cfg.dst_info.data_width, + false) == NULL) + goto err; - err = d40_phy_fill_lli(d40d->lli_phy.dst, + if (d40_phy_buf_to_lli(d40d->lli_phy.dst, dst, size, d40c->dma_cfg.dst_info.psize, @@ -1901,10 +1998,9 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, d40c->dst_def_cfg, true, d40c->dma_cfg.dst_info.data_width, - false); - - if (err) - goto err_fill_lli; + d40c->dma_cfg.src_info.data_width, + false) == NULL) + goto err; (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, d40d->lli_pool.size, DMA_TO_DEVICE); @@ -1913,9 +2009,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, spin_unlock_irqrestore(&d40c->lock, flags); return &d40d->txd; -err_fill_lli: - dev_err(&d40c->chan.dev->device, - "[%s] Failed filling in PHY LLI\n", __func__); err: if (d40d) d40_desc_free(d40c, d40d); @@ -1945,13 +2038,21 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d, dma_addr_t dev_addr = 0; int total_size; - if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) { + d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + return -EINVAL; + } + + if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); return -ENOMEM; } - d40d->lli_len = sg_len; d40d->lli_current = 0; if (direction == DMA_FROM_DEVICE) @@ -1993,13 +2094,21 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, dma_addr_t dst_dev_addr; int res; - if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width); + if (d40d->lli_len < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Unaligned size\n", __func__); + return -EINVAL; + } + + if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) { dev_err(&d40c->chan.dev->device, "[%s] Out of memory\n", __func__); return -ENOMEM; } - d40d->lli_len = sgl_len; d40d->lli_current = 0; if (direction == DMA_FROM_DEVICE) { @@ -2024,6 +2133,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, virt_to_phys(d40d->lli_phy.src), d40c->src_def_cfg, d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, d40c->dma_cfg.src_info.psize); if (res < 0) return res; @@ -2035,6 +2145,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d, virt_to_phys(d40d->lli_phy.dst), d40c->dst_def_cfg, d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.src_info.data_width, d40c->dma_cfg.dst_info.psize); if (res < 0) return res; @@ -2244,6 +2355,8 @@ static void d40_set_runtime_config(struct dma_chan *chan, psize = STEDMA40_PSIZE_PHY_8; else if (config_maxburst >= 4) psize = STEDMA40_PSIZE_PHY_4; + else if (config_maxburst >= 2) + psize = STEDMA40_PSIZE_PHY_2; else psize = STEDMA40_PSIZE_PHY_1; } diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c index 8557cb88b255..0b096a38322d 100644 --- a/drivers/dma/ste_dma40_ll.c +++ b/drivers/dma/ste_dma40_ll.c @@ -1,6 +1,6 @@ /* * Copyright (C) ST-Ericsson SA 2007-2010 - * Author: Per Friden <per.friden@stericsson.com> for ST-Ericsson + * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson * License terms: GNU General Public License (GPL) version 2 */ @@ -122,15 +122,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg, *dst_cfg = dst; } -int d40_phy_fill_lli(struct d40_phy_lli *lli, - dma_addr_t data, - u32 data_size, - int psize, - dma_addr_t next_lli, - u32 reg_cfg, - bool term_int, - u32 data_width, - bool is_device) +static int d40_phy_fill_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width, + bool is_device) { int num_elems; @@ -139,13 +139,6 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli, else num_elems = 2 << psize; - /* - * Size is 16bit. data_width is 8, 16, 32 or 64 bit - * Block large than 64 KiB must be split. - */ - if (data_size > (0xffff << data_width)) - return -EINVAL; - /* Must be aligned */ if (!IS_ALIGNED(data, 0x1 << data_width)) return -EINVAL; @@ -187,55 +180,118 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli, return 0; } +static int d40_seg_size(int size, int data_width1, int data_width2) +{ + u32 max_w = max(data_width1, data_width2); + u32 min_w = min(data_width1, data_width2); + u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); + + if (seg_max > STEDMA40_MAX_SEG_SIZE) + seg_max -= (1 << max_w); + + if (size <= seg_max) + return size; + + if (size <= 2 * seg_max) + return ALIGN(size / 2, 1 << max_w); + + return seg_max; +} + +struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli, + dma_addr_t addr, + u32 size, + int psize, + dma_addr_t lli_phys, + u32 reg_cfg, + bool term_int, + u32 data_width1, + u32 data_width2, + bool is_device) +{ + int err; + dma_addr_t next = lli_phys; + int size_rest = size; + int size_seg = 0; + + do { + size_seg = d40_seg_size(size_rest, data_width1, data_width2); + size_rest -= size_seg; + + if (term_int && size_rest == 0) + next = 0; + else + next = ALIGN(next + sizeof(struct d40_phy_lli), + D40_LLI_ALIGN); + + err = d40_phy_fill_lli(lli, + addr, + size_seg, + psize, + next, + reg_cfg, + !next, + data_width1, + is_device); + + if (err) + goto err; + + lli++; + if (!is_device) + addr += size_seg; + } while (size_rest); + + return lli; + + err: + return NULL; +} + int d40_phy_sg_to_lli(struct scatterlist *sg, int sg_len, dma_addr_t target, - struct d40_phy_lli *lli, + struct d40_phy_lli *lli_sg, dma_addr_t lli_phys, u32 reg_cfg, - u32 data_width, + u32 data_width1, + u32 data_width2, int psize) { int total_size = 0; int i; struct scatterlist *current_sg = sg; - dma_addr_t next_lli_phys; dma_addr_t dst; - int err = 0; + struct d40_phy_lli *lli = lli_sg; + dma_addr_t l_phys = lli_phys; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); - /* If this scatter list entry is the last one, no next link */ - if (sg_len - 1 == i) - next_lli_phys = 0; - else - next_lli_phys = ALIGN(lli_phys + (i + 1) * - sizeof(struct d40_phy_lli), - D40_LLI_ALIGN); - if (target) dst = target; else dst = sg_phys(current_sg); - err = d40_phy_fill_lli(&lli[i], - dst, - sg_dma_len(current_sg), - psize, - next_lli_phys, - reg_cfg, - !next_lli_phys, - data_width, - target == dst); - if (err) - goto err; + l_phys = ALIGN(lli_phys + (lli - lli_sg) * + sizeof(struct d40_phy_lli), D40_LLI_ALIGN); + + lli = d40_phy_buf_to_lli(lli, + dst, + sg_dma_len(current_sg), + psize, + l_phys, + reg_cfg, + sg_len - 1 == i, + data_width1, + data_width2, + target == dst); + if (lli == NULL) + return -EINVAL; } return total_size; -err: - return err; } @@ -315,17 +371,20 @@ void d40_log_lli_lcla_write(struct d40_log_lli *lcla, writel(lli_dst->lcsp13, &lcla[1].lcsp13); } -void d40_log_fill_lli(struct d40_log_lli *lli, - dma_addr_t data, u32 data_size, - u32 reg_cfg, - u32 data_width, - bool addr_inc) +static void d40_log_fill_lli(struct d40_log_lli *lli, + dma_addr_t data, u32 data_size, + u32 reg_cfg, + u32 data_width, + bool addr_inc) { lli->lcsp13 = reg_cfg; /* The number of elements to transfer */ lli->lcsp02 = ((data_size >> data_width) << D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK; + + BUG_ON((data_size >> data_width) > STEDMA40_MAX_SEG_SIZE); + /* 16 LSBs address of the current element */ lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK; /* 16 MSBs address of the current element */ @@ -348,55 +407,94 @@ int d40_log_sg_to_dev(struct scatterlist *sg, int total_size = 0; struct scatterlist *current_sg = sg; int i; + struct d40_log_lli *lli_src = lli->src; + struct d40_log_lli *lli_dst = lli->dst; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); if (direction == DMA_TO_DEVICE) { - d40_log_fill_lli(&lli->src[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp->lcsp1, src_data_width, - true); - d40_log_fill_lli(&lli->dst[i], - dev_addr, - sg_dma_len(current_sg), - lcsp->lcsp3, dst_data_width, - false); + lli_src = + d40_log_buf_to_lli(lli_src, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp->lcsp1, src_data_width, + dst_data_width, + true); + lli_dst = + d40_log_buf_to_lli(lli_dst, + dev_addr, + sg_dma_len(current_sg), + lcsp->lcsp3, dst_data_width, + src_data_width, + false); } else { - d40_log_fill_lli(&lli->dst[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp->lcsp3, dst_data_width, - true); - d40_log_fill_lli(&lli->src[i], - dev_addr, - sg_dma_len(current_sg), - lcsp->lcsp1, src_data_width, - false); + lli_dst = + d40_log_buf_to_lli(lli_dst, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp->lcsp3, dst_data_width, + src_data_width, + true); + lli_src = + d40_log_buf_to_lli(lli_src, + dev_addr, + sg_dma_len(current_sg), + lcsp->lcsp1, src_data_width, + dst_data_width, + false); } } return total_size; } +struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg, + dma_addr_t addr, + int size, + u32 lcsp13, /* src or dst*/ + u32 data_width1, + u32 data_width2, + bool addr_inc) +{ + struct d40_log_lli *lli = lli_sg; + int size_rest = size; + int size_seg = 0; + + do { + size_seg = d40_seg_size(size_rest, data_width1, data_width2); + size_rest -= size_seg; + + d40_log_fill_lli(lli, + addr, + size_seg, + lcsp13, data_width1, + addr_inc); + if (addr_inc) + addr += size_seg; + lli++; + } while (size_rest); + + return lli; +} + int d40_log_sg_to_lli(struct scatterlist *sg, int sg_len, struct d40_log_lli *lli_sg, u32 lcsp13, /* src or dst*/ - u32 data_width) + u32 data_width1, u32 data_width2) { int total_size = 0; struct scatterlist *current_sg = sg; int i; + struct d40_log_lli *lli = lli_sg; for_each_sg(sg, current_sg, sg_len, i) { total_size += sg_dma_len(current_sg); - - d40_log_fill_lli(&lli_sg[i], - sg_phys(current_sg), - sg_dma_len(current_sg), - lcsp13, data_width, - true); + lli = d40_log_buf_to_lli(lli, + sg_phys(current_sg), + sg_dma_len(current_sg), + lcsp13, + data_width1, data_width2, true); } return total_size; } diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h index 9e419b907544..9cc43495bea2 100644 --- a/drivers/dma/ste_dma40_ll.h +++ b/drivers/dma/ste_dma40_ll.h @@ -292,18 +292,20 @@ int d40_phy_sg_to_lli(struct scatterlist *sg, struct d40_phy_lli *lli, dma_addr_t lli_phys, u32 reg_cfg, - u32 data_width, + u32 data_width1, + u32 data_width2, int psize); -int d40_phy_fill_lli(struct d40_phy_lli *lli, - dma_addr_t data, - u32 data_size, - int psize, - dma_addr_t next_lli, - u32 reg_cfg, - bool term_int, - u32 data_width, - bool is_device); +struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width1, + u32 data_width2, + bool is_device); void d40_phy_lli_write(void __iomem *virtbase, u32 phy_chan_num, @@ -312,12 +314,12 @@ void d40_phy_lli_write(void __iomem *virtbase, /* Logical channels */ -void d40_log_fill_lli(struct d40_log_lli *lli, - dma_addr_t data, - u32 data_size, - u32 reg_cfg, - u32 data_width, - bool addr_inc); +struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg, + dma_addr_t addr, + int size, + u32 lcsp13, /* src or dst*/ + u32 data_width1, u32 data_width2, + bool addr_inc); int d40_log_sg_to_dev(struct scatterlist *sg, int sg_len, @@ -332,7 +334,7 @@ int d40_log_sg_to_lli(struct scatterlist *sg, int sg_len, struct d40_log_lli *lli_sg, u32 lcsp13, /* src or dst*/ - u32 data_width); + u32 data_width1, u32 data_width2); void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, struct d40_log_lli *lli_dst, |