diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 36490a3..b05a81f 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -472,6 +472,111 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan) } /** + * fsldma_clean_completed_descriptor - free all descriptors which + * has been completed and acked + * @chan: Freescale DMA channel + * + * This function is used on all completed and acked descriptors. + * All descriptors should only be freed in this function. + */ +static void +fsldma_clean_completed_descriptor(struct fsldma_chan *chan) +{ + struct fsl_desc_sw *desc, *_desc; + + /* Run the callback for each descriptor, in order */ + list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) + if (async_tx_test_ack(&desc->async_tx)) + fsl_dma_free_descriptor(chan, desc); +} + +/** + * fsldma_run_tx_complete_actions - cleanup a single link descriptor + * @chan: Freescale DMA channel + * @desc: descriptor to cleanup and free + * @cookie: Freescale DMA transaction identifier + * + * This function is used on a descriptor which has been executed by the DMA + * controller. It will run any callbacks, submit any dependencies. + */ +static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan, + struct fsl_desc_sw *desc, dma_cookie_t cookie) +{ + struct dma_async_tx_descriptor *txd = &desc->async_tx; + struct device *dev = chan->common.device->dev; + dma_addr_t src = get_desc_src(chan, desc); + dma_addr_t dst = get_desc_dst(chan, desc); + u32 len = get_desc_cnt(chan, desc); + + BUG_ON(txd->cookie < 0); + + if (txd->cookie > 0) { + cookie = txd->cookie; + + /* Run the link descriptor callback function */ + if (txd->callback) { +#ifdef FSL_DMA_LD_DEBUG + chan_dbg(chan, "LD %p callback\n", desc); +#endif + txd->callback(txd->callback_param); + } + + /* Unmap the dst buffer, if requested */ + if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) + dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE); + else + dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE); + } + + /* Unmap the src buffer, if requested */ + if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) + dma_unmap_single(dev, src, len, DMA_TO_DEVICE); + else + dma_unmap_page(dev, src, len, DMA_TO_DEVICE); + } + } + + /* Run any dependencies */ + dma_run_dependencies(txd); + + return cookie; +} + +/** + * fsldma_clean_running_descriptor - move the completed descriptor from + * ld_running to ld_completed + * @chan: Freescale DMA channel + * @desc: the descriptor which is completed + * + * Free the descriptor directly if acked by async_tx api, or move it to + * queue ld_completed. + */ +static void +fsldma_clean_running_descriptor(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + /* Remove from the list of transactions */ + list_del(&desc->node); + + /* + * the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->async_tx)) { + /* + * Move this descriptor to the list of descriptors which is + * completed, but still awaiting the 'ack' bit to be set. + */ + list_add_tail(&desc->node, &chan->ld_completed); + return; + } + + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); +} + +/** * fsl_chan_xfer_ld_queue - transfer any pending transactions * @chan : Freescale DMA channel * @@ -539,51 +644,58 @@ static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) } /** - * fsldma_cleanup_descriptor - cleanup and free a single link descriptor + * fsldma_cleanup_descriptors - cleanup link descriptors which are completed + * and move them to ld_completed to free until flag 'ack' is set * @chan: Freescale DMA channel - * @desc: descriptor to cleanup and free * - * This function is used on a descriptor which has been executed by the DMA - * controller. It will run any callbacks, submit any dependencies, and then - * free the descriptor. + * This function is used on descriptors which have been executed by the DMA + * controller. It will run any callbacks, submit any dependencies, then + * free these descriptors if flag 'ack' is set. */ -static void fsldma_cleanup_descriptor(struct fsldma_chan *chan, - struct fsl_desc_sw *desc) +static void fsldma_cleanup_descriptors(struct fsldma_chan *chan) { - struct dma_async_tx_descriptor *txd = &desc->async_tx; - struct device *dev = chan->common.device->dev; - dma_addr_t src = get_desc_src(chan, desc); - dma_addr_t dst = get_desc_dst(chan, desc); - u32 len = get_desc_cnt(chan, desc); + struct fsl_desc_sw *desc, *_desc; + dma_cookie_t cookie = 0; + dma_addr_t curr_phys = get_cdar(chan); + int seen_current = 0; - /* Run the link descriptor callback function */ - if (txd->callback) { -#ifdef FSL_DMA_LD_DEBUG - chan_dbg(chan, "LD %p callback\n", desc); -#endif - txd->callback(txd->callback_param); - } + fsldma_clean_completed_descriptor(chan); - /* Run any dependencies */ - dma_run_dependencies(txd); + /* Run the callback for each descriptor, in order */ + list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) { + /* + * do not advance past the current descriptor loaded into the + * hardware channel, subsequent descriptors are either in + * process or have not been submitted + */ + if (seen_current) + break; - /* Unmap the dst buffer, if requested */ - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE); - else - dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE); - } + /* + * stop the search if we reach the current descriptor and the + * channel is busy + */ + if (desc->async_tx.phys == curr_phys) { + seen_current = 1; + if (!dma_is_idle(chan)) + break; + } - /* Unmap the src buffer, if requested */ - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(dev, src, len, DMA_TO_DEVICE); - else - dma_unmap_page(dev, src, len, DMA_TO_DEVICE); + cookie = fsldma_run_tx_complete_actions(chan, desc, cookie); + + fsldma_clean_running_descriptor(chan, desc); } - fsl_dma_free_descriptor(chan, desc); + /* + * Start any pending transactions automatically + * + * In the ideal case, we keep the DMA controller busy while we go + * ahead and free the descriptors below. + */ + fsl_chan_xfer_ld_queue(chan); + + if (cookie > 0) + chan->common.completed_cookie = cookie; } /** @@ -654,8 +766,10 @@ static void fsl_dma_free_chan_resources(struct dma_chan *dchan) chan_dbg(chan, "free all channel resources\n"); spin_lock_irqsave(&chan->desc_lock, flags); + fsldma_cleanup_descriptors(chan); fsldma_free_desc_list(chan, &chan->ld_pending); fsldma_free_desc_list(chan, &chan->ld_running); + fsldma_free_desc_list(chan, &chan->ld_completed); spin_unlock_irqrestore(&chan->desc_lock, flags); dma_pool_destroy(chan->desc_pool); @@ -893,6 +1007,7 @@ static int fsl_dma_device_control(struct dma_chan *dchan, /* Remove and free all of the descriptors in the LD queue */ fsldma_free_desc_list(chan, &chan->ld_pending); fsldma_free_desc_list(chan, &chan->ld_running); + fsldma_free_desc_list(chan, &chan->ld_completed); chan->idle = true; spin_unlock_irqrestore(&chan->desc_lock, flags); @@ -956,11 +1071,15 @@ static enum dma_status fsl_tx_status(struct dma_chan *dchan, enum dma_status ret; unsigned long flags; - spin_lock_irqsave(&chan->desc_lock, flags); ret = dma_cookie_status(dchan, cookie, txstate); + if (ret == DMA_SUCCESS) + return ret; + + spin_lock_irqsave(&chan->desc_lock, flags); + fsldma_cleanup_descriptors(chan); spin_unlock_irqrestore(&chan->desc_lock, flags); - return ret; + return dma_cookie_status(dchan, cookie, txstate); } /*----------------------------------------------------------------------------*/ @@ -1037,52 +1156,19 @@ static irqreturn_t fsldma_chan_irq(int irq, void *data) static void dma_do_tasklet(unsigned long data) { struct fsldma_chan *chan = (struct fsldma_chan *)data; - struct fsl_desc_sw *desc, *_desc; - LIST_HEAD(ld_cleanup); unsigned long flags; chan_dbg(chan, "tasklet entry\n"); spin_lock_irqsave(&chan->desc_lock, flags); - /* update the cookie if we have some descriptors to cleanup */ - if (!list_empty(&chan->ld_running)) { - dma_cookie_t cookie; - - desc = to_fsl_desc(chan->ld_running.prev); - cookie = desc->async_tx.cookie; - dma_cookie_complete(&desc->async_tx); - - chan_dbg(chan, "completed_cookie=%d\n", cookie); - } - - /* - * move the descriptors to a temporary list so we can drop the lock - * during the entire cleanup operation - */ - list_splice_tail_init(&chan->ld_running, &ld_cleanup); - /* the hardware is now idle and ready for more */ chan->idle = true; - /* - * Start any pending transactions automatically - * - * In the ideal case, we keep the DMA controller busy while we go - * ahead and free the descriptors below. - */ - fsl_chan_xfer_ld_queue(chan); - spin_unlock_irqrestore(&chan->desc_lock, flags); - - /* Run the callback for each descriptor, in order */ - list_for_each_entry_safe(desc, _desc, &ld_cleanup, node) { + /* Run all cleanup for descriptors which have been completed */ + fsldma_cleanup_descriptors(chan); - /* Remove from the list of transactions */ - list_del(&desc->node); - - /* Run all cleanup for this descriptor */ - fsldma_cleanup_descriptor(chan, desc); - } + spin_unlock_irqrestore(&chan->desc_lock, flags); chan_dbg(chan, "tasklet exit\n"); } @@ -1264,6 +1350,7 @@ static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev, spin_lock_init(&chan->desc_lock); INIT_LIST_HEAD(&chan->ld_pending); INIT_LIST_HEAD(&chan->ld_running); + INIT_LIST_HEAD(&chan->ld_completed); chan->idle = true; chan->common.device = &fdev->common; diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index f5c3879..a58275a 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -138,8 +138,21 @@ struct fsldma_chan { char name[8]; /* Channel name */ struct fsldma_chan_regs __iomem *regs; spinlock_t desc_lock; /* Descriptor operation lock */ - struct list_head ld_pending; /* Link descriptors queue */ - struct list_head ld_running; /* Link descriptors queue */ + /* + * Descriptors which are queued to run, but have not yet been + * submitted to the hardware for execution + */ + struct list_head ld_pending; + /* + * Descriptors which are currently being executed by the hardware + */ + struct list_head ld_running; + /* + * Descriptors which have finished execution by the hardware. These + * descriptors have already had their cleanup actions run. They are + * waiting for the ACK bit to be set by the async_tx API. + */ + struct list_head ld_completed; /* Link descriptors queue */ struct dma_chan common; /* DMA common channel */ struct dma_pool *desc_pool; /* Descriptors pool */ struct device *dev; /* Channel device */