From ea25968a32a621b02c3715d6b649f0c6ef53c24e Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 8 Sep 2009 17:53:02 -0700 Subject: ioat: implement a private tx_list Drop ioatdma's use of tx_list from struct dma_async_tx_descriptor in preparation for removal of this field. Cc: Maciej Sosnowski Signed-off-by: Dan Williams --- drivers/dma/ioat/dma.c | 7 ++++--- drivers/dma/ioat/dma.h | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'drivers/dma/ioat') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 17a518d0386..21527b89590 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -251,12 +251,12 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx) dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); /* write address into NextDescriptor field of last desc in chain */ - first = to_ioat_desc(tx->tx_list.next); + first = to_ioat_desc(desc->tx_list.next); chain_tail = to_ioat_desc(ioat->used_desc.prev); /* make descriptor updates globally visible before chaining */ wmb(); chain_tail->hw->next = first->txd.phys; - list_splice_tail_init(&tx->tx_list, &ioat->used_desc); + list_splice_tail_init(&desc->tx_list, &ioat->used_desc); dump_desc_dbg(ioat, chain_tail); dump_desc_dbg(ioat, first); @@ -297,6 +297,7 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags) memset(desc, 0, sizeof(*desc)); + INIT_LIST_HEAD(&desc_sw->tx_list); dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common); desc_sw->txd.tx_submit = ioat1_tx_submit; desc_sw->hw = desc; @@ -521,7 +522,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, desc->txd.flags = flags; desc->len = total_len; - list_splice(&chain, &desc->txd.tx_list); + list_splice(&chain, &desc->tx_list); hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.compl_write = 1; hw->tx_cnt = tx_cnt; diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index d9d6a7e3cd7..8966fa5453a 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -157,7 +157,7 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie, * struct ioat_desc_sw - wrapper around hardware descriptor * @hw: hardware DMA descriptor * @node: this descriptor will either be on the free list, - * or attached to a transaction list (async_tx.tx_list) + * or attached to a transaction list (tx_list) * @txd: the generic software descriptor for all engines * @id: identifier for debug */ @@ -165,6 +165,7 @@ struct ioat_desc_sw { struct ioat_dma_descriptor *hw; struct list_head node; size_t len; + struct list_head tx_list; struct dma_async_tx_descriptor txd; #ifdef DEBUG int id; -- cgit v1.2.3 From 162b96e63e518aa6ff029ce23de12d7f027483bf Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 8 Sep 2009 17:53:04 -0700 Subject: ioat2,3: cacheline align software descriptor allocations All the necessary fields for handling an ioat2,3 ring entry can fit into one cacheline. Move ->len prior to ->txd in struct ioat_ring_ent, and move allocation of these entries to a hw-cache-aligned kmem cache to reduce the number of cachelines dirtied for descriptor management. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v2.c | 5 +++-- drivers/dma/ioat/dma_v2.h | 3 ++- drivers/dma/ioat/pci.c | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) (limited to 'drivers/dma/ioat') diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 460b7730133..fa3d6db6624 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -399,11 +399,12 @@ static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t f return NULL; memset(hw, 0, sizeof(*hw)); - desc = kzalloc(sizeof(*desc), flags); + desc = kmem_cache_alloc(ioat2_cache, flags); if (!desc) { pci_pool_free(dma->dma_pool, hw, phys); return NULL; } + memset(desc, 0, sizeof(*desc)); dma_async_tx_descriptor_init(&desc->txd, chan); desc->txd.tx_submit = ioat2_tx_submit_unlock; @@ -418,7 +419,7 @@ static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *cha dma = to_ioatdma_device(chan->device); pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys); - kfree(desc); + kmem_cache_free(ioat2_cache, desc); } static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags) diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 9baa3d6065f..ac00adc8197 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -116,8 +116,8 @@ static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len struct ioat_ring_ent { struct ioat_dma_descriptor *hw; - struct dma_async_tx_descriptor txd; size_t len; + struct dma_async_tx_descriptor txd; #ifdef DEBUG int id; #endif @@ -143,4 +143,5 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca); int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca); struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); +extern struct kmem_cache *ioat2_cache; #endif /* IOATDMA_V2_H */ diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c index c4e43226925..61086c6bbf4 100644 --- a/drivers/dma/ioat/pci.c +++ b/drivers/dma/ioat/pci.c @@ -69,6 +69,8 @@ static int ioat_dca_enabled = 1; module_param(ioat_dca_enabled, int, 0644); MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); +struct kmem_cache *ioat2_cache; + #define DRV_NAME "ioatdma" static struct pci_driver ioat_pci_driver = { @@ -168,12 +170,24 @@ static void __devexit ioat_remove(struct pci_dev *pdev) static int __init ioat_init_module(void) { - return pci_register_driver(&ioat_pci_driver); + int err; + + ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ioat2_cache) + return -ENOMEM; + + err = pci_register_driver(&ioat_pci_driver); + if (err) + kmem_cache_destroy(ioat2_cache); + + return err; } module_init(ioat_init_module); static void __exit ioat_exit_module(void) { pci_unregister_driver(&ioat_pci_driver); + kmem_cache_destroy(ioat2_cache); } module_exit(ioat_exit_module); -- cgit v1.2.3