diff options
| -rw-r--r-- | drivers/dma/ioat/dma_v3.c | 37 | ||||
| -rw-r--r-- | drivers/dma/ioat/registers.h | 2 |
2 files changed, 34 insertions, 5 deletions
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 39520f2f7da9..9988f1340186 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -295,15 +295,23 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) ioat->tail += i; BUG_ON(active && !seen_current); /* no active descs have written a completion? */ chan->last_completion = phys_complete; - if (ioat->head == ioat->tail) { + + active = ioat2_ring_active(ioat); + if (active == 0) { dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", __func__); clear_bit(IOAT_COMPLETION_PENDING, &chan->state); mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); } + /* 5 microsecond delay per pending descriptor */ + writew(min((5 * active), IOAT_INTRDELAY_MASK), + chan->device->reg_base + IOAT_INTRDELAY_OFFSET); } -static void ioat3_cleanup(struct ioat2_dma_chan *ioat) +/* try to cleanup, but yield (via spin_trylock) to incoming submissions + * with the expectation that we will immediately poll again shortly + */ +static void ioat3_cleanup_poll(struct ioat2_dma_chan *ioat) { struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; @@ -329,11 +337,32 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat) spin_unlock_bh(&chan->cleanup_lock); } +/* run cleanup now because we already delayed the interrupt via INTRDELAY */ +static void ioat3_cleanup_sync(struct ioat2_dma_chan *ioat) +{ + struct ioat_chan_common *chan = &ioat->base; + unsigned long phys_complete; + + prefetch(chan->completion); + + spin_lock_bh(&chan->cleanup_lock); + if (!ioat_cleanup_preamble(chan, &phys_complete)) { + spin_unlock_bh(&chan->cleanup_lock); + return; + } + spin_lock_bh(&ioat->ring_lock); + + __cleanup(ioat, phys_complete); + + spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&chan->cleanup_lock); +} + static void ioat3_cleanup_tasklet(unsigned long data) { struct ioat2_dma_chan *ioat = (void *) data; - ioat3_cleanup(ioat); + ioat3_cleanup_sync(ioat); writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } @@ -417,7 +446,7 @@ ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie, if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) return DMA_SUCCESS; - ioat3_cleanup(ioat); + ioat3_cleanup_poll(ioat); return ioat_is_complete(c, cookie, done, used); } diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index e8ae63baf588..1391798542b6 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -60,7 +60,7 @@ #define IOAT_PERPORTOFFSET_OFFSET 0x0A /* 16-bit */ #define IOAT_INTRDELAY_OFFSET 0x0C /* 16-bit */ -#define IOAT_INTRDELAY_INT_DELAY_MASK 0x3FFF /* Interrupt Delay Time */ +#define IOAT_INTRDELAY_MASK 0x3FFF /* Interrupt Delay Time */ #define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalescing Supported */ #define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */ |