* [PATCH 18/29] ioat: ___devinit annotate the initialization paths
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, Maciej Sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Mark all single use initialization routines with __devinit.
Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dca.c | 9 ++++++---
drivers/dma/ioat/dma.c | 12 ++++++------
drivers/dma/ioat/dma.h | 11 ++++++-----
drivers/dma/ioat/dma_v2.c | 4 ++--
drivers/dma/ioat/dma_v2.h | 8 ++++----
5 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c
index af1c762..69d0261 100644
--- a/drivers/dma/ioat/dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -242,7 +242,8 @@ static struct dca_ops ioat_dca_ops = {
};
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
@@ -407,7 +408,8 @@ static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
return slots;
}
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
@@ -602,7 +604,8 @@ static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
return slots;
}
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 6dd0af1..abc96c4 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -870,7 +870,7 @@ static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
*/
#define IOAT_TEST_SIZE 2000
-static void ioat_dma_test_callback(void *dma_async_param)
+static void __devinit ioat_dma_test_callback(void *dma_async_param)
{
struct completion *cmp = dma_async_param;
@@ -881,7 +881,7 @@ static void ioat_dma_test_callback(void *dma_async_param)
* ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
* @device: device to be tested
*/
-static int ioat_dma_self_test(struct ioatdma_device *device)
+static int __devinit ioat_dma_self_test(struct ioatdma_device *device)
{
int i;
u8 *src;
@@ -1082,7 +1082,7 @@ static void ioat_disable_interrupts(struct ioatdma_device *device)
writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
}
-int ioat_probe(struct ioatdma_device *device)
+int __devinit ioat_probe(struct ioatdma_device *device)
{
int err = -ENODEV;
struct dma_device *dma = &device->common;
@@ -1142,7 +1142,7 @@ err_dma_pool:
return err;
}
-int ioat_register(struct ioatdma_device *device)
+int __devinit ioat_register(struct ioatdma_device *device)
{
int err = dma_async_device_register(&device->common);
@@ -1169,7 +1169,7 @@ static void ioat1_intr_quirk(struct ioatdma_device *device)
pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
}
-int ioat1_dma_probe(struct ioatdma_device *device, int dca)
+int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
@@ -1200,7 +1200,7 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
return err;
}
-void ioat_dma_remove(struct ioatdma_device *device)
+void __devexit ioat_dma_remove(struct ioatdma_device *device)
{
struct dma_device *dma = &device->common;
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 5fd6e2d..e47083b 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -217,11 +217,12 @@ ioat_chan_by_index(struct ioatdma_device *device, int index)
return device->idx[index];
}
-int ioat_probe(struct ioatdma_device *device);
-int ioat_register(struct ioatdma_device *device);
-int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
-void ioat_dma_remove(struct ioatdma_device *device);
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+int __devinit ioat_probe(struct ioatdma_device *device);
+int __devinit ioat_register(struct ioatdma_device *device);
+int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
+void __devexit ioat_dma_remove(struct ioatdma_device *device);
+struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
+ void __iomem *iobase);
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx,
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index be617db..a0def66 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -684,7 +684,7 @@ ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
return ioat_is_complete(c, cookie, done, used);
}
-int ioat2_dma_probe(struct ioatdma_device *device, int dca)
+int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
@@ -723,7 +723,7 @@ int ioat2_dma_probe(struct ioatdma_device *device, int dca)
return err;
}
-int ioat3_dma_probe(struct ioatdma_device *device, int dca)
+int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index c72ccb5..bdde537 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -127,8 +127,8 @@ ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
return ioat->ring[idx & ioat2_ring_mask(ioat)];
}
-int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
-int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
+struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
#endif /* IOATDMA_V2_H */
^ permalink raw reply related
* [PATCH 19/29] ioat1: trim ioat_dma_desc_sw
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Save 4 bytes per software descriptor by transmitting tx_cnt in an unused
portion of the hardware descriptor.
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma.c | 4 ++--
drivers/dma/ioat/dma.h | 2 --
drivers/dma/ioat/hw.h | 6 +++++-
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index abc96c4..f59b6f4 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -396,7 +396,7 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
dump_desc_dbg(ioat, chain_tail);
dump_desc_dbg(ioat, first);
- ioat->pending += desc->tx_cnt;
+ ioat->pending += desc->hw->tx_cnt;
if (ioat->pending >= ioat_pending_level)
__ioat1_dma_memcpy_issue_pending(ioat);
spin_unlock_bh(&ioat->desc_lock);
@@ -655,11 +655,11 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
spin_unlock_bh(&ioat->desc_lock);
desc->txd.flags = flags;
- desc->tx_cnt = tx_cnt;
desc->len = total_len;
list_splice(&chain, &desc->txd.tx_list);
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1;
+ hw->tx_cnt = tx_cnt;
dump_desc_dbg(ioat, desc);
return &desc->txd;
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index e47083b..ec851cf 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -165,14 +165,12 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
* @hw: hardware DMA descriptor
* @node: this descriptor will either be on the free list,
* or attached to a transaction list (async_tx.tx_list)
- * @tx_cnt: number of descriptors required to complete the transaction
* @txd: the generic software descriptor for all engines
* @id: identifier for debug
*/
struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
- int tx_cnt;
size_t len;
struct dma_async_tx_descriptor txd;
#ifdef DEBUG
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
index e13f3ed..7481fb1 100644
--- a/drivers/dma/ioat/hw.h
+++ b/drivers/dma/ioat/hw.h
@@ -63,7 +63,11 @@ struct ioat_dma_descriptor {
uint64_t next;
uint64_t rsv1;
uint64_t rsv2;
- uint64_t user1;
+ /* store some driver data in an unused portion of the descriptor */
+ union {
+ uint64_t user1;
+ uint64_t tx_cnt;
+ };
uint64_t user2;
};
#endif
^ permalink raw reply related
* [PATCH 20/29] ioat: switch watchdog and reset handler from workqueue to timer
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
In order to support dynamic resizing of the descriptor ring or polling
for a descriptor in the presence of a hung channel the reset handler
needs to make progress while in a non-preemptible context. The current
workqueue implementation precludes polling channel reset completion
under spin_lock().
This conversion also allows us to return to opportunistic cleanup in the
ioat2 case as the timer implementation guarantees at least one cleanup
after every descriptor is submitted. This means the worst case
completion latency becomes the timer frequency (for exceptional
circumstances), but with the benefit of avoiding busy waiting when the
lock is contended.
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma.c | 351 ++++++++++++++++--------------------------
drivers/dma/ioat/dma.h | 112 +++++++++++--
drivers/dma/ioat/dma_v2.c | 321 +++++++++++++++++---------------------
drivers/dma/ioat/dma_v2.h | 10 +
drivers/dma/ioat/registers.h | 22 +--
5 files changed, 388 insertions(+), 428 deletions(-)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index f59b6f4..17a518d 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -99,23 +99,26 @@ static void ioat1_cleanup_tasklet(unsigned long data);
/* common channel initialization */
void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx,
- work_func_t work_fn, void (*tasklet)(unsigned long),
- unsigned long tasklet_data)
+ void (*timer_fn)(unsigned long),
+ void (*tasklet)(unsigned long),
+ unsigned long ioat)
{
struct dma_device *dma = &device->common;
chan->device = device;
chan->reg_base = device->reg_base + (0x80 * (idx + 1));
- INIT_DELAYED_WORK(&chan->work, work_fn);
spin_lock_init(&chan->cleanup_lock);
chan->common.device = dma;
list_add_tail(&chan->common.device_node, &dma->channels);
device->idx[idx] = chan;
- tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
+ init_timer(&chan->timer);
+ chan->timer.function = timer_fn;
+ chan->timer.data = ioat;
+ tasklet_init(&chan->cleanup_task, tasklet, ioat);
tasklet_disable(&chan->cleanup_task);
}
-static void ioat1_reset_part2(struct work_struct *work);
+static void ioat1_timer_event(unsigned long data);
/**
* ioat1_dma_enumerate_channels - find and initialize the device's channels
@@ -153,7 +156,7 @@ static int ioat1_enumerate_channels(struct ioatdma_device *device)
break;
ioat_init_channel(device, &ioat->base, i,
- ioat1_reset_part2,
+ ioat1_timer_event,
ioat1_cleanup_tasklet,
(unsigned long) ioat);
ioat->xfercap = xfercap;
@@ -193,61 +196,6 @@ static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
}
/**
- * ioat1_reset_part2 - reinit the channel after a reset
- */
-static void ioat1_reset_part2(struct work_struct *work)
-{
- struct ioat_chan_common *chan;
- struct ioat_dma_chan *ioat;
- struct ioat_desc_sw *desc;
- int dmacount;
- bool start_null = false;
-
- chan = container_of(work, struct ioat_chan_common, work.work);
- ioat = container_of(chan, struct ioat_dma_chan, base);
- spin_lock_bh(&chan->cleanup_lock);
- spin_lock_bh(&ioat->desc_lock);
-
- *chan->completion = 0;
- ioat->pending = 0;
-
- /* count the descriptors waiting */
- dmacount = 0;
- if (ioat->used_desc.prev) {
- desc = to_ioat_desc(ioat->used_desc.prev);
- do {
- dmacount++;
- desc = to_ioat_desc(desc->node.next);
- } while (&desc->node != ioat->used_desc.next);
- }
-
- if (dmacount) {
- /*
- * write the new starting descriptor address
- * this puts channel engine into ARMED state
- */
- desc = to_ioat_desc(ioat->used_desc.prev);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
- writeb(IOAT_CHANCMD_START, chan->reg_base
- + IOAT_CHANCMD_OFFSET(chan->device->version));
- } else
- start_null = true;
- spin_unlock_bh(&ioat->desc_lock);
- spin_unlock_bh(&chan->cleanup_lock);
-
- dev_err(to_dev(chan),
- "chan%d reset - %d descs waiting, %d total desc\n",
- chan_num(chan), dmacount, ioat->desccount);
-
- if (start_null)
- ioat1_dma_start_null_desc(ioat);
-}
-
-/**
* ioat1_reset_channel - restart a channel
* @ioat: IOAT DMA channel handle
*/
@@ -257,12 +205,9 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
void __iomem *reg_base = chan->reg_base;
u32 chansts, chanerr;
- if (!ioat->used_desc.prev)
- return;
-
- dev_dbg(to_dev(chan), "%s\n", __func__);
+ dev_warn(to_dev(chan), "reset\n");
chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
- chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+ chansts = *chan->completion & IOAT_CHANSTS_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
@@ -278,93 +223,11 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
* while we're waiting.
*/
- spin_lock_bh(&ioat->desc_lock);
ioat->pending = INT_MIN;
writeb(IOAT_CHANCMD_RESET,
reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
- spin_unlock_bh(&ioat->desc_lock);
-
- /* schedule the 2nd half instead of sleeping a long time */
- schedule_delayed_work(&chan->work, RESET_DELAY);
-}
-
-/**
- * ioat1_chan_watchdog - watch for stuck channels
- */
-static void ioat1_chan_watchdog(struct work_struct *work)
-{
- struct ioatdma_device *device =
- container_of(work, struct ioatdma_device, work.work);
- struct ioat_dma_chan *ioat;
- struct ioat_chan_common *chan;
- int i;
- u64 completion;
- u32 completion_low;
- unsigned long compl_desc_addr_hw;
-
- for (i = 0; i < device->common.chancnt; i++) {
- chan = ioat_chan_by_index(device, i);
- ioat = container_of(chan, struct ioat_dma_chan, base);
-
- if (/* have we started processing anything yet */
- chan->last_completion
- /* have we completed any since last watchdog cycle? */
- && (chan->last_completion == chan->watchdog_completion)
- /* has TCP stuck on one cookie since last watchdog? */
- && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
- && (chan->watchdog_tcp_cookie != chan->completed_cookie)
- /* is there something in the chain to be processed? */
- /* CB1 chain always has at least the last one processed */
- && (ioat->used_desc.prev != ioat->used_desc.next)
- && ioat->pending == 0) {
-
- /*
- * check CHANSTS register for completed
- * descriptor address.
- * if it is different than completion writeback,
- * it is not zero
- * and it has changed since the last watchdog
- * we can assume that channel
- * is still working correctly
- * and the problem is in completion writeback.
- * update completion writeback
- * with actual CHANSTS value
- * else
- * try resetting the channel
- */
-
- /* we need to read the low address first as this
- * causes the chipset to latch the upper bits
- * for the subsequent read
- */
- completion_low = readl(chan->reg_base +
- IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
- completion = readl(chan->reg_base +
- IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
- completion <<= 32;
- completion |= completion_low;
- compl_desc_addr_hw = completion &
- IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-
- if ((compl_desc_addr_hw != 0)
- && (compl_desc_addr_hw != chan->watchdog_completion)
- && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
- chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
- *chan->completion = completion;
- } else {
- ioat1_reset_channel(ioat);
- chan->watchdog_completion = 0;
- chan->last_compl_desc_addr_hw = 0;
- }
- } else {
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_completion = chan->last_completion;
- }
-
- chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
- }
-
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+ set_bit(IOAT_RESET_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + RESET_DELAY);
}
static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
@@ -372,6 +235,7 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
struct dma_chan *c = tx->chan;
struct ioat_dma_chan *ioat = to_ioat_chan(c);
struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+ struct ioat_chan_common *chan = &ioat->base;
struct ioat_desc_sw *first;
struct ioat_desc_sw *chain_tail;
dma_cookie_t cookie;
@@ -396,6 +260,9 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
dump_desc_dbg(ioat, chain_tail);
dump_desc_dbg(ioat, first);
+ if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
ioat->pending += desc->hw->tx_cnt;
if (ioat->pending >= ioat_pending_level)
__ioat1_dma_memcpy_issue_pending(ioat);
@@ -520,6 +387,7 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)
return;
tasklet_disable(&chan->cleanup_task);
+ del_timer_sync(&chan->timer);
ioat1_cleanup(ioat);
/* Delay 100ms after reset to allow internal DMA logic to quiesce
@@ -560,9 +428,6 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)
chan->last_completion = 0;
chan->completion_dma = 0;
- chan->watchdog_completion = 0;
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
ioat->pending = 0;
ioat->desccount = 0;
}
@@ -705,15 +570,15 @@ unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
u64 completion;
completion = *chan->completion;
- phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+ phys_complete = ioat_chansts_to_addr(completion);
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete);
- if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
- IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
+ if (is_ioat_halted(completion)) {
+ u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
- readl(chan->reg_base + IOAT_CHANERR_OFFSET));
+ chanerr);
/* TODO do something to salvage the situation */
}
@@ -721,48 +586,31 @@ unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
return phys_complete;
}
-/**
- * ioat1_cleanup - cleanup up finished descriptors
- * @chan: ioat channel to be cleaned up
- */
-static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+ unsigned long *phys_complete)
{
- struct ioat_chan_common *chan = &ioat->base;
- unsigned long phys_complete;
- struct ioat_desc_sw *desc, *_desc;
- dma_cookie_t cookie = 0;
- struct dma_async_tx_descriptor *tx;
-
- prefetch(chan->completion);
-
- if (!spin_trylock_bh(&chan->cleanup_lock))
- return;
+ *phys_complete = ioat_get_current_completion(chan);
+ if (*phys_complete == chan->last_completion)
+ return false;
+ clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
- phys_complete = ioat_get_current_completion(chan);
- if (phys_complete == chan->last_completion) {
- spin_unlock_bh(&chan->cleanup_lock);
- /*
- * perhaps we're stuck so hard that the watchdog can't go off?
- * try to catch it after 2 seconds
- */
- if (time_after(jiffies,
- chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
- ioat1_chan_watchdog(&(chan->device->work.work));
- chan->last_completion_time = jiffies;
- }
- return;
- }
- chan->last_completion_time = jiffies;
+ return true;
+}
- cookie = 0;
- if (!spin_trylock_bh(&ioat->desc_lock)) {
- spin_unlock_bh(&chan->cleanup_lock);
- return;
- }
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ struct list_head *_desc, *n;
+ struct dma_async_tx_descriptor *tx;
dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
__func__, phys_complete);
- list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+ list_for_each_safe(_desc, n, &ioat->used_desc) {
+ struct ioat_desc_sw *desc;
+
+ prefetch(n);
+ desc = list_entry(_desc, typeof(*desc), node);
tx = &desc->txd;
/*
* Incoming DMA requests may use multiple descriptors,
@@ -771,7 +619,8 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
*/
dump_desc_dbg(ioat, desc);
if (tx->cookie) {
- cookie = tx->cookie;
+ chan->completed_cookie = tx->cookie;
+ tx->cookie = 0;
ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
if (tx->callback) {
tx->callback(tx->callback_param);
@@ -786,27 +635,110 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
*/
if (async_tx_test_ack(tx))
list_move_tail(&desc->node, &ioat->free_desc);
- else
- tx->cookie = 0;
} else {
/*
* last used desc. Do not remove, so we can
- * append from it, but don't look at it next
- * time, either
+ * append from it.
*/
- tx->cookie = 0;
+
+ /* if nothing else is pending, cancel the
+ * completion timeout
+ */
+ if (n == &ioat->used_desc) {
+ dev_dbg(to_dev(chan),
+ "%s cancel completion timeout\n",
+ __func__);
+ clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ }
/* TODO check status bits? */
break;
}
}
+ chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ unsigned long phys_complete;
+
+ prefetch(chan->completion);
+
+ if (!spin_trylock_bh(&chan->cleanup_lock))
+ return;
+
+ if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ if (!spin_trylock_bh(&ioat->desc_lock)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ __cleanup(ioat, phys_complete);
+
spin_unlock_bh(&ioat->desc_lock);
+ spin_unlock_bh(&chan->cleanup_lock);
+}
- chan->last_completion = phys_complete;
- if (cookie != 0)
- chan->completed_cookie = cookie;
+static void ioat1_timer_event(unsigned long data)
+{
+ struct ioat_dma_chan *ioat = (void *) data;
+ struct ioat_chan_common *chan = &ioat->base;
+ dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+ spin_lock_bh(&chan->cleanup_lock);
+ if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+ struct ioat_desc_sw *desc;
+
+ spin_lock_bh(&ioat->desc_lock);
+
+ /* restart active descriptors */
+ desc = to_ioat_desc(ioat->used_desc.prev);
+ ioat_set_chainaddr(ioat, desc->txd.phys);
+ ioat_start(chan);
+
+ ioat->pending = 0;
+ set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ spin_unlock_bh(&ioat->desc_lock);
+ } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+ unsigned long phys_complete;
+
+ spin_lock_bh(&ioat->desc_lock);
+ /* if we haven't made progress and we have already
+ * acknowledged a pending completion once, then be more
+ * forceful with a restart
+ */
+ if (ioat_cleanup_preamble(chan, &phys_complete))
+ __cleanup(ioat, phys_complete);
+ else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+ ioat1_reset_channel(ioat);
+ else {
+ u64 status = ioat_chansts(chan);
+
+ /* manually update the last completion address */
+ if (ioat_chansts_to_addr(status) != 0)
+ *chan->completion = status;
+
+ set_bit(IOAT_COMPLETION_ACK, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ }
+ spin_unlock_bh(&ioat->desc_lock);
+ }
spin_unlock_bh(&chan->cleanup_lock);
}
@@ -855,13 +787,8 @@ static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
list_add_tail(&desc->node, &ioat->used_desc);
dump_desc_dbg(ioat, desc);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
- writeb(IOAT_CHANCMD_START, chan->reg_base
- + IOAT_CHANCMD_OFFSET(chan->device->version));
+ ioat_set_chainaddr(ioat, desc->txd.phys);
+ ioat_start(chan);
spin_unlock_bh(&ioat->desc_lock);
}
@@ -1194,9 +1121,6 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
if (dca)
device->dca = ioat_dca_init(pdev, device->reg_base);
- INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-
return err;
}
@@ -1204,9 +1128,6 @@ void __devexit ioat_dma_remove(struct ioatdma_device *device)
{
struct dma_device *dma = &device->common;
- if (device->version != IOAT_VER_3_0)
- cancel_delayed_work(&device->work);
-
ioat_disable_interrupts(device);
dma_async_device_unregister(dma);
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index ec851cf..dbfccac 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -23,6 +23,7 @@
#include <linux/dmaengine.h>
#include "hw.h"
+#include "registers.h"
#include <linux/init.h>
#include <linux/dmapool.h>
#include <linux/cache.h>
@@ -33,7 +34,6 @@
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
#define IOAT_DMA_DCA_ANY_CPU ~0
-#define IOAT_WATCHDOG_PERIOD (2 * HZ)
#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
@@ -42,9 +42,6 @@
#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
-#define RESET_DELAY msecs_to_jiffies(100)
-#define WATCHDOG_DELAY round_jiffies(msecs_to_jiffies(2000))
-
/*
* workaround for IOAT ver.3.0 null descriptor issue
* (channel returns error when size is 0)
@@ -72,7 +69,6 @@ struct ioatdma_device {
struct pci_pool *completion_pool;
struct dma_device common;
u8 version;
- struct delayed_work work;
struct msix_entry msix_entries[4];
struct ioat_chan_common *idx[4];
struct dca_provider *dca;
@@ -81,24 +77,21 @@ struct ioatdma_device {
};
struct ioat_chan_common {
+ struct dma_chan common;
void __iomem *reg_base;
-
unsigned long last_completion;
- unsigned long last_completion_time;
-
spinlock_t cleanup_lock;
dma_cookie_t completed_cookie;
- unsigned long watchdog_completion;
- int watchdog_tcp_cookie;
- u32 watchdog_last_tcp_cookie;
- struct delayed_work work;
-
+ unsigned long state;
+ #define IOAT_COMPLETION_PENDING 0
+ #define IOAT_COMPLETION_ACK 1
+ #define IOAT_RESET_PENDING 2
+ struct timer_list timer;
+ #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+ #define RESET_DELAY msecs_to_jiffies(100)
struct ioatdma_device *device;
- struct dma_chan common;
-
dma_addr_t completion_dma;
u64 *completion;
- unsigned long last_compl_desc_addr_hw;
struct tasklet_struct cleanup_task;
};
@@ -148,7 +141,6 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
last_used = c->cookie;
last_complete = chan->completed_cookie;
- chan->watchdog_tcp_cookie = cookie;
if (done)
*done = last_complete;
@@ -215,6 +207,85 @@ ioat_chan_by_index(struct ioatdma_device *device, int index)
return device->idx[index];
}
+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
+{
+ u8 ver = chan->device->version;
+ u64 status;
+ u32 status_lo;
+
+ /* We need to read the low address first as this causes the
+ * chipset to latch the upper bits for the subsequent read
+ */
+ status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
+ status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
+ status <<= 32;
+ status |= status_lo;
+
+ return status;
+}
+
+static inline void ioat_start(struct ioat_chan_common *chan)
+{
+ u8 ver = chan->device->version;
+
+ writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline u64 ioat_chansts_to_addr(u64 status)
+{
+ return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+}
+
+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
+{
+ return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+}
+
+static inline void ioat_suspend(struct ioat_chan_common *chan)
+{
+ u8 ver = chan->device->version;
+
+ writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+
+ writel(addr & 0x00000000FFFFFFFF,
+ chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
+ writel(addr >> 32,
+ chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
+}
+
+static inline bool is_ioat_active(unsigned long status)
+{
+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
+}
+
+static inline bool is_ioat_idle(unsigned long status)
+{
+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
+}
+
+static inline bool is_ioat_halted(unsigned long status)
+{
+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
+}
+
+static inline bool is_ioat_suspended(unsigned long status)
+{
+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
+}
+
+/* channel was fatally programmed */
+static inline bool is_ioat_bug(unsigned long err)
+{
+ return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
+ IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
+ IOAT_CHANERR_LENGTH_ERR));
+}
+
int __devinit ioat_probe(struct ioatdma_device *device);
int __devinit ioat_register(struct ioatdma_device *device);
int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
@@ -224,8 +295,11 @@ struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx,
- work_func_t work_fn, void (*tasklet)(unsigned long),
- unsigned long tasklet_data);
+ void (*timer_fn)(unsigned long),
+ void (*tasklet)(unsigned long),
+ unsigned long ioat);
void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
size_t len, struct ioat_dma_descriptor *hw);
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+ unsigned long *phys_complete);
#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index a0def66..a92b797 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -49,7 +49,7 @@ static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
void * __iomem reg_base = ioat->base.reg_base;
ioat->pending = 0;
- ioat->dmacount += ioat2_ring_pending(ioat);
+ ioat->dmacount += ioat2_ring_pending(ioat);;
ioat->issued = ioat->head;
/* make descriptor updates globally visible before notifying channel */
wmb();
@@ -92,7 +92,6 @@ static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
{
- void __iomem *reg_base = ioat->base.reg_base;
struct ioat_ring_ent *desc;
struct ioat_dma_descriptor *hw;
int idx;
@@ -119,10 +118,7 @@ static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
hw->src_addr = 0;
hw->dst_addr = 0;
async_tx_ack(&desc->txd);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+ ioat2_set_chainaddr(ioat, desc->txd.phys);
dump_desc_dbg(ioat, desc);
__ioat2_issue_pending(ioat);
}
@@ -134,177 +130,14 @@ static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
spin_unlock_bh(&ioat->ring_lock);
}
-static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
-
-/**
- * ioat2_reset_part2 - reinit the channel after a reset
- */
-static void ioat2_reset_part2(struct work_struct *work)
-{
- struct ioat_chan_common *chan;
- struct ioat2_dma_chan *ioat;
-
- chan = container_of(work, struct ioat_chan_common, work.work);
- ioat = container_of(chan, struct ioat2_dma_chan, base);
-
- /* ensure that ->tail points to the stalled descriptor
- * (ioat->pending is set to 2 at this point so no new
- * descriptors will be issued while we perform this cleanup)
- */
- ioat2_cleanup(ioat);
-
- spin_lock_bh(&chan->cleanup_lock);
- spin_lock_bh(&ioat->ring_lock);
-
- /* set the tail to be re-issued */
- ioat->issued = ioat->tail;
- ioat->dmacount = 0;
-
- dev_dbg(to_dev(&ioat->base),
- "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
- __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
-
- if (ioat2_ring_pending(ioat)) {
- struct ioat_ring_ent *desc;
-
- desc = ioat2_get_ring_ent(ioat, ioat->tail);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
- __ioat2_issue_pending(ioat);
- } else
- __ioat2_start_null_desc(ioat);
-
- spin_unlock_bh(&ioat->ring_lock);
- spin_unlock_bh(&chan->cleanup_lock);
-
- dev_info(to_dev(chan),
- "chan%d reset - %d descs waiting, %d total desc\n",
- chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
-}
-
-/**
- * ioat2_reset_channel - restart a channel
- * @ioat: IOAT DMA channel handle
- */
-static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
{
- u32 chansts, chanerr;
struct ioat_chan_common *chan = &ioat->base;
- u16 active;
-
- spin_lock_bh(&ioat->ring_lock);
- active = ioat2_ring_active(ioat);
- spin_unlock_bh(&ioat->ring_lock);
- if (!active)
- return;
-
- chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
- chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
- if (chanerr) {
- dev_err(to_dev(chan),
- "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
- chan_num(chan), chansts, chanerr);
- writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
- }
-
- spin_lock_bh(&ioat->ring_lock);
- ioat->pending = 2;
- writeb(IOAT_CHANCMD_RESET,
- chan->reg_base
- + IOAT_CHANCMD_OFFSET(chan->device->version));
- spin_unlock_bh(&ioat->ring_lock);
- schedule_delayed_work(&chan->work, RESET_DELAY);
-}
-
-/**
- * ioat2_chan_watchdog - watch for stuck channels
- */
-static void ioat2_chan_watchdog(struct work_struct *work)
-{
- struct ioatdma_device *device =
- container_of(work, struct ioatdma_device, work.work);
- struct ioat2_dma_chan *ioat;
- struct ioat_chan_common *chan;
- u16 active;
- int i;
-
- dev_dbg(&device->pdev->dev, "%s\n", __func__);
-
- for (i = 0; i < device->common.chancnt; i++) {
- chan = ioat_chan_by_index(device, i);
- ioat = container_of(chan, struct ioat2_dma_chan, base);
-
- /*
- * for version 2.0 if there are descriptors yet to be processed
- * and the last completed hasn't changed since the last watchdog
- * if they haven't hit the pending level
- * issue the pending to push them through
- * else
- * try resetting the channel
- */
- spin_lock_bh(&ioat->ring_lock);
- active = ioat2_ring_active(ioat);
- spin_unlock_bh(&ioat->ring_lock);
-
- if (active &&
- chan->last_completion &&
- chan->last_completion == chan->watchdog_completion) {
-
- if (ioat->pending == 1)
- ioat2_issue_pending(&chan->common);
- else {
- ioat2_reset_channel(ioat);
- chan->watchdog_completion = 0;
- }
- } else {
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_completion = chan->last_completion;
- }
- chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
- }
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-}
-
-/**
- * ioat2_cleanup - clean finished descriptors (advance tail pointer)
- * @chan: ioat channel to be cleaned up
- */
-static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
-{
- struct ioat_chan_common *chan = &ioat->base;
- unsigned long phys_complete;
+ struct dma_async_tx_descriptor *tx;
struct ioat_ring_ent *desc;
bool seen_current = false;
u16 active;
int i;
- struct dma_async_tx_descriptor *tx;
-
- prefetch(chan->completion);
-
- spin_lock_bh(&chan->cleanup_lock);
- phys_complete = ioat_get_current_completion(chan);
- if (phys_complete == chan->last_completion) {
- spin_unlock_bh(&chan->cleanup_lock);
- /*
- * perhaps we're stuck so hard that the watchdog can't go off?
- * try to catch it after WATCHDOG_DELAY seconds
- */
- if (chan->device->version < IOAT_VER_3_0) {
- unsigned long tmo;
-
- tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
- if (time_after(jiffies, tmo)) {
- ioat2_chan_watchdog(&(chan->device->work.work));
- chan->last_completion_time = jiffies;
- }
- }
- return;
- }
- chan->last_completion_time = jiffies;
-
- spin_lock_bh(&ioat->ring_lock);
dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued);
@@ -330,10 +163,42 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
}
ioat->tail += i;
BUG_ON(!seen_current); /* no active descs have written a completion? */
- spin_unlock_bh(&ioat->ring_lock);
chan->last_completion = phys_complete;
+ if (ioat->head == ioat->tail) {
+ dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+ __func__);
+ clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ }
+}
+
+/**
+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
+ * @chan: ioat channel to be cleaned up
+ */
+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ unsigned long phys_complete;
+ prefetch(chan->completion);
+
+ if (!spin_trylock_bh(&chan->cleanup_lock))
+ return;
+
+ if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ if (!spin_trylock_bh(&ioat->ring_lock)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ __cleanup(ioat, phys_complete);
+
+ spin_unlock_bh(&ioat->ring_lock);
spin_unlock_bh(&chan->cleanup_lock);
}
@@ -345,6 +210,90 @@ static void ioat2_cleanup_tasklet(unsigned long data)
writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
}
+static void __restart_chan(struct ioat2_dma_chan *ioat)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+
+ /* set the tail to be re-issued */
+ ioat->issued = ioat->tail;
+ ioat->dmacount = 0;
+ set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+ dev_dbg(to_dev(chan),
+ "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+ __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+
+ if (ioat2_ring_pending(ioat)) {
+ struct ioat_ring_ent *desc;
+
+ desc = ioat2_get_ring_ent(ioat, ioat->tail);
+ ioat2_set_chainaddr(ioat, desc->txd.phys);
+ __ioat2_issue_pending(ioat);
+ } else
+ __ioat2_start_null_desc(ioat);
+}
+
+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ unsigned long phys_complete;
+ u32 status;
+
+ status = ioat_chansts(chan);
+ if (is_ioat_active(status) || is_ioat_idle(status))
+ ioat_suspend(chan);
+ while (is_ioat_active(status) || is_ioat_idle(status)) {
+ status = ioat_chansts(chan);
+ cpu_relax();
+ }
+
+ if (ioat_cleanup_preamble(chan, &phys_complete))
+ __cleanup(ioat, phys_complete);
+
+ __restart_chan(ioat);
+}
+
+static void ioat2_timer_event(unsigned long data)
+{
+ struct ioat2_dma_chan *ioat = (void *) data;
+ struct ioat_chan_common *chan = &ioat->base;
+
+ spin_lock_bh(&chan->cleanup_lock);
+ if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+ unsigned long phys_complete;
+ u64 status;
+
+ spin_lock_bh(&ioat->ring_lock);
+ status = ioat_chansts(chan);
+
+ /* when halted due to errors check for channel
+ * programming errors before advancing the completion state
+ */
+ if (is_ioat_halted(status)) {
+ u32 chanerr;
+
+ chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+ BUG_ON(is_ioat_bug(chanerr));
+ }
+
+ /* if we haven't made progress and we have already
+ * acknowledged a pending completion once, then be more
+ * forceful with a restart
+ */
+ if (ioat_cleanup_preamble(chan, &phys_complete))
+ __cleanup(ioat, phys_complete);
+ else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+ ioat2_restart_channel(ioat);
+ else {
+ set_bit(IOAT_COMPLETION_ACK, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ }
+ spin_unlock_bh(&ioat->ring_lock);
+ }
+ spin_unlock_bh(&chan->cleanup_lock);
+}
+
/**
* ioat2_enumerate_channels - find and initialize the device's channels
* @device: the device to be enumerated
@@ -382,7 +331,7 @@ static int ioat2_enumerate_channels(struct ioatdma_device *device)
break;
ioat_init_channel(device, &ioat->base, i,
- ioat2_reset_part2,
+ ioat2_timer_event,
ioat2_cleanup_tasklet,
(unsigned long) ioat);
ioat->xfercap_log = xfercap_log;
@@ -396,6 +345,7 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
{
struct dma_chan *c = tx->chan;
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+ struct ioat_chan_common *chan = &ioat->base;
dma_cookie_t cookie = c->cookie;
cookie++;
@@ -405,6 +355,8 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
c->cookie = cookie;
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+ if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
ioat2_update_pending(ioat);
spin_unlock_bh(&ioat->ring_lock);
@@ -544,9 +496,18 @@ static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_d
ioat->issued);
spin_unlock_bh(&ioat->ring_lock);
- /* do direct reclaim in the allocation failure case */
- ioat2_cleanup(ioat);
-
+ /* progress reclaim in the allocation failure case we
+ * may be called under bh_disabled so we need to trigger
+ * the timer event directly
+ */
+ spin_lock_bh(&chan->cleanup_lock);
+ if (jiffies > chan->timer.expires &&
+ timer_pending(&chan->timer)) {
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ spin_unlock_bh(&chan->cleanup_lock);
+ ioat2_timer_event((unsigned long) ioat);
+ } else
+ spin_unlock_bh(&chan->cleanup_lock);
return -ENOMEM;
}
@@ -625,6 +586,7 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
return;
tasklet_disable(&chan->cleanup_task);
+ del_timer_sync(&chan->timer);
ioat2_cleanup(ioat);
/* Delay 100ms after reset to allow internal DMA logic to quiesce
@@ -664,10 +626,6 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
chan->completion_dma = 0;
ioat->pending = 0;
ioat->dmacount = 0;
- chan->watchdog_completion = 0;
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_tcp_cookie = 0;
- chan->watchdog_last_tcp_cookie = 0;
}
static enum dma_status
@@ -717,9 +675,6 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
if (dca)
device->dca = ioat2_dca_init(pdev, device->reg_base);
- INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-
return err;
}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index bdde537..73b04a2 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -127,6 +127,16 @@ ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
return ioat->ring[idx & ioat2_ring_mask(ioat)];
}
+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+
+ writel(addr & 0x00000000FFFFFFFF,
+ chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
+ writel(addr >> 32,
+ chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+}
+
int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index 4380f6f..e4334a1 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -101,11 +101,11 @@
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x7ULL
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
+#define IOAT_CHANSTS_STATUS 0x7ULL
+#define IOAT_CHANSTS_ACTIVE 0x0
+#define IOAT_CHANSTS_DONE 0x1
+#define IOAT_CHANSTS_SUSPENDED 0x2
+#define IOAT_CHANSTS_HALTED 0x3
@@ -208,18 +208,18 @@
#define IOAT_CDAR_OFFSET_HIGH 0x24
#define IOAT_CHANERR_OFFSET 0x28 /* 32-bit Channel Error Register */
-#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR 0x0001
-#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR 0x0004
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR 0x0008
+#define IOAT_CHANERR_SRC_ADDR_ERR 0x0001
+#define IOAT_CHANERR_DEST_ADDR_ERR 0x0002
+#define IOAT_CHANERR_NEXT_ADDR_ERR 0x0004
+#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR 0x0008
#define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR 0x0010
#define IOAT_CHANERR_CHANCMD_ERR 0x0020
#define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0040
#define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0080
#define IOAT_CHANERR_READ_DATA_ERR 0x0100
#define IOAT_CHANERR_WRITE_DATA_ERR 0x0200
-#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR 0x0400
-#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR 0x0800
+#define IOAT_CHANERR_CONTROL_ERR 0x0400
+#define IOAT_CHANERR_LENGTH_ERR 0x0800
#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
#define IOAT_CHANERR_SOFT_ERR 0x4000
^ permalink raw reply related
* [PATCH 21/29] ioat2,3: dynamically resize descriptor ring
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Increment the allocation order of the descriptor ring every time we run
out of descriptors up to a maximum of allocation order specified by the
module parameter 'ioat_max_alloc_order'. After each idle period
decrement the allocation order to a minimum order of
'ioat_ring_alloc_order' (i.e. the default ring size, tunable as a module
parameter).
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma.h | 1
drivers/dma/ioat/dma_v2.c | 215 +++++++++++++++++++++++++++++++++++++++------
drivers/dma/ioat/dma_v2.h | 2
3 files changed, 187 insertions(+), 31 deletions(-)
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index dbfccac..d9d6a7e 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -88,6 +88,7 @@ struct ioat_chan_common {
#define IOAT_RESET_PENDING 2
struct timer_list timer;
#define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+ #define IDLE_TIMEOUT msecs_to_jiffies(2000)
#define RESET_DELAY msecs_to_jiffies(100)
struct ioatdma_device *device;
dma_addr_t completion_dma;
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index a92b797..9ea1a47 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -43,6 +43,10 @@ static int ioat_ring_alloc_order = 8;
module_param(ioat_ring_alloc_order, int, 0644);
MODULE_PARM_DESC(ioat_ring_alloc_order,
"ioat2+: allocate 2^n descriptors per channel (default: n=8)");
+static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
+module_param(ioat_ring_max_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_max_alloc_order,
+ "ioat2+: upper limit for dynamic ring resizing (default: n=16)");
static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
{
@@ -169,6 +173,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
__func__);
clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
}
}
@@ -254,6 +259,8 @@ static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
__restart_chan(ioat);
}
+static bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
+
static void ioat2_timer_event(unsigned long data)
{
struct ioat2_dma_chan *ioat = (void *) data;
@@ -290,6 +297,23 @@ static void ioat2_timer_event(unsigned long data)
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
}
spin_unlock_bh(&ioat->ring_lock);
+ } else {
+ u16 active;
+
+ /* if the ring is idle, empty, and oversized try to step
+ * down the size
+ */
+ spin_lock_bh(&ioat->ring_lock);
+ active = ioat2_ring_active(ioat);
+ if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+ reshape_ring(ioat, ioat->alloc_order-1);
+ spin_unlock_bh(&ioat->ring_lock);
+
+ /* keep shrinking until we get back to our minimum
+ * default size
+ */
+ if (ioat->alloc_order > ioat_get_alloc_order())
+ mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
}
spin_unlock_bh(&chan->cleanup_lock);
}
@@ -363,7 +387,7 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
return cookie;
}
-static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
+static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
{
struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *desc;
@@ -371,12 +395,12 @@ static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
dma_addr_t phys;
dma = to_ioatdma_device(chan->device);
- hw = pci_pool_alloc(dma->dma_pool, GFP_KERNEL, &phys);
+ hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
if (!hw)
return NULL;
memset(hw, 0, sizeof(*hw));
- desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+ desc = kzalloc(sizeof(*desc), flags);
if (!desc) {
pci_pool_free(dma->dma_pool, hw, phys);
return NULL;
@@ -398,6 +422,42 @@ static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *cha
kfree(desc);
}
+static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
+{
+ struct ioat_ring_ent **ring;
+ int descs = 1 << order;
+ int i;
+
+ if (order > ioat_get_max_alloc_order())
+ return NULL;
+
+ /* allocate the array to hold the software ring */
+ ring = kcalloc(descs, sizeof(*ring), flags);
+ if (!ring)
+ return NULL;
+ for (i = 0; i < descs; i++) {
+ ring[i] = ioat2_alloc_ring_ent(c, flags);
+ if (!ring[i]) {
+ while (i--)
+ ioat2_free_ring_ent(ring[i], c);
+ kfree(ring);
+ return NULL;
+ }
+ set_desc_id(ring[i], i);
+ }
+
+ /* link descs */
+ for (i = 0; i < descs-1; i++) {
+ struct ioat_ring_ent *next = ring[i+1];
+ struct ioat_dma_descriptor *hw = ring[i]->hw;
+
+ hw->next = next->txd.phys;
+ }
+ ring[i]->hw->next = ring[0]->txd.phys;
+
+ return ring;
+}
+
/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
* @chan: channel to be initialized
*/
@@ -407,8 +467,7 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)
struct ioat_chan_common *chan = &ioat->base;
struct ioat_ring_ent **ring;
u32 chanerr;
- int descs;
- int i;
+ int order;
/* have we already been set up? */
if (ioat->ring)
@@ -436,32 +495,10 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)
writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
- ioat->alloc_order = ioat_get_alloc_order();
- descs = 1 << ioat->alloc_order;
-
- /* allocate the array to hold the software ring */
- ring = kcalloc(descs, sizeof(*ring), GFP_KERNEL);
+ order = ioat_get_alloc_order();
+ ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
if (!ring)
return -ENOMEM;
- for (i = 0; i < descs; i++) {
- ring[i] = ioat2_alloc_ring_ent(c);
- if (!ring[i]) {
- while (i--)
- ioat2_free_ring_ent(ring[i], c);
- kfree(ring);
- return -ENOMEM;
- }
- set_desc_id(ring[i], i);
- }
-
- /* link descs */
- for (i = 0; i < descs-1; i++) {
- struct ioat_ring_ent *next = ring[i+1];
- struct ioat_dma_descriptor *hw = ring[i]->hw;
-
- hw->next = next->txd.phys;
- }
- ring[i]->hw->next = ring[0]->txd.phys;
spin_lock_bh(&ioat->ring_lock);
ioat->ring = ring;
@@ -469,12 +506,120 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)
ioat->issued = 0;
ioat->tail = 0;
ioat->pending = 0;
+ ioat->alloc_order = order;
spin_unlock_bh(&ioat->ring_lock);
tasklet_enable(&chan->cleanup_task);
ioat2_start_null_desc(ioat);
- return descs;
+ return 1 << ioat->alloc_order;
+}
+
+static bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
+{
+ /* reshape differs from normal ring allocation in that we want
+ * to allocate a new software ring while only
+ * extending/truncating the hardware ring
+ */
+ struct ioat_chan_common *chan = &ioat->base;
+ struct dma_chan *c = &chan->common;
+ const u16 curr_size = ioat2_ring_mask(ioat) + 1;
+ const u16 active = ioat2_ring_active(ioat);
+ const u16 new_size = 1 << order;
+ struct ioat_ring_ent **ring;
+ u16 i;
+
+ if (order > ioat_get_max_alloc_order())
+ return false;
+
+ /* double check that we have at least 1 free descriptor */
+ if (active == curr_size)
+ return false;
+
+ /* when shrinking, verify that we can hold the current active
+ * set in the new ring
+ */
+ if (active >= new_size)
+ return false;
+
+ /* allocate the array to hold the software ring */
+ ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
+ if (!ring)
+ return false;
+
+ /* allocate/trim descriptors as needed */
+ if (new_size > curr_size) {
+ /* copy current descriptors to the new ring */
+ for (i = 0; i < curr_size; i++) {
+ u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+ u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+ ring[new_idx] = ioat->ring[curr_idx];
+ set_desc_id(ring[new_idx], new_idx);
+ }
+
+ /* add new descriptors to the ring */
+ for (i = curr_size; i < new_size; i++) {
+ u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+ ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
+ if (!ring[new_idx]) {
+ while (i--) {
+ u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+ ioat2_free_ring_ent(ring[new_idx], c);
+ }
+ kfree(ring);
+ return false;
+ }
+ set_desc_id(ring[new_idx], new_idx);
+ }
+
+ /* hw link new descriptors */
+ for (i = curr_size-1; i < new_size; i++) {
+ u16 new_idx = (ioat->tail+i) & (new_size-1);
+ struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
+ struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
+
+ hw->next = next->txd.phys;
+ }
+ } else {
+ struct ioat_dma_descriptor *hw;
+ struct ioat_ring_ent *next;
+
+ /* copy current descriptors to the new ring, dropping the
+ * removed descriptors
+ */
+ for (i = 0; i < new_size; i++) {
+ u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+ u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+ ring[new_idx] = ioat->ring[curr_idx];
+ set_desc_id(ring[new_idx], new_idx);
+ }
+
+ /* free deleted descriptors */
+ for (i = new_size; i < curr_size; i++) {
+ struct ioat_ring_ent *ent;
+
+ ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
+ ioat2_free_ring_ent(ent, c);
+ }
+
+ /* fix up hardware ring */
+ hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
+ next = ring[(ioat->tail+new_size) & (new_size-1)];
+ hw->next = next->txd.phys;
+ }
+
+ dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+ __func__, new_size);
+
+ kfree(ioat->ring);
+ ioat->ring = ring;
+ ioat->alloc_order = order;
+
+ return true;
}
/**
@@ -488,7 +633,15 @@ static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_d
struct ioat_chan_common *chan = &ioat->base;
spin_lock_bh(&ioat->ring_lock);
- if (unlikely(ioat2_ring_space(ioat) < num_descs)) {
+ /* never allow the last descriptor to be consumed, we need at
+ * least one free at all times to allow for on-the-fly ring
+ * resizing.
+ */
+ while (unlikely(ioat2_ring_space(ioat) <= num_descs)) {
+ if (reshape_ring(ioat, ioat->alloc_order + 1) &&
+ ioat2_ring_space(ioat) > num_descs)
+ break;
+
if (printk_ratelimit())
dev_dbg(to_dev(chan),
"%s: ring full! num_descs: %d (%x:%x:%x)\n",
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 73b04a2..9baa3d6 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -37,6 +37,8 @@ extern int ioat_pending_level;
#define IOAT_MAX_ORDER 16
#define ioat_get_alloc_order() \
(min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
+#define ioat_get_max_alloc_order() \
+ (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
* @base: common ioat channel parameters
^ permalink raw reply related
* [PATCH 22/29] net_dma: poll for a descriptor after allocation failure
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Handle descriptor allocation failures by polling for a descriptor. The
driver will force forward progress when polled. In the best case this
polling interval will be the time it takes for one dma memcpy
transaction to complete. In the worst case, channel hang, we will need
to wait 100ms for the cleanup watchdog to fire (ioatdma driver).
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/iovlock.c | 10 ++++++++++
1 files changed, 10 insertions(+), 0 deletions(-)
diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c
index 9f6fe46..c0a272c 100644
--- a/drivers/dma/iovlock.c
+++ b/drivers/dma/iovlock.c
@@ -183,6 +183,11 @@ dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
iov_byte_offset,
kdata,
copy);
+ /* poll for a descriptor slot */
+ if (unlikely(dma_cookie < 0)) {
+ dma_async_issue_pending(chan);
+ continue;
+ }
len -= copy;
iov[iovec_idx].iov_len -= copy;
@@ -248,6 +253,11 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
page,
offset,
copy);
+ /* poll for a descriptor slot */
+ if (unlikely(dma_cookie < 0)) {
+ dma_async_issue_pending(chan);
+ continue;
+ }
len -= copy;
iov[iovec_idx].iov_len -= copy;
^ permalink raw reply related
* [PATCH 23/29] dw_dmac: implement a private tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, Haavard Skinnemoen, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Drop dw_dmac's use of tx_list from struct dma_async_tx_descriptor in
preparation for removal of this field.
Cc: Haavard Skinnemoen <haavard.skinnemoen@atmel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/dw_dmac.c | 19 ++++++++++---------
drivers/dma/dw_dmac_regs.h | 1 +
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 98c9a84..514ef7d 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -116,7 +116,7 @@ static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
{
struct dw_desc *child;
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
dma_sync_single_for_cpu(chan2parent(&dwc->chan),
child->txd.phys, sizeof(child->lli),
DMA_TO_DEVICE);
@@ -137,11 +137,11 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
dwc_sync_desc_for_cpu(dwc, desc);
spin_lock_bh(&dwc->lock);
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
dev_vdbg(chan2dev(&dwc->chan),
"moving child desc %p to freelist\n",
child);
- list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+ list_splice_init(&desc->tx_list, &dwc->free_list);
dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
list_add(&desc->desc_node, &dwc->free_list);
spin_unlock_bh(&dwc->lock);
@@ -209,7 +209,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
param = txd->callback_param;
dwc_sync_desc_for_cpu(dwc, desc);
- list_splice_init(&txd->tx_list, &dwc->free_list);
+ list_splice_init(&desc->tx_list, &dwc->free_list);
list_move(&desc->desc_node, &dwc->free_list);
/*
@@ -289,7 +289,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
/* This one is currently in progress */
return;
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
if (child->lli.llp == llp)
/* Currently in progress */
return;
@@ -356,7 +356,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
" cookie: %d\n", bad_desc->txd.cookie);
dwc_dump_lli(dwc, &bad_desc->lli);
- list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &bad_desc->tx_list, desc_node)
dwc_dump_lli(dwc, &child->lli);
/* Pretend the descriptor completed successfully */
@@ -608,7 +608,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
prev->txd.phys, sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
}
@@ -700,7 +700,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
total_len += len;
@@ -746,7 +746,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
total_len += len;
@@ -902,6 +902,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
break;
}
+ INIT_LIST_HEAD(&desc->tx_list);
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = dwc_tx_submit;
desc->txd.flags = DMA_CTRL_ACK;
diff --git a/drivers/dma/dw_dmac_regs.h b/drivers/dma/dw_dmac_regs.h
index 13a5807..d9a939f 100644
--- a/drivers/dma/dw_dmac_regs.h
+++ b/drivers/dma/dw_dmac_regs.h
@@ -217,6 +217,7 @@ struct dw_desc {
/* THEN values for driver housekeeping */
struct list_head desc_node;
+ struct list_head tx_list;
struct dma_async_tx_descriptor txd;
size_t len;
};
^ permalink raw reply related
* [PATCH 24/29] fsldma: implement a private tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, Li Yang, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Drop fsldma's use of tx_list from struct dma_async_tx_descriptor in
preparation for removal of this field.
Cc: Li Yang <leoli@freescale.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/fsldma.c | 7 ++++---
drivers/dma/fsldma.h | 1 +
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index f18d1bd..fb44eaa 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -337,7 +337,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
fsl_chan->common.cookie = cookie;
append_ld_queue(fsl_chan, tx_to_fsl_desc(tx));
- list_splice_init(&tx->tx_list, fsl_chan->ld_queue.prev);
+ list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev);
spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
@@ -359,6 +359,7 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
if (desc_sw) {
memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
+ INIT_LIST_HEAD(&desc_sw->tx_list);
dma_async_tx_descriptor_init(&desc_sw->async_tx,
&fsl_chan->common);
desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
@@ -448,7 +449,7 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
new->async_tx.flags = flags;
/* Insert the link descriptor to the LD ring */
- list_add_tail(&new->node, &new->async_tx.tx_list);
+ list_add_tail(&new->node, &new->tx_list);
/* Set End-of-link to the last link descriptor of new list*/
set_ld_eol(fsl_chan, new);
@@ -506,7 +507,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
dma_dest += copy;
/* Insert the link descriptor to the LD ring */
- list_add_tail(&new->node, &first->async_tx.tx_list);
+ list_add_tail(&new->node, &first->tx_list);
} while (len);
new->async_tx.flags = flags; /* client is in control of this ack */
diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h
index 4f21a51..02816e8 100644
--- a/drivers/dma/fsldma.h
+++ b/drivers/dma/fsldma.h
@@ -89,6 +89,7 @@ struct fsl_dma_ld_hw {
struct fsl_desc_sw {
struct fsl_dma_ld_hw hw;
struct list_head node;
+ struct list_head tx_list;
struct dma_async_tx_descriptor async_tx;
struct list_head *ld;
void *priv;
^ permalink raw reply related
* [PATCH 25/29] iop-adma: implement a private tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Drop iop-adma's use of tx_list from struct dma_async_tx_descriptor in
preparation for removal of this field.
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
arch/arm/include/asm/hardware/iop_adma.h | 2 ++
drivers/dma/iop-adma.c | 9 +++++----
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/arch/arm/include/asm/hardware/iop_adma.h b/arch/arm/include/asm/hardware/iop_adma.h
index 385c6e8..95dc133 100644
--- a/arch/arm/include/asm/hardware/iop_adma.h
+++ b/arch/arm/include/asm/hardware/iop_adma.h
@@ -86,6 +86,7 @@ struct iop_adma_chan {
* @idx: pool index
* @unmap_src_cnt: number of xor sources
* @unmap_len: transaction bytecount
+ * @tx_list: list of descriptors that are associated with one operation
* @async_tx: support for the async_tx api
* @group_list: list of slots that make up a multi-descriptor transaction
* for example transfer lengths larger than the supported hw max
@@ -102,6 +103,7 @@ struct iop_adma_desc_slot {
u16 idx;
u16 unmap_src_cnt;
size_t unmap_len;
+ struct list_head tx_list;
struct dma_async_tx_descriptor async_tx;
union {
u32 *xor_check_result;
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 2f05226..9f6c16f 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -370,7 +370,7 @@ retry:
}
alloc_tail->group_head = alloc_start;
alloc_tail->async_tx.cookie = -EBUSY;
- list_splice(&chain, &alloc_tail->async_tx.tx_list);
+ list_splice(&chain, &alloc_tail->tx_list);
iop_chan->last_used = last_used;
iop_desc_clear_next_desc(alloc_start);
iop_desc_clear_next_desc(alloc_tail);
@@ -429,7 +429,7 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
old_chain_tail = list_entry(iop_chan->chain.prev,
struct iop_adma_desc_slot, chain_node);
- list_splice_init(&sw_desc->async_tx.tx_list,
+ list_splice_init(&sw_desc->tx_list,
&old_chain_tail->chain_node);
/* fix up the hardware chain */
@@ -496,6 +496,7 @@ static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
dma_async_tx_descriptor_init(&slot->async_tx, chan);
slot->async_tx.tx_submit = iop_adma_tx_submit;
+ INIT_LIST_HEAD(&slot->tx_list);
INIT_LIST_HEAD(&slot->chain_node);
INIT_LIST_HEAD(&slot->slot_node);
hw_desc = (char *) iop_chan->device->dma_desc_pool;
@@ -1296,7 +1297,7 @@ static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
if (sw_desc) {
grp_start = sw_desc->group_head;
- list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
async_tx_ack(&sw_desc->async_tx);
iop_desc_init_memcpy(grp_start, 0);
iop_desc_set_byte_count(grp_start, iop_chan, 0);
@@ -1352,7 +1353,7 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
if (sw_desc) {
grp_start = sw_desc->group_head;
- list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
async_tx_ack(&sw_desc->async_tx);
iop_desc_init_null_xor(grp_start, 2, 0);
iop_desc_set_byte_count(grp_start, iop_chan, 0);
^ permalink raw reply related
* [PATCH 26/29] ioat: implement a private tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, Maciej Sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Drop ioatdma's use of tx_list from struct dma_async_tx_descriptor in
preparation for removal of this field.
Cc: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma.c | 7 ++++---
drivers/dma/ioat/dma.h | 3 ++-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 17a518d..21527b8 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -251,12 +251,12 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
/* write address into NextDescriptor field of last desc in chain */
- first = to_ioat_desc(tx->tx_list.next);
+ first = to_ioat_desc(desc->tx_list.next);
chain_tail = to_ioat_desc(ioat->used_desc.prev);
/* make descriptor updates globally visible before chaining */
wmb();
chain_tail->hw->next = first->txd.phys;
- list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
+ list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
dump_desc_dbg(ioat, chain_tail);
dump_desc_dbg(ioat, first);
@@ -297,6 +297,7 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
memset(desc, 0, sizeof(*desc));
+ INIT_LIST_HEAD(&desc_sw->tx_list);
dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
desc_sw->txd.tx_submit = ioat1_tx_submit;
desc_sw->hw = desc;
@@ -521,7 +522,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
desc->txd.flags = flags;
desc->len = total_len;
- list_splice(&chain, &desc->txd.tx_list);
+ list_splice(&chain, &desc->tx_list);
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1;
hw->tx_cnt = tx_cnt;
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index d9d6a7e..8966fa5 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -157,7 +157,7 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
* struct ioat_desc_sw - wrapper around hardware descriptor
* @hw: hardware DMA descriptor
* @node: this descriptor will either be on the free list,
- * or attached to a transaction list (async_tx.tx_list)
+ * or attached to a transaction list (tx_list)
* @txd: the generic software descriptor for all engines
* @id: identifier for debug
*/
@@ -165,6 +165,7 @@ struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
size_t len;
+ struct list_head tx_list;
struct dma_async_tx_descriptor txd;
#ifdef DEBUG
int id;
^ permalink raw reply related
* [PATCH 27/29] mv_xor: implement a private tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski, Saeed Bishara
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
Drop mv_xor's use of tx_list from struct dma_async_tx_descriptor in
preparation for removal of this field.
Cc: Saeed Bishara <saeed@marvell.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/mv_xor.c | 7 ++++---
drivers/dma/mv_xor.h | 4 ++--
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index ddab94f..d7277ac 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -517,7 +517,7 @@ retry:
}
alloc_tail->group_head = alloc_start;
alloc_tail->async_tx.cookie = -EBUSY;
- list_splice(&chain, &alloc_tail->async_tx.tx_list);
+ list_splice(&chain, &alloc_tail->tx_list);
mv_chan->last_used = last_used;
mv_desc_clear_next_desc(alloc_start);
mv_desc_clear_next_desc(alloc_tail);
@@ -565,14 +565,14 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
cookie = mv_desc_assign_cookie(mv_chan, sw_desc);
if (list_empty(&mv_chan->chain))
- list_splice_init(&sw_desc->async_tx.tx_list, &mv_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &mv_chan->chain);
else {
new_hw_chain = 0;
old_chain_tail = list_entry(mv_chan->chain.prev,
struct mv_xor_desc_slot,
chain_node);
- list_splice_init(&grp_start->async_tx.tx_list,
+ list_splice_init(&grp_start->tx_list,
&old_chain_tail->chain_node);
if (!mv_can_chain(grp_start))
@@ -632,6 +632,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
slot->async_tx.tx_submit = mv_xor_tx_submit;
INIT_LIST_HEAD(&slot->chain_node);
INIT_LIST_HEAD(&slot->slot_node);
+ INIT_LIST_HEAD(&slot->tx_list);
hw_desc = (char *) mv_chan->device->dma_desc_pool;
slot->async_tx.phys =
(dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE];
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06cafe1..977b592 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -126,9 +126,8 @@ struct mv_xor_chan {
* @idx: pool index
* @unmap_src_cnt: number of xor sources
* @unmap_len: transaction bytecount
+ * @tx_list: list of slots that make up a multi-descriptor transaction
* @async_tx: support for the async_tx api
- * @group_list: list of slots that make up a multi-descriptor transaction
- * for example transfer lengths larger than the supported hw max
* @xor_check_result: result of zero sum
* @crc32_result: result crc calculation
*/
@@ -145,6 +144,7 @@ struct mv_xor_desc_slot {
u16 unmap_src_cnt;
u32 value;
size_t unmap_len;
+ struct list_head tx_list;
struct dma_async_tx_descriptor async_tx;
union {
u32 *xor_check_result;
^ permalink raw reply related
* [PATCH 28/29] dmaengine: kill tx_list
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
The tx_list attribute of struct dma_async_tx_descriptor is common to
most, but not all dma driver implementations. None of the upper level
code (dmaengine/async_tx) uses it, so allow drivers to implement it
locally if they need it. This saves sizeof(struct list_head) bytes for
drivers that do not manage descriptors with a linked list (e.g.: ioatdma
v2,3).
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/dmaengine.c | 1 -
include/linux/dmaengine.h | 3 ---
2 files changed, 0 insertions(+), 4 deletions(-)
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 5a87384..562d182 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -933,7 +933,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
{
tx->chan = chan;
spin_lock_init(&tx->lock);
- INIT_LIST_HEAD(&tx->tx_list);
}
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index ffefba8..f114bc7 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -180,8 +180,6 @@ typedef void (*dma_async_tx_callback)(void *dma_async_param);
* @flags: flags to augment operation preparation, control completion, and
* communicate status
* @phys: physical address of the descriptor
- * @tx_list: driver common field for operations that require multiple
- * descriptors
* @chan: target channel for this operation
* @tx_submit: set the prepared descriptor(s) to be executed by the engine
* @callback: routine to call after this operation is complete
@@ -195,7 +193,6 @@ struct dma_async_tx_descriptor {
dma_cookie_t cookie;
enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
dma_addr_t phys;
- struct list_head tx_list;
struct dma_chan *chan;
dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
dma_async_tx_callback callback;
^ permalink raw reply related
* [PATCH 29/29] ioat2, 3: cacheline align software descriptor allocations
From: Dan Williams @ 2009-09-04 2:32 UTC (permalink / raw)
To: linux-kernel; +Cc: linux-raid, netdev, maciej.sosnowski
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
All the necessary fields for handling an ioat2,3 ring entry can fit into
one cacheline. Move ->len prior to ->txd in struct ioat_ring_ent, and
move allocation of these entries to a hw-cache-aligned kmem cache to
reduce the number of cachelines dirtied for descriptor management.
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma_v2.c | 5 +++--
drivers/dma/ioat/dma_v2.h | 3 ++-
drivers/dma/ioat/pci.c | 16 +++++++++++++++-
3 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index 9ea1a47..f0c4876 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -400,11 +400,12 @@ static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t f
return NULL;
memset(hw, 0, sizeof(*hw));
- desc = kzalloc(sizeof(*desc), flags);
+ desc = kmem_cache_alloc(ioat2_cache, flags);
if (!desc) {
pci_pool_free(dma->dma_pool, hw, phys);
return NULL;
}
+ memset(desc, 0, sizeof(*desc));
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = ioat2_tx_submit_unlock;
@@ -419,7 +420,7 @@ static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *cha
dma = to_ioatdma_device(chan->device);
pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
- kfree(desc);
+ kmem_cache_free(ioat2_cache, desc);
}
static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 9baa3d6..ac00adc 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -116,8 +116,8 @@ static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len
struct ioat_ring_ent {
struct ioat_dma_descriptor *hw;
- struct dma_async_tx_descriptor txd;
size_t len;
+ struct dma_async_tx_descriptor txd;
#ifdef DEBUG
int id;
#endif
@@ -143,4 +143,5 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+extern struct kmem_cache *ioat2_cache;
#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index c4e4322..61086c6 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -69,6 +69,8 @@ static int ioat_dca_enabled = 1;
module_param(ioat_dca_enabled, int, 0644);
MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
+struct kmem_cache *ioat2_cache;
+
#define DRV_NAME "ioatdma"
static struct pci_driver ioat_pci_driver = {
@@ -168,12 +170,24 @@ static void __devexit ioat_remove(struct pci_dev *pdev)
static int __init ioat_init_module(void)
{
- return pci_register_driver(&ioat_pci_driver);
+ int err;
+
+ ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
+ 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!ioat2_cache)
+ return -ENOMEM;
+
+ err = pci_register_driver(&ioat_pci_driver);
+ if (err)
+ kmem_cache_destroy(ioat2_cache);
+
+ return err;
}
module_init(ioat_init_module);
static void __exit ioat_exit_module(void)
{
pci_unregister_driver(&ioat_pci_driver);
+ kmem_cache_destroy(ioat2_cache);
}
module_exit(ioat_exit_module);
^ permalink raw reply related
* [net-next PATCH 00/11] enic: updates
From: Scott Feldman @ 2009-09-04 3:01 UTC (permalink / raw)
To: davem; +Cc: netdev
The following series implements updates to the enic netdev driver for
Cisco's 10G Ethernet NIC:
01 enic: add support for multiple BARs
02 enic: workaround A0 erratum
03 enic: bug fix: split TSO fragments larger than 16K into
multiple descs
04 enic: use netdev_alloc_skb
05 enic: bug fix: protect fw call i/f with spinlock
06 enic: bug fix: included MAC drops in rx_dropped netstat
07 enic: provision for multiple Rx/Tx queues; prepare for RSS support
08 enic: bug fix: enable VLAN filtering
09 enic: changes to driver/firmware interface
10 enic: bug fix: check for zero port MTU before posting warning
11 enic: organize device initialization/deinit into separate functions
Please apply.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
^ permalink raw reply
* [net-next PATCH 01/11] enic: add support for multiple BARs
From: Scott Feldman @ 2009-09-04 3:01 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: add support for multiple BARs
Nic firmware can place resources (queues, intrs, etc) on multiple BARs, so
allow driver to discover/map resources beyond BAR0.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic.h | 8 ++++---
drivers/net/enic/enic_main.c | 48 +++++++++++++++++++++---------------------
drivers/net/enic/vnic_dev.c | 41 +++++++++++++++++++++++++++++-------
drivers/net/enic/vnic_dev.h | 5 ++++
4 files changed, 66 insertions(+), 36 deletions(-)
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index c26cea0..cfe94b2 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -33,13 +33,15 @@
#define DRV_NAME "enic"
#define DRV_DESCRIPTION "Cisco 10G Ethernet Driver"
-#define DRV_VERSION "1.0.0.933"
-#define DRV_COPYRIGHT "Copyright 2008 Cisco Systems, Inc"
+#define DRV_VERSION "1.1.0.100"
+#define DRV_COPYRIGHT "Copyright 2008-2009 Cisco Systems, Inc"
#define PFX DRV_NAME ": "
#define ENIC_LRO_MAX_DESC 8
#define ENIC_LRO_MAX_AGGR 64
+#define ENIC_BARS_MAX 6
+
enum enic_cq_index {
ENIC_CQ_RQ,
ENIC_CQ_WQ,
@@ -73,7 +75,7 @@ struct enic {
struct net_device *netdev;
struct pci_dev *pdev;
struct vnic_enet_config config;
- struct vnic_dev_bar bar0;
+ struct vnic_dev_bar bar[ENIC_BARS_MAX];
struct vnic_dev *vdev;
struct timer_list notify_timer;
struct work_struct reset;
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 8005b60..b17fe4b 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -1608,12 +1608,6 @@ static void enic_clear_intr_mode(struct enic *enic)
vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN);
}
-static void enic_iounmap(struct enic *enic)
-{
- if (enic->bar0.vaddr)
- iounmap(enic->bar0.vaddr);
-}
-
static const struct net_device_ops enic_netdev_ops = {
.ndo_open = enic_open,
.ndo_stop = enic_stop,
@@ -1632,6 +1626,15 @@ static const struct net_device_ops enic_netdev_ops = {
#endif
};
+static void enic_iounmap(struct enic *enic)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(enic->bar); i++)
+ if (enic->bar[i].vaddr)
+ iounmap(enic->bar[i].vaddr);
+}
+
static int __devinit enic_probe(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
@@ -1709,31 +1712,28 @@ static int __devinit enic_probe(struct pci_dev *pdev,
using_dac = 1;
}
- /* Map vNIC resources from BAR0
+ /* Map vNIC resources from BAR0-5
*/
- if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
- printk(KERN_ERR PFX
- "BAR0 not memory-map'able, aborting.\n");
- err = -ENODEV;
- goto err_out_release_regions;
- }
-
- enic->bar0.vaddr = pci_iomap(pdev, 0, enic->bar0.len);
- enic->bar0.bus_addr = pci_resource_start(pdev, 0);
- enic->bar0.len = pci_resource_len(pdev, 0);
-
- if (!enic->bar0.vaddr) {
- printk(KERN_ERR PFX
- "Cannot memory-map BAR0 res hdr, aborting.\n");
- err = -ENODEV;
- goto err_out_release_regions;
+ for (i = 0; i < ARRAY_SIZE(enic->bar); i++) {
+ if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
+ continue;
+ enic->bar[i].len = pci_resource_len(pdev, i);
+ enic->bar[i].vaddr = pci_iomap(pdev, i, enic->bar[i].len);
+ if (!enic->bar[i].vaddr) {
+ printk(KERN_ERR PFX
+ "Cannot memory-map BAR %d, aborting.\n", i);
+ err = -ENODEV;
+ goto err_out_iounmap;
+ }
+ enic->bar[i].bus_addr = pci_resource_start(pdev, i);
}
/* Register vNIC device
*/
- enic->vdev = vnic_dev_register(NULL, enic, pdev, &enic->bar0);
+ enic->vdev = vnic_dev_register(NULL, enic, pdev, enic->bar,
+ ARRAY_SIZE(enic->bar));
if (!enic->vdev) {
printk(KERN_ERR PFX
"vNIC registration failed, aborting.\n");
diff --git a/drivers/net/enic/vnic_dev.c b/drivers/net/enic/vnic_dev.c
index e21b9d6..d5c28ef 100644
--- a/drivers/net/enic/vnic_dev.c
+++ b/drivers/net/enic/vnic_dev.c
@@ -31,6 +31,7 @@
struct vnic_res {
void __iomem *vaddr;
+ dma_addr_t bus_addr;
unsigned int count;
};
@@ -67,12 +68,15 @@ void *vnic_dev_priv(struct vnic_dev *vdev)
}
static int vnic_dev_discover_res(struct vnic_dev *vdev,
- struct vnic_dev_bar *bar)
+ struct vnic_dev_bar *bar, unsigned int num_bars)
{
struct vnic_resource_header __iomem *rh;
struct vnic_resource __iomem *r;
u8 type;
+ if (num_bars == 0)
+ return -EINVAL;
+
if (bar->len < VNIC_MAX_RES_HDR_SIZE) {
printk(KERN_ERR "vNIC BAR0 res hdr length error\n");
return -EINVAL;
@@ -104,7 +108,10 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev,
r++;
- if (bar_num != 0) /* only mapping in BAR0 resources */
+ if (bar_num >= num_bars)
+ continue;
+
+ if (!bar[bar_num].len || !bar[bar_num].vaddr)
continue;
switch (type) {
@@ -114,13 +121,13 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev,
case RES_TYPE_INTR_CTRL:
/* each count is stride bytes long */
len = count * VNIC_RES_STRIDE;
- if (len + bar_offset > bar->len) {
+ if (len + bar_offset > bar[bar_num].len) {
printk(KERN_ERR "vNIC BAR0 resource %d "
"out-of-bounds, offset 0x%x + "
"size 0x%x > bar len 0x%lx\n",
type, bar_offset,
len,
- bar->len);
+ bar[bar_num].len);
return -EINVAL;
}
break;
@@ -133,7 +140,9 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev,
}
vdev->res[type].count = count;
- vdev->res[type].vaddr = (char __iomem *)bar->vaddr + bar_offset;
+ vdev->res[type].vaddr = (char __iomem *)bar[bar_num].vaddr +
+ bar_offset;
+ vdev->res[type].bus_addr = bar[bar_num].bus_addr + bar_offset;
}
return 0;
@@ -163,6 +172,21 @@ void __iomem *vnic_dev_get_res(struct vnic_dev *vdev, enum vnic_res_type type,
}
}
+dma_addr_t vnic_dev_get_res_bus_addr(struct vnic_dev *vdev,
+ enum vnic_res_type type, unsigned int index)
+{
+ switch (type) {
+ case RES_TYPE_WQ:
+ case RES_TYPE_RQ:
+ case RES_TYPE_CQ:
+ case RES_TYPE_INTR_CTRL:
+ return vdev->res[type].bus_addr +
+ index * VNIC_RES_STRIDE;
+ default:
+ return vdev->res[type].bus_addr;
+ }
+}
+
unsigned int vnic_dev_desc_ring_size(struct vnic_dev_ring *ring,
unsigned int desc_count, unsigned int desc_size)
{
@@ -257,7 +281,7 @@ int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
iowrite32(cmd, &devcmd->cmd);
if ((_CMD_FLAGS(cmd) & _CMD_FLAGS_NOWAIT))
- return 0;
+ return 0;
for (delay = 0; delay < wait; delay++) {
@@ -684,7 +708,8 @@ void vnic_dev_unregister(struct vnic_dev *vdev)
}
struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev,
- void *priv, struct pci_dev *pdev, struct vnic_dev_bar *bar)
+ void *priv, struct pci_dev *pdev, struct vnic_dev_bar *bar,
+ unsigned int num_bars)
{
if (!vdev) {
vdev = kzalloc(sizeof(struct vnic_dev), GFP_ATOMIC);
@@ -695,7 +720,7 @@ struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev,
vdev->priv = priv;
vdev->pdev = pdev;
- if (vnic_dev_discover_res(vdev, bar))
+ if (vnic_dev_discover_res(vdev, bar, num_bars))
goto err_out;
vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0);
diff --git a/drivers/net/enic/vnic_dev.h b/drivers/net/enic/vnic_dev.h
index 8aa8db2..d960edb 100644
--- a/drivers/net/enic/vnic_dev.h
+++ b/drivers/net/enic/vnic_dev.h
@@ -75,6 +75,8 @@ unsigned int vnic_dev_get_res_count(struct vnic_dev *vdev,
enum vnic_res_type type);
void __iomem *vnic_dev_get_res(struct vnic_dev *vdev, enum vnic_res_type type,
unsigned int index);
+dma_addr_t vnic_dev_get_res_bus_addr(struct vnic_dev *vdev,
+ enum vnic_res_type type, unsigned int index);
unsigned int vnic_dev_desc_ring_size(struct vnic_dev_ring *ring,
unsigned int desc_count, unsigned int desc_size);
void vnic_dev_clear_desc_ring(struct vnic_dev_ring *ring);
@@ -117,6 +119,7 @@ void vnic_dev_set_intr_mode(struct vnic_dev *vdev,
enum vnic_dev_intr_mode vnic_dev_get_intr_mode(struct vnic_dev *vdev);
void vnic_dev_unregister(struct vnic_dev *vdev);
struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev,
- void *priv, struct pci_dev *pdev, struct vnic_dev_bar *bar);
+ void *priv, struct pci_dev *pdev, struct vnic_dev_bar *bar,
+ unsigned int num_bars);
#endif /* _VNIC_DEV_H_ */
^ permalink raw reply related
* [net-next PATCH 02/11] enic: workaround A0 erratum
From: Scott Feldman @ 2009-09-04 3:01 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: workaround A0 erratum
A0 revision ASIC has an erratum on the RQ desc cache on chip where the
cache can become corrupted causing pkt buf writes to wrong locations. The s/w
workaround is to post a dummy RQ desc in the ring every 32 descs, causing a
flush of the cache. A0 parts are not production, but there are enough of
these parts in the wild in test setups to warrant including workaround. A1
revision ASIC parts fix erratum.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic_main.c | 50 +++++++++++++++++++++++++++++++++++++++---
drivers/net/enic/vnic_dev.c | 19 ++++++++++++++++
drivers/net/enic/vnic_dev.h | 8 +++++++
drivers/net/enic/vnic_rq.h | 7 +++++-
4 files changed, 80 insertions(+), 4 deletions(-)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index b17fe4b..3d7838d 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -850,6 +850,50 @@ static int enic_rq_alloc_buf(struct vnic_rq *rq)
return 0;
}
+static int enic_rq_alloc_buf_a1(struct vnic_rq *rq)
+{
+ struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
+
+ if (vnic_rq_posting_soon(rq)) {
+
+ /* SW workaround for A0 HW erratum: if we're just about
+ * to write posted_index, insert a dummy desc
+ * of type resvd
+ */
+
+ rq_enet_desc_enc(desc, 0, RQ_ENET_TYPE_RESV2, 0);
+ vnic_rq_post(rq, 0, 0, 0, 0);
+ } else {
+ return enic_rq_alloc_buf(rq);
+ }
+
+ return 0;
+}
+
+static int enic_set_rq_alloc_buf(struct enic *enic)
+{
+ enum vnic_dev_hw_version hw_ver;
+ int err;
+
+ err = vnic_dev_hw_version(enic->vdev, &hw_ver);
+ if (err)
+ return err;
+
+ switch (hw_ver) {
+ case VNIC_DEV_HW_VER_A1:
+ enic->rq_alloc_buf = enic_rq_alloc_buf_a1;
+ break;
+ case VNIC_DEV_HW_VER_A2:
+ case VNIC_DEV_HW_VER_UNKNOWN:
+ enic->rq_alloc_buf = enic_rq_alloc_buf;
+ break;
+ default:
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
static int enic_get_skb_header(struct sk_buff *skb, void **iphdr,
void **tcph, u64 *hdr_flags, void *priv)
{
@@ -1057,7 +1101,7 @@ static int enic_poll(struct napi_struct *napi, int budget)
/* Replenish RQ
*/
- vnic_rq_fill(&enic->rq[0], enic_rq_alloc_buf);
+ vnic_rq_fill(&enic->rq[0], enic->rq_alloc_buf);
} else {
@@ -1092,7 +1136,7 @@ static int enic_poll_msix(struct napi_struct *napi, int budget)
/* Replenish RQ
*/
- vnic_rq_fill(&enic->rq[0], enic_rq_alloc_buf);
+ vnic_rq_fill(&enic->rq[0], enic->rq_alloc_buf);
/* Return intr event credits for this polling
* cycle. An intr event is the completion of a
@@ -1268,7 +1312,7 @@ static int enic_open(struct net_device *netdev)
}
for (i = 0; i < enic->rq_count; i++) {
- err = vnic_rq_fill(&enic->rq[i], enic_rq_alloc_buf);
+ err = vnic_rq_fill(&enic->rq[i], enic->rq_alloc_buf);
if (err) {
printk(KERN_ERR PFX
"%s: Unable to alloc receive buffers.\n",
diff --git a/drivers/net/enic/vnic_dev.c b/drivers/net/enic/vnic_dev.c
index d5c28ef..c8d3fc7 100644
--- a/drivers/net/enic/vnic_dev.c
+++ b/drivers/net/enic/vnic_dev.c
@@ -349,6 +349,25 @@ int vnic_dev_fw_info(struct vnic_dev *vdev,
return err;
}
+int vnic_dev_hw_version(struct vnic_dev *vdev, enum vnic_dev_hw_version *hw_ver)
+{
+ struct vnic_devcmd_fw_info *fw_info;
+ int err;
+
+ err = vnic_dev_fw_info(vdev, &fw_info);
+ if (err)
+ return err;
+
+ if (strncmp(fw_info->hw_version, "A1", sizeof("A1")) == 0)
+ *hw_ver = VNIC_DEV_HW_VER_A1;
+ else if (strncmp(fw_info->hw_version, "A2", sizeof("A2")) == 0)
+ *hw_ver = VNIC_DEV_HW_VER_A2;
+ else
+ *hw_ver = VNIC_DEV_HW_VER_UNKNOWN;
+
+ return 0;
+}
+
int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, unsigned int size,
void *value)
{
diff --git a/drivers/net/enic/vnic_dev.h b/drivers/net/enic/vnic_dev.h
index d960edb..db1d63e 100644
--- a/drivers/net/enic/vnic_dev.h
+++ b/drivers/net/enic/vnic_dev.h
@@ -41,6 +41,12 @@ static inline void writeq(u64 val, void __iomem *reg)
}
#endif
+enum vnic_dev_hw_version {
+ VNIC_DEV_HW_VER_UNKNOWN,
+ VNIC_DEV_HW_VER_A1,
+ VNIC_DEV_HW_VER_A2,
+};
+
enum vnic_dev_intr_mode {
VNIC_DEV_INTR_MODE_UNKNOWN,
VNIC_DEV_INTR_MODE_INTX,
@@ -88,6 +94,8 @@ int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
u64 *a0, u64 *a1, int wait);
int vnic_dev_fw_info(struct vnic_dev *vdev,
struct vnic_devcmd_fw_info **fw_info);
+int vnic_dev_hw_version(struct vnic_dev *vdev,
+ enum vnic_dev_hw_version *hw_ver);
int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, unsigned int size,
void *value);
int vnic_dev_stats_clear(struct vnic_dev *vdev);
diff --git a/drivers/net/enic/vnic_rq.h b/drivers/net/enic/vnic_rq.h
index fd0ef66..f7b5730 100644
--- a/drivers/net/enic/vnic_rq.h
+++ b/drivers/net/enic/vnic_rq.h
@@ -143,6 +143,11 @@ static inline void vnic_rq_post(struct vnic_rq *rq,
}
}
+static inline int vnic_rq_posting_soon(struct vnic_rq *rq)
+{
+ return ((rq->to_use->index & VNIC_RQ_RETURN_RATE) == 0);
+}
+
static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
{
rq->ring.desc_avail += count;
@@ -186,7 +191,7 @@ static inline int vnic_rq_fill(struct vnic_rq *rq,
{
int err;
- while (vnic_rq_desc_avail(rq) > 1) {
+ while (vnic_rq_desc_avail(rq) > 0) {
err = (*buf_fill)(rq);
if (err)
^ permalink raw reply related
* [net-next PATCH 05/11] enic: bug fix: protect fw call i/f with spinlock
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: bug fix: protect fw call i/f with spinlock
Some driver -> nic firmware calls weren't guarded with a spinlock, exposing
the call i/f to a race between two threads
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic_main.c | 12 ++++++++++++
1 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index d77119a..8d54deb 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -1315,6 +1315,7 @@ static int enic_notify_set(struct enic *enic)
{
int err;
+ spin_lock(&enic->devcmd_lock);
switch (vnic_dev_get_intr_mode(enic->vdev)) {
case VNIC_DEV_INTR_MODE_INTX:
err = vnic_dev_notify_set(enic->vdev, ENIC_INTX_NOTIFY);
@@ -1326,6 +1327,7 @@ static int enic_notify_set(struct enic *enic)
err = vnic_dev_notify_set(enic->vdev, -1 /* no intr */);
break;
}
+ spin_unlock(&enic->devcmd_lock);
return err;
}
@@ -1379,12 +1381,16 @@ static int enic_open(struct net_device *netdev)
for (i = 0; i < enic->rq_count; i++)
vnic_rq_enable(&enic->rq[i]);
+ spin_lock(&enic->devcmd_lock);
enic_add_station_addr(enic);
+ spin_unlock(&enic->devcmd_lock);
enic_set_multicast_list(netdev);
netif_wake_queue(netdev);
napi_enable(&enic->napi);
+ spin_lock(&enic->devcmd_lock);
vnic_dev_enable(enic->vdev);
+ spin_unlock(&enic->devcmd_lock);
for (i = 0; i < enic->intr_count; i++)
vnic_intr_unmask(&enic->intr[i]);
@@ -1394,7 +1400,9 @@ static int enic_open(struct net_device *netdev)
return 0;
err_out_notify_unset:
+ spin_lock(&enic->devcmd_lock);
vnic_dev_notify_unset(enic->vdev);
+ spin_unlock(&enic->devcmd_lock);
err_out_free_intr:
enic_free_intr(enic);
@@ -1410,7 +1418,9 @@ static int enic_stop(struct net_device *netdev)
del_timer_sync(&enic->notify_timer);
+ spin_lock(&enic->devcmd_lock);
vnic_dev_disable(enic->vdev);
+ spin_unlock(&enic->devcmd_lock);
napi_disable(&enic->napi);
netif_stop_queue(netdev);
@@ -1428,7 +1438,9 @@ static int enic_stop(struct net_device *netdev)
return err;
}
+ spin_lock(&enic->devcmd_lock);
vnic_dev_notify_unset(enic->vdev);
+ spin_unlock(&enic->devcmd_lock);
enic_free_intr(enic);
(void)vnic_cq_service(&enic->cq[ENIC_CQ_RQ],
^ permalink raw reply related
* [net-next PATCH 06/11] enic: bug fix: included MAC drops in rx_dropped netstat
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: bug fix: included MAC drops in rx_dropped netstat
Bug fix: included MAC drops in rx_dropped netstat. Also track Rx trunctations
stat at the MAC
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic.h | 1 +
drivers/net/enic/enic_main.c | 11 ++++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index cfe94b2..f7c5b33 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -99,6 +99,7 @@ struct enic {
____cacheline_aligned struct vnic_rq rq[1];
unsigned int rq_count;
int (*rq_alloc_buf)(struct vnic_rq *rq);
+ u64 rq_truncated_pkts;
u64 rq_bad_fcs;
struct napi_struct napi;
struct net_lro_mgr lro_mgr;
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 8d54deb..139c380 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -737,8 +737,9 @@ static struct net_device_stats *enic_get_stats(struct net_device *netdev)
net_stats->rx_bytes = stats->rx.rx_bytes_ok;
net_stats->rx_errors = stats->rx.rx_errors;
net_stats->multicast = stats->rx.rx_multicast_frames_ok;
+ net_stats->rx_over_errors = enic->rq_truncated_pkts;
net_stats->rx_crc_errors = enic->rq_bad_fcs;
- net_stats->rx_dropped = stats->rx.rx_no_bufs;
+ net_stats->rx_dropped = stats->rx.rx_no_bufs + stats->rx.rx_drop;
return net_stats;
}
@@ -1028,8 +1029,12 @@ static void enic_rq_indicate_buf(struct vnic_rq *rq,
if (packet_error) {
- if (bytes_written > 0 && !fcs_ok)
- enic->rq_bad_fcs++;
+ if (!fcs_ok) {
+ if (bytes_written > 0)
+ enic->rq_bad_fcs++;
+ else if (bytes_written == 0)
+ enic->rq_truncated_pkts++;
+ }
dev_kfree_skb_any(skb);
^ permalink raw reply related
* [net-next PATCH 07/11] enic: provision for multiple Rx/Tx queues; prepare for RSS support
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: provision for multiple Rx/Tx queues; prepare for RSS support
Provision for multiple Rx/Tx queues. Max of 8 WQs and 8 RQs. Max for
completion queue is 8+8=16 and max for interrupt resources is 8+8+2.
Add driver/firmware interface for setting up RSS secret key and indirection
table.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic.h | 16 ++++++++++------
drivers/net/enic/enic_main.c | 6 +++---
drivers/net/enic/enic_res.c | 33 ++++++++++++++++++++++++++++-----
drivers/net/enic/enic_res.h | 2 ++
drivers/net/enic/vnic_nic.h | 7 +++++++
5 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index f7c5b33..e1c2076 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -29,6 +29,7 @@
#include "vnic_cq.h"
#include "vnic_intr.h"
#include "vnic_stats.h"
+#include "vnic_nic.h"
#include "vnic_rss.h"
#define DRV_NAME "enic"
@@ -42,17 +43,20 @@
#define ENIC_BARS_MAX 6
+#define ENIC_WQ_MAX 8
+#define ENIC_RQ_MAX 8
+#define ENIC_CQ_MAX (ENIC_WQ_MAX + ENIC_RQ_MAX)
+#define ENIC_INTR_MAX (ENIC_CQ_MAX + 2)
+
enum enic_cq_index {
ENIC_CQ_RQ,
ENIC_CQ_WQ,
- ENIC_CQ_MAX,
};
enum enic_intx_intr_index {
ENIC_INTX_WQ_RQ,
ENIC_INTX_ERR,
ENIC_INTX_NOTIFY,
- ENIC_INTX_MAX,
};
enum enic_msix_intr_index {
@@ -90,13 +94,13 @@ struct enic {
u32 port_mtu;
/* work queue cache line section */
- ____cacheline_aligned struct vnic_wq wq[1];
- spinlock_t wq_lock[1];
+ ____cacheline_aligned struct vnic_wq wq[ENIC_WQ_MAX];
+ spinlock_t wq_lock[ENIC_WQ_MAX];
unsigned int wq_count;
struct vlan_group *vlan_group;
/* receive queue cache line section */
- ____cacheline_aligned struct vnic_rq rq[1];
+ ____cacheline_aligned struct vnic_rq rq[ENIC_RQ_MAX];
unsigned int rq_count;
int (*rq_alloc_buf)(struct vnic_rq *rq);
u64 rq_truncated_pkts;
@@ -106,7 +110,7 @@ struct enic {
struct net_lro_desc lro_desc[ENIC_LRO_MAX_DESC];
/* interrupt resource cache line section */
- ____cacheline_aligned struct vnic_intr intr[ENIC_MSIX_MAX];
+ ____cacheline_aligned struct vnic_intr intr[ENIC_INTR_MAX];
unsigned int intr_count;
u32 __iomem *legacy_pba; /* memory-mapped */
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 139c380..6068904 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -1585,7 +1585,7 @@ static int enic_set_niccfg(struct enic *enic)
const u8 ig_vlan_strip_en = 1;
/* Enable VLAN tag stripping. RSS not enabled (yet).
- */
+ */
return enic_set_nic_cfg(enic,
rss_default_cpu, rss_hash_type,
@@ -1620,8 +1620,8 @@ static void enic_reset(struct work_struct *work)
static int enic_set_intr_mode(struct enic *enic)
{
- unsigned int n = ARRAY_SIZE(enic->rq);
- unsigned int m = ARRAY_SIZE(enic->wq);
+ unsigned int n = 1;
+ unsigned int m = 1;
unsigned int i;
/* Set interrupt mode (INTx, MSI, MSI-X) depending
diff --git a/drivers/net/enic/enic_res.c b/drivers/net/enic/enic_res.c
index e5fc938..3211114 100644
--- a/drivers/net/enic/enic_res.c
+++ b/drivers/net/enic/enic_res.c
@@ -156,6 +156,22 @@ int enic_set_nic_cfg(struct enic *enic, u8 rss_default_cpu, u8 rss_hash_type,
return vnic_dev_cmd(enic->vdev, CMD_NIC_CFG, &a0, &a1, wait);
}
+int enic_set_rss_key(struct enic *enic, dma_addr_t key_pa, u64 len)
+{
+ u64 a0 = (u64)key_pa, a1 = len;
+ int wait = 1000;
+
+ return vnic_dev_cmd(enic->vdev, CMD_RSS_KEY, &a0, &a1, wait);
+}
+
+int enic_set_rss_cpu(struct enic *enic, dma_addr_t cpu_pa, u64 len)
+{
+ u64 a0 = (u64)cpu_pa, a1 = len;
+ int wait = 1000;
+
+ return vnic_dev_cmd(enic->vdev, CMD_RSS_CPU, &a0, &a1, wait);
+}
+
void enic_free_vnic_resources(struct enic *enic)
{
unsigned int i;
@@ -172,11 +188,18 @@ void enic_free_vnic_resources(struct enic *enic)
void enic_get_res_counts(struct enic *enic)
{
- enic->wq_count = vnic_dev_get_res_count(enic->vdev, RES_TYPE_WQ);
- enic->rq_count = vnic_dev_get_res_count(enic->vdev, RES_TYPE_RQ);
- enic->cq_count = vnic_dev_get_res_count(enic->vdev, RES_TYPE_CQ);
- enic->intr_count = vnic_dev_get_res_count(enic->vdev,
- RES_TYPE_INTR_CTRL);
+ enic->wq_count = min_t(int,
+ vnic_dev_get_res_count(enic->vdev, RES_TYPE_WQ),
+ ENIC_WQ_MAX);
+ enic->rq_count = min_t(int,
+ vnic_dev_get_res_count(enic->vdev, RES_TYPE_RQ),
+ ENIC_RQ_MAX);
+ enic->cq_count = min_t(int,
+ vnic_dev_get_res_count(enic->vdev, RES_TYPE_CQ),
+ ENIC_CQ_MAX);
+ enic->intr_count = min_t(int,
+ vnic_dev_get_res_count(enic->vdev, RES_TYPE_INTR_CTRL),
+ ENIC_INTR_MAX);
printk(KERN_INFO PFX "vNIC resources avail: "
"wq %d rq %d cq %d intr %d\n",
diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
index 7bf272f..abc1974 100644
--- a/drivers/net/enic/enic_res.h
+++ b/drivers/net/enic/enic_res.h
@@ -139,6 +139,8 @@ void enic_del_vlan(struct enic *enic, u16 vlanid);
int enic_set_nic_cfg(struct enic *enic, u8 rss_default_cpu, u8 rss_hash_type,
u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable, u8 tso_ipid_split_en,
u8 ig_vlan_strip_en);
+int enic_set_rss_key(struct enic *enic, dma_addr_t key_pa, u64 len);
+int enic_set_rss_cpu(struct enic *enic, dma_addr_t cpu_pa, u64 len);
void enic_get_res_counts(struct enic *enic);
void enic_init_vnic_resources(struct enic *enic);
int enic_alloc_vnic_resources(struct enic *);
diff --git a/drivers/net/enic/vnic_nic.h b/drivers/net/enic/vnic_nic.h
index dadf26f..eeaf329 100644
--- a/drivers/net/enic/vnic_nic.h
+++ b/drivers/net/enic/vnic_nic.h
@@ -41,6 +41,13 @@
#define NIC_CFG_IG_VLAN_STRIP_EN_MASK_FIELD 1UL
#define NIC_CFG_IG_VLAN_STRIP_EN_SHIFT 24
+#define NIC_CFG_RSS_HASH_TYPE_IPV4 (1 << 0)
+#define NIC_CFG_RSS_HASH_TYPE_TCP_IPV4 (1 << 1)
+#define NIC_CFG_RSS_HASH_TYPE_IPV6 (1 << 2)
+#define NIC_CFG_RSS_HASH_TYPE_TCP_IPV6 (1 << 3)
+#define NIC_CFG_RSS_HASH_TYPE_IPV6_EX (1 << 4)
+#define NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX (1 << 5)
+
static inline void vnic_set_nic_cfg(u32 *nic_cfg,
u8 rss_default_cpu, u8 rss_hash_type,
u8 rss_hash_bits, u8 rss_base_cpu,
^ permalink raw reply related
* [net-next PATCH 08/11] enic: bug fix: enable VLAN filtering
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: bug fix: enable VLAN filtering
Bug fix: enable VLAN filtering
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic_main.c | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 6068904..3a55669 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -1967,7 +1967,8 @@ static int __devinit enic_probe(struct pci_dev *pdev,
break;
}
- netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
+ netdev->features |= NETIF_F_HW_VLAN_TX |
+ NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER;
if (ENIC_SETTING(enic, TXCSUM))
netdev->features |= NETIF_F_SG | NETIF_F_HW_CSUM;
if (ENIC_SETTING(enic, TSO))
^ permalink raw reply related
* [net-next PATCH 09/11] enic: changes to driver/firmware interface
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: changes to driver/firmware interface
Deprecate some old APIa; change arguments to stats dump all API; add new
interrupt assert API
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/vnic_dev.c | 14 ++++++++++++++
drivers/net/enic/vnic_dev.h | 1 +
drivers/net/enic/vnic_devcmd.h | 20 ++++++++++----------
drivers/net/enic/vnic_intr.c | 5 +++++
4 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/drivers/net/enic/vnic_dev.c b/drivers/net/enic/vnic_dev.c
index c8d3fc7..29a48e8 100644
--- a/drivers/net/enic/vnic_dev.c
+++ b/drivers/net/enic/vnic_dev.c
@@ -560,6 +560,20 @@ void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr)
printk(KERN_ERR "Can't del addr [%pM], %d\n", addr, err);
}
+int vnic_dev_raise_intr(struct vnic_dev *vdev, u16 intr)
+{
+ u64 a0 = intr, a1 = 0;
+ int wait = 1000;
+ int err;
+
+ err = vnic_dev_cmd(vdev, CMD_IAR, &a0, &a1, wait);
+ if (err)
+ printk(KERN_ERR "Failed to raise INTR[%d], err %d\n",
+ intr, err);
+
+ return err;
+}
+
int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr)
{
u64 a0, a1;
diff --git a/drivers/net/enic/vnic_dev.h b/drivers/net/enic/vnic_dev.h
index db1d63e..fc5e3eb 100644
--- a/drivers/net/enic/vnic_dev.h
+++ b/drivers/net/enic/vnic_dev.h
@@ -106,6 +106,7 @@ void vnic_dev_packet_filter(struct vnic_dev *vdev, int directed, int multicast,
void vnic_dev_add_addr(struct vnic_dev *vdev, u8 *addr);
void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr);
int vnic_dev_mac_addr(struct vnic_dev *vdev, u8 *mac_addr);
+int vnic_dev_raise_intr(struct vnic_dev *vdev, u16 intr);
int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr);
void vnic_dev_notify_unset(struct vnic_dev *vdev);
int vnic_dev_link_status(struct vnic_dev *vdev);
diff --git a/drivers/net/enic/vnic_devcmd.h b/drivers/net/enic/vnic_devcmd.h
index 2587f34..d78bbcc 100644
--- a/drivers/net/enic/vnic_devcmd.h
+++ b/drivers/net/enic/vnic_devcmd.h
@@ -105,14 +105,6 @@ enum vnic_devcmd_cmd {
CMD_MAC_ADDR = _CMDC(_CMD_DIR_READ,
_CMD_VTYPE_ENET | _CMD_VTYPE_FC, 9),
- /* disable/enable promisc mode: (u8)a0=0/1 */
-/***** XXX DEPRECATED *****/
- CMD_PROMISC_MODE = _CMDCNW(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 10),
-
- /* disable/enable all-multi mode: (u8)a0=0/1 */
-/***** XXX DEPRECATED *****/
- CMD_ALLMULTI_MODE = _CMDCNW(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 11),
-
/* add addr from (u48)a0 */
CMD_ADDR_ADD = _CMDCNW(_CMD_DIR_WRITE,
_CMD_VTYPE_ENET | _CMD_VTYPE_FC, 12),
@@ -182,7 +174,9 @@ enum vnic_devcmd_cmd {
/* disable virtual link */
CMD_DISABLE = _CMDC(_CMD_DIR_NONE, _CMD_VTYPE_ALL, 29),
- /* stats dump all vnics on uplink in mem: (u64)a0=paddr (u32)a1=uif */
+ /* stats dump sum of all vnic stats on same uplink in mem:
+ * (u64)a0=paddr
+ * (u16)a1=sizeof stats area */
CMD_STATS_DUMP_ALL = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 30),
/* init status:
@@ -211,7 +205,12 @@ enum vnic_devcmd_cmd {
/* persistent binding info
* in: (u64)a0=paddr of arg
* (u32)a1=CMD_PERBI_XXX */
- CMD_PERBI = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_FC, 37),
+ CMD_PERBI = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_FC, 37),
+
+ /* Interrupt Assert Register functionality
+ * in: (u16)a0=interrupt number to assert
+ */
+ CMD_IAR = _CMDCNW(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 38),
};
/* flags for CMD_OPEN */
@@ -244,6 +243,7 @@ enum vnic_devcmd_error {
ERR_ENOMEM = 7,
ERR_ETIMEDOUT = 8,
ERR_ELINKDOWN = 9,
+ ERR_EMAXRES = 10,
};
struct vnic_devcmd_fw_info {
diff --git a/drivers/net/enic/vnic_intr.c b/drivers/net/enic/vnic_intr.c
index ddc38f8..1f8786d 100644
--- a/drivers/net/enic/vnic_intr.c
+++ b/drivers/net/enic/vnic_intr.c
@@ -60,3 +60,8 @@ void vnic_intr_clean(struct vnic_intr *intr)
{
iowrite32(0, &intr->ctrl->int_credits);
}
+
+void vnic_intr_raise(struct vnic_intr *intr)
+{
+ vnic_dev_raise_intr(intr->vdev, (u16)intr->index);
+}
^ permalink raw reply related
* [net-next PATCH 10/11] enic: bug fix: check for zero port MTU before posting warning
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: bug fix: check for zero port MTU before posting warning
Nic firmware can return zero for port MTU, so check for non-zero value
before checking for change in port MTU.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic_main.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 3a55669..9368dae 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -362,7 +362,7 @@ static void enic_mtu_check(struct enic *enic)
{
u32 mtu = vnic_dev_mtu(enic->vdev);
- if (mtu != enic->port_mtu) {
+ if (mtu && mtu != enic->port_mtu) {
if (mtu < enic->netdev->mtu)
printk(KERN_WARNING PFX
"%s: interface MTU (%d) set higher "
^ permalink raw reply related
* [net-next PATCH 11/11] enic: organize device initialization/deinit into separate functions
From: Scott Feldman @ 2009-09-04 3:02 UTC (permalink / raw)
To: davem; +Cc: netdev
In-Reply-To: <20090904030046.5047.46509.stgit@palito_client100.nuovasystems.com>
enic: organize device initialization/deinit into separate functions
To unclutter probe() a little bit, put all device initialization code
in one spot and device deinit code in another spot. Also remove unused
rq->buf_index variable/func.
Signed-off-by: Scott Feldman <scofeldm@cisco.com>
---
drivers/net/enic/enic_main.c | 145 ++++++++++++++++++++++++++----------------
drivers/net/enic/vnic_rq.c | 27 +++++---
drivers/net/enic/vnic_rq.h | 12 +--
drivers/net/enic/vnic_wq.c | 20 +++++-
drivers/net/enic/vnic_wq.h | 4 +
5 files changed, 133 insertions(+), 75 deletions(-)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 9368dae..2737c0d 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -1740,6 +1740,88 @@ static const struct net_device_ops enic_netdev_ops = {
#endif
};
+void enic_dev_deinit(struct enic *enic)
+{
+ netif_napi_del(&enic->napi);
+ enic_free_vnic_resources(enic);
+ enic_clear_intr_mode(enic);
+}
+
+int enic_dev_init(struct enic *enic)
+{
+ struct net_device *netdev = enic->netdev;
+ int err;
+
+ /* Get vNIC configuration
+ */
+
+ err = enic_get_vnic_config(enic);
+ if (err) {
+ printk(KERN_ERR PFX
+ "Get vNIC configuration failed, aborting.\n");
+ return err;
+ }
+
+ /* Get available resource counts
+ */
+
+ enic_get_res_counts(enic);
+
+ /* Set interrupt mode based on resource counts and system
+ * capabilities
+ */
+
+ err = enic_set_intr_mode(enic);
+ if (err) {
+ printk(KERN_ERR PFX
+ "Failed to set intr mode, aborting.\n");
+ return err;
+ }
+
+ /* Allocate and configure vNIC resources
+ */
+
+ err = enic_alloc_vnic_resources(enic);
+ if (err) {
+ printk(KERN_ERR PFX
+ "Failed to alloc vNIC resources, aborting.\n");
+ goto err_out_free_vnic_resources;
+ }
+
+ enic_init_vnic_resources(enic);
+
+ err = enic_set_rq_alloc_buf(enic);
+ if (err) {
+ printk(KERN_ERR PFX
+ "Failed to set RQ buffer allocator, aborting.\n");
+ goto err_out_free_vnic_resources;
+ }
+
+ err = enic_set_niccfg(enic);
+ if (err) {
+ printk(KERN_ERR PFX
+ "Failed to config nic, aborting.\n");
+ goto err_out_free_vnic_resources;
+ }
+
+ switch (vnic_dev_get_intr_mode(enic->vdev)) {
+ default:
+ netif_napi_add(netdev, &enic->napi, enic_poll, 64);
+ break;
+ case VNIC_DEV_INTR_MODE_MSIX:
+ netif_napi_add(netdev, &enic->napi, enic_poll_msix, 64);
+ break;
+ }
+
+ return 0;
+
+err_out_free_vnic_resources:
+ enic_clear_intr_mode(enic);
+ enic_free_vnic_resources(enic);
+
+ return err;
+}
+
static void enic_iounmap(struct enic *enic)
{
unsigned int i;
@@ -1882,51 +1964,13 @@ static int __devinit enic_probe(struct pci_dev *pdev,
goto err_out_dev_close;
}
- /* Get vNIC configuration
- */
-
- err = enic_get_vnic_config(enic);
+ err = enic_dev_init(enic);
if (err) {
printk(KERN_ERR PFX
- "Get vNIC configuration failed, aborting.\n");
+ "Device initialization failed, aborting.\n");
goto err_out_dev_close;
}
- /* Get available resource counts
- */
-
- enic_get_res_counts(enic);
-
- /* Set interrupt mode based on resource counts and system
- * capabilities
- */
-
- err = enic_set_intr_mode(enic);
- if (err) {
- printk(KERN_ERR PFX
- "Failed to set intr mode, aborting.\n");
- goto err_out_dev_close;
- }
-
- /* Allocate and configure vNIC resources
- */
-
- err = enic_alloc_vnic_resources(enic);
- if (err) {
- printk(KERN_ERR PFX
- "Failed to alloc vNIC resources, aborting.\n");
- goto err_out_free_vnic_resources;
- }
-
- enic_init_vnic_resources(enic);
-
- err = enic_set_niccfg(enic);
- if (err) {
- printk(KERN_ERR PFX
- "Failed to config nic, aborting.\n");
- goto err_out_free_vnic_resources;
- }
-
/* Setup notification timer, HW reset task, and locks
*/
@@ -1951,22 +1995,13 @@ static int __devinit enic_probe(struct pci_dev *pdev,
if (err) {
printk(KERN_ERR PFX
"Invalid MAC address, aborting.\n");
- goto err_out_free_vnic_resources;
+ goto err_out_dev_deinit;
}
netdev->netdev_ops = &enic_netdev_ops;
netdev->watchdog_timeo = 2 * HZ;
netdev->ethtool_ops = &enic_ethtool_ops;
- switch (vnic_dev_get_intr_mode(enic->vdev)) {
- default:
- netif_napi_add(netdev, &enic->napi, enic_poll, 64);
- break;
- case VNIC_DEV_INTR_MODE_MSIX:
- netif_napi_add(netdev, &enic->napi, enic_poll_msix, 64);
- break;
- }
-
netdev->features |= NETIF_F_HW_VLAN_TX |
NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER;
if (ENIC_SETTING(enic, TXCSUM))
@@ -1994,17 +2029,16 @@ static int __devinit enic_probe(struct pci_dev *pdev,
if (err) {
printk(KERN_ERR PFX
"Cannot register net device, aborting.\n");
- goto err_out_free_vnic_resources;
+ goto err_out_dev_deinit;
}
return 0;
-err_out_free_vnic_resources:
- enic_free_vnic_resources(enic);
+err_out_dev_deinit:
+ enic_dev_deinit(enic);
err_out_dev_close:
vnic_dev_close(enic->vdev);
err_out_vnic_unregister:
- enic_clear_intr_mode(enic);
vnic_dev_unregister(enic->vdev);
err_out_iounmap:
enic_iounmap(enic);
@@ -2028,9 +2062,8 @@ static void __devexit enic_remove(struct pci_dev *pdev)
flush_scheduled_work();
unregister_netdev(netdev);
- enic_free_vnic_resources(enic);
+ enic_dev_deinit(enic);
vnic_dev_close(enic->vdev);
- enic_clear_intr_mode(enic);
vnic_dev_unregister(enic->vdev);
enic_iounmap(enic);
pci_release_regions(pdev);
diff --git a/drivers/net/enic/vnic_rq.c b/drivers/net/enic/vnic_rq.c
index 9365e63..7558397 100644
--- a/drivers/net/enic/vnic_rq.c
+++ b/drivers/net/enic/vnic_rq.c
@@ -62,7 +62,6 @@ static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
}
rq->to_use = rq->to_clean = rq->bufs[0];
- rq->buf_index = 0;
return 0;
}
@@ -113,12 +112,12 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
return 0;
}
-void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+ unsigned int fetch_index, unsigned int posted_index,
unsigned int error_interrupt_enable,
unsigned int error_interrupt_offset)
{
u64 paddr;
- u32 fetch_index;
paddr = (u64)rq->ring.base_addr | VNIC_PADDR_TARGET;
writeq(paddr, &rq->ctrl->ring_base);
@@ -128,15 +127,27 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset);
iowrite32(0, &rq->ctrl->dropped_packet_count);
iowrite32(0, &rq->ctrl->error_status);
+ iowrite32(fetch_index, &rq->ctrl->fetch_index);
+ iowrite32(posted_index, &rq->ctrl->posted_index);
- /* Use current fetch_index as the ring starting point */
- fetch_index = ioread32(&rq->ctrl->fetch_index);
rq->to_use = rq->to_clean =
&rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES]
[fetch_index % VNIC_RQ_BUF_BLK_ENTRIES];
- iowrite32(fetch_index, &rq->ctrl->posted_index);
+}
+
+void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+ unsigned int error_interrupt_enable,
+ unsigned int error_interrupt_offset)
+{
+ u32 fetch_index;
- rq->buf_index = 0;
+ /* Use current fetch_index as the ring starting point */
+ fetch_index = ioread32(&rq->ctrl->fetch_index);
+
+ vnic_rq_init_start(rq, cq_index,
+ fetch_index, fetch_index,
+ error_interrupt_enable,
+ error_interrupt_offset);
}
unsigned int vnic_rq_error_status(struct vnic_rq *rq)
@@ -192,8 +203,6 @@ void vnic_rq_clean(struct vnic_rq *rq,
[fetch_index % VNIC_RQ_BUF_BLK_ENTRIES];
iowrite32(fetch_index, &rq->ctrl->posted_index);
- rq->buf_index = 0;
-
vnic_dev_clear_desc_ring(&rq->ring);
}
diff --git a/drivers/net/enic/vnic_rq.h b/drivers/net/enic/vnic_rq.h
index f7b5730..35e736c 100644
--- a/drivers/net/enic/vnic_rq.h
+++ b/drivers/net/enic/vnic_rq.h
@@ -1,5 +1,5 @@
/*
- * Copyright 2008 Cisco Systems, Inc. All rights reserved.
+ * Copyright 2008, 2009 Cisco Systems, Inc. All rights reserved.
* Copyright 2007 Nuova Systems, Inc. All rights reserved.
*
* This program is free software; you may redistribute it and/or modify
@@ -79,7 +79,6 @@ struct vnic_rq {
struct vnic_rq_buf *to_use;
struct vnic_rq_buf *to_clean;
void *os_buf_head;
- unsigned int buf_index;
unsigned int pkts_outstanding;
};
@@ -105,11 +104,6 @@ static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
return rq->to_use->index;
}
-static inline unsigned int vnic_rq_next_buf_index(struct vnic_rq *rq)
-{
- return rq->buf_index++;
-}
-
static inline void vnic_rq_post(struct vnic_rq *rq,
void *os_buf, unsigned int os_buf_index,
dma_addr_t dma_addr, unsigned int len)
@@ -204,6 +198,10 @@ static inline int vnic_rq_fill(struct vnic_rq *rq,
void vnic_rq_free(struct vnic_rq *rq);
int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
unsigned int desc_count, unsigned int desc_size);
+void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+ unsigned int fetch_index, unsigned int posted_index,
+ unsigned int error_interrupt_enable,
+ unsigned int error_interrupt_offset);
void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
unsigned int error_interrupt_enable,
unsigned int error_interrupt_offset);
diff --git a/drivers/net/enic/vnic_wq.c b/drivers/net/enic/vnic_wq.c
index a576d04..d2e00e5 100644
--- a/drivers/net/enic/vnic_wq.c
+++ b/drivers/net/enic/vnic_wq.c
@@ -112,7 +112,8 @@ int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
return 0;
}
-void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
+void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index,
+ unsigned int fetch_index, unsigned int posted_index,
unsigned int error_interrupt_enable,
unsigned int error_interrupt_offset)
{
@@ -121,12 +122,25 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
paddr = (u64)wq->ring.base_addr | VNIC_PADDR_TARGET;
writeq(paddr, &wq->ctrl->ring_base);
iowrite32(wq->ring.desc_count, &wq->ctrl->ring_size);
- iowrite32(0, &wq->ctrl->fetch_index);
- iowrite32(0, &wq->ctrl->posted_index);
+ iowrite32(fetch_index, &wq->ctrl->fetch_index);
+ iowrite32(posted_index, &wq->ctrl->posted_index);
iowrite32(cq_index, &wq->ctrl->cq_index);
iowrite32(error_interrupt_enable, &wq->ctrl->error_interrupt_enable);
iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset);
iowrite32(0, &wq->ctrl->error_status);
+
+ wq->to_use = wq->to_clean =
+ &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES]
+ [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES];
+}
+
+void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
+ unsigned int error_interrupt_enable,
+ unsigned int error_interrupt_offset)
+{
+ vnic_wq_init_start(wq, cq_index, 0, 0,
+ error_interrupt_enable,
+ error_interrupt_offset);
}
unsigned int vnic_wq_error_status(struct vnic_wq *wq)
diff --git a/drivers/net/enic/vnic_wq.h b/drivers/net/enic/vnic_wq.h
index c826137..9c34d41 100644
--- a/drivers/net/enic/vnic_wq.h
+++ b/drivers/net/enic/vnic_wq.h
@@ -149,6 +149,10 @@ static inline void vnic_wq_service(struct vnic_wq *wq,
void vnic_wq_free(struct vnic_wq *wq);
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
unsigned int desc_count, unsigned int desc_size);
+void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index,
+ unsigned int fetch_index, unsigned int posted_index,
+ unsigned int error_interrupt_enable,
+ unsigned int error_interrupt_offset);
void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
unsigned int error_interrupt_enable,
unsigned int error_interrupt_offset);
^ permalink raw reply related
* Re: [PATCH NEXT 1/3] netxen: fix lro buffer allocation
From: David Miller @ 2009-09-04 3:06 UTC (permalink / raw)
To: dhananjay; +Cc: davem, netdev, dhananjay
In-Reply-To: <1252019455-30683-1-git-send-email-dhananjay@netxen.com>
From: Dhananjay Phadke <dhananjay@netxen.com>
Date: Thu, 3 Sep 2009 16:10:53 -0700
> From: Dhananjay Phadke <dhananjay@qlogic.com>
>
> Alloc 12k skbuffs so that firmware can aggregate more
> packets into one buffer. This doesn't raise memory
> consumption since 9k skbs use 16k slab cache anyway.
>
> Signed-off-by: Dhananjay Phadke <dhananjay@netxen.com>
Applied. But allocating such huge linear buffers is asking for
guarenteed trouble. Can't you split them up into page sized chunks
with this chip?
^ permalink raw reply
* Re: [PATCH NEXT 2/3] netxen: remove duplicate napi_add
From: David Miller @ 2009-09-04 3:07 UTC (permalink / raw)
To: dhananjay; +Cc: davem, netdev
In-Reply-To: <1252019455-30683-2-git-send-email-dhananjay@netxen.com>
From: Dhananjay Phadke <dhananjay@netxen.com>
Date: Thu, 3 Sep 2009 16:10:54 -0700
> Remove duplicate calls to netxen_napi_add().
>
> Signed-off-by: Dhananjay Phadke <dhananjay@netxen.com>
Applied, but:
> @@ -2038,8 +2034,6 @@ netxen_remove_sysfs_entries(struct netxen_adapter *adapter)
> device_remove_file(dev, &dev_attr_bridged_mode);
> }
>
> -static void netxen_watchdog(unsigned long);
> -
> #ifdef CONFIG_INET
>
> #define is_netxen_netdev(dev) (dev->netdev_ops == &netxen_netdev_ops)
Give me a friggin' break, really.
If you're going to slip unrelated cleanups like this into your
patches, at least DOCUMENT them in the commit message.
^ permalink raw reply
* Re: [PATCH NEXT 3/3] netxen: fix infinite loop on dma mapping failure
From: David Miller @ 2009-09-04 3:07 UTC (permalink / raw)
To: dhananjay; +Cc: davem, netdev
In-Reply-To: <1252019455-30683-3-git-send-email-dhananjay@netxen.com>
From: Dhananjay Phadke <dhananjay@netxen.com>
Date: Thu, 3 Sep 2009 16:10:55 -0700
> Fix a perpetual while() loop in unwinding partial
> mapped tx skb on dma mapping failure.
>
> Reported-by: "Juha Leppanen" <juha_motorsportcom@luukku.com>
> Signed-off-by: Dhananjay Phadke <dhananjay@netxen.com>
Applied.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox