From: Dan Williams <dan.j.williams@intel.com>
To: linux-kernel@vger.kernel.org
Cc: linux-raid@vger.kernel.org, netdev@vger.kernel.org,
Maciej Sosnowski <maciej.sosnowski@intel.com>
Subject: [PATCH 15/29] ioat: cleanup completion status reads
Date: Thu, 03 Sep 2009 19:31:46 -0700 [thread overview]
Message-ID: <20090904023146.32667.48008.stgit@dwillia2-linux.ch.intel.com> (raw)
In-Reply-To: <20090904022733.32667.77626.stgit@dwillia2-linux.ch.intel.com>
The cleanup path makes an effort to only perform an atomic read of the
64-bit completion address. However in the 32-bit case it does not
matter if we read the upper-32 and lower-32 non-atomically because the
upper-32 will always be zero.
Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/ioat/dma.c | 78 ++++++++++++++++--------------------------
drivers/dma/ioat/dma.h | 10 +----
drivers/dma/ioat/dma_v2.c | 25 ++++++-------
drivers/dma/ioat/registers.h | 8 ++--
4 files changed, 46 insertions(+), 75 deletions(-)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index edf4f5e..08417ad 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -201,8 +201,7 @@ static void ioat1_reset_part2(struct work_struct *work)
spin_lock_bh(&chan->cleanup_lock);
spin_lock_bh(&ioat->desc_lock);
- chan->completion_virt->low = 0;
- chan->completion_virt->high = 0;
+ *chan->completion = 0;
ioat->pending = 0;
/* count the descriptors waiting */
@@ -256,8 +255,7 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
dev_dbg(to_dev(chan), "%s\n", __func__);
chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
- chansts = (chan->completion_virt->low
- & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
+ chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
@@ -293,14 +291,8 @@ static void ioat1_chan_watchdog(struct work_struct *work)
struct ioat_dma_chan *ioat;
struct ioat_chan_common *chan;
int i;
-
- union {
- u64 full;
- struct {
- u32 low;
- u32 high;
- };
- } completion_hw;
+ u64 completion;
+ u32 completion_low;
unsigned long compl_desc_addr_hw;
for (i = 0; i < device->common.chancnt; i++) {
@@ -334,25 +326,24 @@ static void ioat1_chan_watchdog(struct work_struct *work)
* try resetting the channel
*/
- completion_hw.low = readl(chan->reg_base +
+ /* we need to read the low address first as this
+ * causes the chipset to latch the upper bits
+ * for the subsequent read
+ */
+ completion_low = readl(chan->reg_base +
IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
- completion_hw.high = readl(chan->reg_base +
+ completion = readl(chan->reg_base +
IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
-#if (BITS_PER_LONG == 64)
- compl_desc_addr_hw =
- completion_hw.full
- & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
- compl_desc_addr_hw =
- completion_hw.low & IOAT_LOW_COMPLETION_MASK;
-#endif
+ completion <<= 32;
+ completion |= completion_low;
+ compl_desc_addr_hw = completion &
+ IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
if ((compl_desc_addr_hw != 0)
&& (compl_desc_addr_hw != chan->watchdog_completion)
&& (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
- chan->completion_virt->low = completion_hw.low;
- chan->completion_virt->high = completion_hw.high;
+ *chan->completion = completion;
} else {
ioat1_reset_channel(ioat);
chan->watchdog_completion = 0;
@@ -492,14 +483,12 @@ static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
- chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
- GFP_KERNEL,
- &chan->completion_addr);
- memset(chan->completion_virt, 0,
- sizeof(*chan->completion_virt));
- writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
+ chan->completion = pci_pool_alloc(chan->device->completion_pool,
+ GFP_KERNEL, &chan->completion_dma);
+ memset(chan->completion, 0, sizeof(*chan->completion));
+ writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
- writel(((u64) chan->completion_addr) >> 32,
+ writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
tasklet_enable(&chan->cleanup_task);
@@ -558,15 +547,16 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)
spin_unlock_bh(&ioat->desc_lock);
pci_pool_free(ioatdma_device->completion_pool,
- chan->completion_virt,
- chan->completion_addr);
+ chan->completion,
+ chan->completion_dma);
/* one is ok since we left it on there on purpose */
if (in_use_descs > 1)
dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
in_use_descs - 1);
- chan->last_completion = chan->completion_addr = 0;
+ chan->last_completion = 0;
+ chan->completion_dma = 0;
chan->watchdog_completion = 0;
chan->last_compl_desc_addr_hw = 0;
chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
@@ -709,25 +699,15 @@ void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
{
unsigned long phys_complete;
+ u64 completion;
- /* The completion writeback can happen at any time,
- so reads by the driver need to be atomic operations
- The descriptor physical addresses are limited to 32-bits
- when the CPU can only do a 32-bit mov */
-
-#if (BITS_PER_LONG == 64)
- phys_complete =
- chan->completion_virt->full
- & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
- phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
-#endif
+ completion = *chan->completion;
+ phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete);
- if ((chan->completion_virt->full
- & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
+ if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
readl(chan->reg_base + IOAT_CHANERR_OFFSET));
@@ -750,7 +730,7 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
dma_cookie_t cookie = 0;
struct dma_async_tx_descriptor *tx;
- prefetch(chan->completion_virt);
+ prefetch(chan->completion);
if (!spin_trylock_bh(&chan->cleanup_lock))
return;
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 9f9edc2..5fd6e2d 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -96,14 +96,8 @@ struct ioat_chan_common {
struct ioatdma_device *device;
struct dma_chan common;
- dma_addr_t completion_addr;
- union {
- u64 full; /* HW completion writeback */
- struct {
- u32 low;
- u32 high;
- };
- } *completion_virt;
+ dma_addr_t completion_dma;
+ u64 *completion;
unsigned long last_compl_desc_addr_hw;
struct tasklet_struct cleanup_task;
};
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index eecda23..9bd3d14 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -201,8 +201,7 @@ static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
return;
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
- chansts = (chan->completion_virt->low
- & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
+ chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
@@ -282,7 +281,7 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
int i;
struct dma_async_tx_descriptor *tx;
- prefetch(chan->completion_virt);
+ prefetch(chan->completion);
spin_lock_bh(&chan->cleanup_lock);
phys_complete = ioat_get_current_completion(chan);
@@ -471,17 +470,15 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)
/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
- chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
- GFP_KERNEL,
- &chan->completion_addr);
- if (!chan->completion_virt)
+ chan->completion = pci_pool_alloc(chan->device->completion_pool,
+ GFP_KERNEL, &chan->completion_dma);
+ if (!chan->completion)
return -ENOMEM;
- memset(chan->completion_virt, 0,
- sizeof(*chan->completion_virt));
- writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
+ memset(chan->completion, 0, sizeof(*chan->completion));
+ writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
- writel(((u64) chan->completion_addr) >> 32,
+ writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
ioat->alloc_order = ioat_get_alloc_order();
@@ -656,12 +653,12 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
ioat->ring = NULL;
ioat->alloc_order = 0;
pci_pool_free(ioatdma_device->completion_pool,
- chan->completion_virt,
- chan->completion_addr);
+ chan->completion,
+ chan->completion_dma);
spin_unlock_bh(&ioat->ring_lock);
chan->last_completion = 0;
- chan->completion_addr = 0;
+ chan->completion_dma = 0;
ioat->pending = 0;
ioat->dmacount = 0;
chan->watchdog_completion = 0;
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index 49bc277..a83c733 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -94,10 +94,10 @@
#define IOAT2_CHANSTS_OFFSET_HIGH 0x0C
#define IOAT_CHANSTS_OFFSET_HIGH(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
-#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ~0x3F
-#define IOAT_CHANSTS_SOFT_ERR 0x0000000000000010
-#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x0000000000000008
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x0000000000000007
+#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
+#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
+#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
+#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x7ULL
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
next prev parent reply other threads:[~2009-09-04 2:31 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-09-04 2:30 [PATCH 00/29] ioatdma: towards raid support Dan Williams
2009-09-04 2:30 ` [PATCH 01/29] ioat: move to drivers/dma/ioat/ Dan Williams
2009-09-04 2:30 ` [PATCH 02/29] ioat: move definitions to dma.h Dan Williams
2009-09-04 2:30 ` [PATCH 03/29] ioat: convert ioat_probe to pcim/devm Dan Williams
2009-09-04 2:30 ` [PATCH 04/29] ioat: cleanup some long deref chains and 80 column collisions Dan Williams
2009-09-04 2:30 ` [PATCH 05/29] ioat: kill function prototype ifdef guards Dan Williams
2009-09-04 2:31 ` [PATCH 06/29] ioat: split ioat_dma_probe into core/version-specific routines Dan Williams
2009-09-04 2:31 ` [PATCH 07/29] ioat: fix type mismatch for ->dmacount Dan Williams
2009-09-04 2:31 ` [PATCH 08/29] ioat: define descriptor control bit-field Dan Williams
2009-09-04 2:31 ` [PATCH 09/29] ioat1: move descriptor allocation from submit to prep Dan Williams
2009-09-04 2:31 ` [PATCH 10/29] ioat: fix self test interrupts Dan Williams
2009-09-04 2:31 ` [PATCH 11/29] ioat: prepare the code for ioat[12]_dma_chan split Dan Williams
2009-09-04 2:31 ` [PATCH 12/29] ioat2,3: convert to a true ring buffer Dan Williams
2009-09-04 2:31 ` [PATCH 13/29] ioat1: kill unused unmap parameters Dan Williams
2009-09-04 2:31 ` [PATCH 14/29] ioat: add some dev_dbg() calls Dan Williams
2009-09-04 2:31 ` Dan Williams [this message]
2009-09-04 2:31 ` [PATCH 16/29] ioat: ignore reserved bits for chancnt and xfercap Dan Williams
2009-09-04 2:31 ` [PATCH 17/29] ioat: preserve chanctrl bits when re-arming interrupts Dan Williams
2009-09-04 2:32 ` [PATCH 18/29] ioat: ___devinit annotate the initialization paths Dan Williams
2009-09-04 2:32 ` [PATCH 19/29] ioat1: trim ioat_dma_desc_sw Dan Williams
2009-09-14 14:55 ` Sosnowski, Maciej
2009-09-04 2:32 ` [PATCH 20/29] ioat: switch watchdog and reset handler from workqueue to timer Dan Williams
2009-09-14 14:59 ` Sosnowski, Maciej
2009-09-04 2:32 ` [PATCH 21/29] ioat2,3: dynamically resize descriptor ring Dan Williams
2009-09-14 15:00 ` Sosnowski, Maciej
2009-09-15 23:07 ` Dan Williams
2009-09-04 2:32 ` [PATCH 22/29] net_dma: poll for a descriptor after allocation failure Dan Williams
2009-09-14 15:00 ` Sosnowski, Maciej
2009-09-04 2:32 ` [PATCH 23/29] dw_dmac: implement a private tx_list Dan Williams
2009-09-04 2:32 ` [PATCH 24/29] fsldma: " Dan Williams
2009-09-04 19:42 ` Dan Williams
2009-09-04 2:32 ` [PATCH 25/29] iop-adma: " Dan Williams
2009-09-04 2:32 ` [PATCH 26/29] ioat: " Dan Williams
2009-09-14 15:01 ` Sosnowski, Maciej
2009-09-04 2:32 ` [PATCH 27/29] mv_xor: " Dan Williams
2009-09-04 2:32 ` [PATCH 28/29] dmaengine: kill tx_list Dan Williams
2009-09-14 15:01 ` Sosnowski, Maciej
2009-09-04 2:32 ` [PATCH 29/29] ioat2, 3: cacheline align software descriptor allocations Dan Williams
2009-09-14 15:02 ` Sosnowski, Maciej
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20090904023146.32667.48008.stgit@dwillia2-linux.ch.intel.com \
--to=dan.j.williams@intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-raid@vger.kernel.org \
--cc=maciej.sosnowski@intel.com \
--cc=netdev@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).