[PATCH 3/4] thunderbolt: Use shadow copy for ring interrupt mask

[Mika Westerberg <mika.westerberg@linux.intel.com>]

Each time we mask or unmask ring interrupt we do a read-modify-write and
that is not without a cost when on a hot path. We can get rid of that
read by making a shadow copy of the interrupt mask register and using
that in place of a register read.

Assisted-by: LLM
Signed-off-by: Mika Westerberg <mika.westerberg@linux.intel.com>
---
 drivers/thunderbolt/nhi.c   | 55 +++++++++++++++++++++++--------------
 include/linux/thunderbolt.h |  2 ++
 2 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index aaaf9e7966c8..a7e6184cdfe1 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -53,15 +53,28 @@ static int ring_interrupt_index(const struct tb_ring *ring)
 static void nhi_mask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
 {
 	int offset = reg_index * 4;
+	u32 val;
 
-	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT) {
-		u32 val;
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] & ~mask;
+	nhi->interrupt_mask[reg_index] = val;
 
-		val = ioread32(nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
-		iowrite32(val & ~mask, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
-	} else {
+	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
+		iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
+	else
 		iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + offset);
-	}
+}
+
+static void nhi_unmask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
+{
+	int offset = reg_index * 4;
+	u32 val;
+
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] | mask;
+	nhi->interrupt_mask[reg_index] = val;
+
+	iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
 }
 
 static void nhi_clear_interrupt(struct tb_nhi *nhi, int reg_index)
@@ -129,7 +142,7 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 			  ring->vector * 4);
 	}
 
-	old = ioread32(ring->nhi->iobase + reg);
+	old = ring->nhi->interrupt_mask[reg_index];
 	if (active)
 		new = old | mask;
 	else
@@ -141,10 +154,10 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 
 	if (new == old) {
 		/*
-		 * Rings that are polled mask the interrupt using while
-		 * the completions are being advanced (see
-		 * __ring_interrupt()) so for those it can already be
-		 * disabled by the time the ring is stopped.
+		 * Rings that are polled mask the interrupt while the
+		 * completions are being advanced (see __ring_interrupt())
+		 * so for those it can already be disabled by the time
+		 * the ring is stopped.
 		 */
 		if (active || !ring->start_poll)
 			dev_WARN(ring->nhi->dev,
@@ -154,7 +167,7 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 	}
 
 	if (active)
-		iowrite32(new, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, mask, reg_index);
 	else
 		nhi_mask_interrupt(ring->nhi, mask, reg_index);
 }
@@ -437,17 +450,14 @@ EXPORT_SYMBOL_GPL(tb_ring_poll);
 
 static void __ring_interrupt_mask(struct tb_ring *ring, bool mask)
 {
-	int idx = ring_interrupt_index(ring);
-	int reg = REG_RING_INTERRUPT_BASE + idx / 32 * 4;
-	int bit = idx % 32;
-	u32 val;
+	int interrupt_index = ring_interrupt_index(ring);
+	int reg_index = interrupt_index / 32;
+	int interrupt_bit = interrupt_index % 32;
 
-	val = ioread32(ring->nhi->iobase + reg);
 	if (mask)
-		val &= ~BIT(bit);
+		nhi_mask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 	else
-		val |= BIT(bit);
-	iowrite32(val, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 }
 
 /* Both @nhi->lock and @ring->lock should be held */
@@ -1315,7 +1325,10 @@ int nhi_probe(struct tb_nhi *nhi)
 				     sizeof(*nhi->tx_rings), GFP_KERNEL);
 	nhi->rx_rings = devm_kcalloc(dev, nhi->hop_count,
 				     sizeof(*nhi->rx_rings), GFP_KERNEL);
-	if (!nhi->tx_rings || !nhi->rx_rings)
+	nhi->interrupt_mask = devm_kcalloc(dev, RING_INTERRUPT_REG_COUNT(nhi),
+					   sizeof(*nhi->interrupt_mask),
+					   GFP_KERNEL);
+	if (!nhi->tx_rings || !nhi->rx_rings || !nhi->interrupt_mask)
 		return -ENOMEM;
 
 	nhi_reset(nhi);
diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h
index 7fb9e7e1aae4..8ca941e8b5d0 100644
--- a/include/linux/thunderbolt.h
+++ b/include/linux/thunderbolt.h
@@ -507,6 +507,7 @@ void tb_service_properties_changed(struct tb_service *svc);
  * @iobase: MMIO space of the NHI
  * @tx_rings: All Tx rings available on this host controller
  * @rx_rings: All Rx rings available on this host controller
+ * @interrupt_mask: Shadow copy of the ring interrupt mask register
  * @going_away: The host controller device is about to disappear so when
  *		this flag is set, avoid touching the hardware anymore.
  * @iommu_dma_protection: An IOMMU will isolate external-facing ports.
@@ -528,6 +529,7 @@ struct tb_nhi {
 	void __iomem *iobase;
 	struct tb_ring **tx_rings;
 	struct tb_ring **rx_rings;
+	u32 *interrupt_mask;
 	bool going_away;
 	bool iommu_dma_protection;
 	struct work_struct interrupt_work;
-- 
2.50.1