[PATCH 6.18 231/752] iommu/amd: move wait_on_sem() out of spinlock

Sasha Levin
2026-02-28T17:51:23Z

From: Ankit Soni <Ankit.Soni@amd.com>

[ Upstream commit d2a0cac10597068567d336e85fa3cbdbe8ca62bf ]

With iommu.strict=1, the existing completion wait path can cause soft
lockups under stressed environment, as wait_on_sem() busy-waits under the
spinlock with interrupts disabled.

Move the completion wait in iommu_completion_wait() out of the spinlock.
wait_on_sem() only polls the hardware-updated cmd_sem and does not require
iommu->lock, so holding the lock during the busy wait unnecessarily
increases contention and extends the time with interrupts disabled.

Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
Reviewed-by: Vasant Hegde <vasant.hegde@amd.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
drivers/iommu/amd/iommu.c | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 30dd482fe0953..3f2b687947dba 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1156,7 +1156,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data)
{
int i = 0;

- while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {
+ /*
+ * cmd_sem holds a monotonically non-decreasing completion sequence
+ * number.
+ */
+ while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 &&
+ i < LOOP_TIMEOUT) {
udelay(1);
i += 1;
}
@@ -1401,14 +1406,13 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
raw_spin_lock_irqsave(&iommu->lock, flags);

ret = __iommu_queue_command_sync(iommu, &cmd, false);
+ raw_spin_unlock_irqrestore(&iommu->lock, flags);
+
if (ret)
- goto out_unlock;
+ return ret;

ret = wait_on_sem(iommu, data);

-out_unlock:
- raw_spin_unlock_irqrestore(&iommu->lock, flags);
-
return ret;
}

@@ -3088,13 +3092,18 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __iommu_queue_command_sync(iommu, &cmd, true);
if (ret)
- goto out;
+ goto out_err;
ret = __iommu_queue_command_sync(iommu, &cmd2, false);
if (ret)
- goto out;
+ goto out_err;
+ raw_spin_unlock_irqrestore(&iommu->lock, flags);
+
wait_on_sem(iommu, data);
-out:
+ return;
+
+out_err:
raw_spin_unlock_irqrestore(&iommu->lock, flags);
+ return;
}

static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)
--
2.51.0