[PATCH] iommu/amd: serialize sequence allocation under concurrent TLB invalidations

[PATCH] iommu/amd: serialize sequence allocation under concurrent TLB invalidations

[Ankit Soni]

With concurrent TLB invalidations, completion wait randomly gets timed out
because cmd_sem_val was incremented outside the IOMMU spinlock, allowing
CMD_COMPL_WAIT commands to be queued out of sequence and breaking the
ordering assumption in wait_on_sem().
Move the cmd_sem_val increment under iommu->lock so completion sequence
allocation is serialized with command queuing.
And remove the unnecessary return.

Fixes: d2a0cac10597 ("iommu/amd: move wait_on_sem() out of spinlock")

Tested-by: Srikanth Aithal <sraithal@amd.com>
Reported-by: Srikanth Aithal <sraithal@amd.com>
Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
---
 drivers/iommu/amd/iommu.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index d7f457338de7..593fb879b7b0 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1422,6 +1422,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
 	return iommu_queue_command_sync(iommu, cmd, true);
 }
 
+static u64 get_cmdsem_val(struct amd_iommu *iommu)
+{
+	lockdep_assert_held(&iommu->lock);
+	return atomic64_inc_return(&iommu->cmd_sem_val);
+}
+
 /*
  * This function queues a completion wait command into the command
  * buffer of an IOMMU
@@ -1436,11 +1442,11 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
 	if (!iommu->need_sync)
 		return 0;
 
-	data = atomic64_inc_return(&iommu->cmd_sem_val);
-	build_completion_wait(&cmd, iommu, data);
-
 	raw_spin_lock_irqsave(&iommu->lock, flags);
 
+	data = get_cmdsem_val(iommu);
+	build_completion_wait(&cmd, iommu, data);
+
 	ret = __iommu_queue_command_sync(iommu, &cmd, false);
 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
 
@@ -3119,10 +3125,11 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
 		return;
 
 	build_inv_irt(&cmd, devid);
-	data = atomic64_inc_return(&iommu->cmd_sem_val);
-	build_completion_wait(&cmd2, iommu, data);
 
 	raw_spin_lock_irqsave(&iommu->lock, flags);
+	data = get_cmdsem_val(iommu);
+	build_completion_wait(&cmd2, iommu, data);
+
 	ret = __iommu_queue_command_sync(iommu, &cmd, true);
 	if (ret)
 		goto out_err;
@@ -3136,7 +3143,6 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
 
 out_err:
 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
-	return;
 }
 
 static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)
-- 
2.43.0

Re: [PATCH] iommu/amd: serialize sequence allocation under concurrent TLB invalidations

[Jacob Pan]

Hi Ankit,

On Tue, 20 Jan 2026 09:05:07 +0000
Ankit Soni <Ankit.Soni@amd.com> wrote:

> With concurrent TLB invalidations, completion wait randomly gets
> timed out because cmd_sem_val was incremented outside the IOMMU
> spinlock, allowing CMD_COMPL_WAIT commands to be queued out of
> sequence and breaking the ordering assumption in wait_on_sem().
> Move the cmd_sem_val increment under iommu->lock so completion
> sequence allocation is serialized with command queuing.
> And remove the unnecessary return.
> 
> Fixes: d2a0cac10597 ("iommu/amd: move wait_on_sem() out of spinlock")
> 
> Tested-by: Srikanth Aithal <sraithal@amd.com>
> Reported-by: Srikanth Aithal <sraithal@amd.com>
> Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
> ---
>  drivers/iommu/amd/iommu.c | 18 ++++++++++++------
>  1 file changed, 12 insertions(+), 6 deletions(-)
> 
> diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
> index d7f457338de7..593fb879b7b0 100644
> --- a/drivers/iommu/amd/iommu.c
> +++ b/drivers/iommu/amd/iommu.c
> @@ -1422,6 +1422,12 @@ static int iommu_queue_command(struct
> amd_iommu *iommu, struct iommu_cmd *cmd) return
> iommu_queue_command_sync(iommu, cmd, true); }
>  
> +static u64 get_cmdsem_val(struct amd_iommu *iommu)
> +{
> +	lockdep_assert_held(&iommu->lock);
> +	return atomic64_inc_return(&iommu->cmd_sem_val);
Do we still need this to be atomic now that it’s protected by a
spinlock?

> +}
> +
>  /*
>   * This function queues a completion wait command into the command
>   * buffer of an IOMMU
> @@ -1436,11 +1442,11 @@ static int iommu_completion_wait(struct
> amd_iommu *iommu) if (!iommu->need_sync)
>  		return 0;
>  
> -	data = atomic64_inc_return(&iommu->cmd_sem_val);
> -	build_completion_wait(&cmd, iommu, data);
> -
>  	raw_spin_lock_irqsave(&iommu->lock, flags);
>  
> +	data = get_cmdsem_val(iommu);
> +	build_completion_wait(&cmd, iommu, data);
> +
>  	ret = __iommu_queue_command_sync(iommu, &cmd, false);
>  	raw_spin_unlock_irqrestore(&iommu->lock, flags);
>  
> @@ -3119,10 +3125,11 @@ static void
> iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
> return; 
>  	build_inv_irt(&cmd, devid);
> -	data = atomic64_inc_return(&iommu->cmd_sem_val);
> -	build_completion_wait(&cmd2, iommu, data);
>  
>  	raw_spin_lock_irqsave(&iommu->lock, flags);
> +	data = get_cmdsem_val(iommu);
> +	build_completion_wait(&cmd2, iommu, data);
> +
>  	ret = __iommu_queue_command_sync(iommu, &cmd, true);
>  	if (ret)
>  		goto out_err;
> @@ -3136,7 +3143,6 @@ static void iommu_flush_irt_and_complete(struct
> amd_iommu *iommu, u16 devid) 
>  out_err:
>  	raw_spin_unlock_irqrestore(&iommu->lock, flags);
> -	return;
>  }
>  
>  static inline u8 iommu_get_int_tablen(struct iommu_dev_data
> *dev_data)

Re: [PATCH] iommu/amd: serialize sequence allocation under concurrent TLB invalidations

[Ankit Soni]

On Tue, Jan 20, 2026 at 09:24:27AM -0800, Jacob Pan wrote:
> Hi Ankit,
> 
> On Tue, 20 Jan 2026 09:05:07 +0000
> Ankit Soni <Ankit.Soni@amd.com> wrote:
> 
> > With concurrent TLB invalidations, completion wait randomly gets
> > timed out because cmd_sem_val was incremented outside the IOMMU
> > spinlock, allowing CMD_COMPL_WAIT commands to be queued out of
> > sequence and breaking the ordering assumption in wait_on_sem().
> > Move the cmd_sem_val increment under iommu->lock so completion
> > sequence allocation is serialized with command queuing.
> > And remove the unnecessary return.
> > 
> > Fixes: d2a0cac10597 ("iommu/amd: move wait_on_sem() out of spinlock")
> > 
> > Tested-by: Srikanth Aithal <sraithal@amd.com>
> > Reported-by: Srikanth Aithal <sraithal@amd.com>
> > Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
> > ---
> >  drivers/iommu/amd/iommu.c | 18 ++++++++++++------
> >  1 file changed, 12 insertions(+), 6 deletions(-)
> > 
> > diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
> > index d7f457338de7..593fb879b7b0 100644
> > --- a/drivers/iommu/amd/iommu.c
> > +++ b/drivers/iommu/amd/iommu.c
> > @@ -1422,6 +1422,12 @@ static int iommu_queue_command(struct
> > amd_iommu *iommu, struct iommu_cmd *cmd) return
> > iommu_queue_command_sync(iommu, cmd, true); }
> >  
> > +static u64 get_cmdsem_val(struct amd_iommu *iommu)
> > +{
> > +	lockdep_assert_held(&iommu->lock);
> > +	return atomic64_inc_return(&iommu->cmd_sem_val);
> Do we still need this to be atomic now that it’s protected by a
> spinlock?
> 

Hi Jacob,
Thanks for pointing this out, we can remove atomic operation here.
I will change and post v2.

-Ankit