[PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[Lu Baolu]

As this iommu driver now supports page faults for requests without
PASID, page requests should be drained when a domain is removed from
the RID2PASID entry.

This results in the intel_iommu_drain_pasid_prq() call being moved to
intel_pasid_tear_down_entry(). This indicates that when a translation
is removed from any PASID entry and the PRI has been enabled on the
device, page requests are flushed in the domain detachment path.

The intel_iommu_drain_pasid_prq() helper has been modified to support
sending device TLB invalidation requests for both PASID and non-PASID
cases.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
 drivers/iommu/intel/iommu.c |  1 -
 drivers/iommu/intel/pasid.c |  1 +
 drivers/iommu/intel/prq.c   | 22 +++++++---------------
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 87a3563dfe54..3878f35be09d 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -4069,7 +4069,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
 	intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
 	kfree(dev_pasid);
 	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
-	intel_iommu_drain_pasid_prq(dev, pasid);
 }
 
 static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 7e76062a7ad2..31665fb62e1c 100644
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -265,6 +265,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
 
 	devtlb_invalidation_with_pasid(iommu, dev, pasid);
+	intel_iommu_drain_pasid_prq(dev, pasid);
 }
 
 /*
diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
index 3c50c848893f..ae7f6f34462f 100644
--- a/drivers/iommu/intel/prq.c
+++ b/drivers/iommu/intel/prq.c
@@ -66,12 +66,8 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
 	struct pci_dev *pdev;
 	int head, tail;
 	u16 sid, did;
-	int qdep;
 
 	info = dev_iommu_priv_get(dev);
-	if (WARN_ON(!info || !dev_is_pci(dev)))
-		return;
-
 	if (!info->pri_enabled)
 		return;
 
@@ -81,8 +77,6 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
 	sid = PCI_DEVID(info->bus, info->devfn);
 	did = domain ? domain_id_iommu(domain, iommu) : FLPT_DEFAULT_DID;
 
-	qdep = pci_ats_queue_depth(pdev);
-
 	/*
 	 * Check and wait until all pending page requests in the queue are
 	 * handled by the prq handling thread.
@@ -114,15 +108,13 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
 	desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
 			QI_IWD_FENCE |
 			QI_IWD_TYPE;
-	desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
-			QI_EIOTLB_DID(did) |
-			QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
-			QI_EIOTLB_TYPE;
-	desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
-			QI_DEV_EIOTLB_SID(sid) |
-			QI_DEV_EIOTLB_QDEP(qdep) |
-			QI_DEIOTLB_TYPE |
-			QI_DEV_IOTLB_PFSID(info->pfsid);
+	qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
+	if (pasid == IOMMU_NO_PASID)
+		qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
+				  MAX_AGAW_PFN_WIDTH, &desc[2]);
+	else
+		qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
+					0, MAX_AGAW_PFN_WIDTH, &desc[2]);
 qi_retry:
 	reinit_completion(&iommu->prq_complete);
 	qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);
-- 
2.43.0

Re: [PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[Joel Granados]

On Thu, Oct 31, 2024 at 05:51:39PM +0800, Lu Baolu wrote:
> As this iommu driver now supports page faults for requests without
> PASID, page requests should be drained when a domain is removed from
> the RID2PASID entry.
> 
> This results in the intel_iommu_drain_pasid_prq() call being moved to
> intel_pasid_tear_down_entry(). This indicates that when a translation
> is removed from any PASID entry and the PRI has been enabled on the
> device, page requests are flushed in the domain detachment path.
> 
> The intel_iommu_drain_pasid_prq() helper has been modified to support
> sending device TLB invalidation requests for both PASID and non-PASID
> cases.
> 
> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
> ---
>  drivers/iommu/intel/iommu.c |  1 -
>  drivers/iommu/intel/pasid.c |  1 +
>  drivers/iommu/intel/prq.c   | 22 +++++++---------------
>  3 files changed, 8 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> index 87a3563dfe54..3878f35be09d 100644
> --- a/drivers/iommu/intel/iommu.c
> +++ b/drivers/iommu/intel/iommu.c
> @@ -4069,7 +4069,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
>  	intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
>  	kfree(dev_pasid);
>  	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
> -	intel_iommu_drain_pasid_prq(dev, pasid);
>  }
>  
>  static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
> diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
> index 7e76062a7ad2..31665fb62e1c 100644
> --- a/drivers/iommu/intel/pasid.c
> +++ b/drivers/iommu/intel/pasid.c
> @@ -265,6 +265,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
>  		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>  
>  	devtlb_invalidation_with_pasid(iommu, dev, pasid);
> +	intel_iommu_drain_pasid_prq(dev, pasid);
>  }
>  
>  /*
> diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
> index 3c50c848893f..ae7f6f34462f 100644
> --- a/drivers/iommu/intel/prq.c
> +++ b/drivers/iommu/intel/prq.c
> @@ -66,12 +66,8 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>  	struct pci_dev *pdev;
>  	int head, tail;
>  	u16 sid, did;
> -	int qdep;
>  
>  	info = dev_iommu_priv_get(dev);
> -	if (WARN_ON(!info || !dev_is_pci(dev)))
You do this on purpose because
1. It is not possible to go through this path without an iommu set in
   the device
2. PRI within the intel driver is only relevant for PCI
3. There will always be a struct device_domain_info related to the
   device iommu when doing PRI.
right?

Best

-- 

Joel Granados

Re: [PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[Baolu Lu]

On 10/31/24 22:56, Joel Granados wrote:
> On Thu, Oct 31, 2024 at 05:51:39PM +0800, Lu Baolu wrote:
>> As this iommu driver now supports page faults for requests without
>> PASID, page requests should be drained when a domain is removed from
>> the RID2PASID entry.
>>
>> This results in the intel_iommu_drain_pasid_prq() call being moved to
>> intel_pasid_tear_down_entry(). This indicates that when a translation
>> is removed from any PASID entry and the PRI has been enabled on the
>> device, page requests are flushed in the domain detachment path.
>>
>> The intel_iommu_drain_pasid_prq() helper has been modified to support
>> sending device TLB invalidation requests for both PASID and non-PASID
>> cases.
>>
>> Signed-off-by: Lu Baolu<baolu.lu@linux.intel.com>
>> ---
>>   drivers/iommu/intel/iommu.c |  1 -
>>   drivers/iommu/intel/pasid.c |  1 +
>>   drivers/iommu/intel/prq.c   | 22 +++++++---------------
>>   3 files changed, 8 insertions(+), 16 deletions(-)
>>
>> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
>> index 87a3563dfe54..3878f35be09d 100644
>> --- a/drivers/iommu/intel/iommu.c
>> +++ b/drivers/iommu/intel/iommu.c
>> @@ -4069,7 +4069,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
>>   	intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
>>   	kfree(dev_pasid);
>>   	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
>> -	intel_iommu_drain_pasid_prq(dev, pasid);
>>   }
>>   
>>   static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
>> diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
>> index 7e76062a7ad2..31665fb62e1c 100644
>> --- a/drivers/iommu/intel/pasid.c
>> +++ b/drivers/iommu/intel/pasid.c
>> @@ -265,6 +265,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
>>   		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>>   
>>   	devtlb_invalidation_with_pasid(iommu, dev, pasid);
>> +	intel_iommu_drain_pasid_prq(dev, pasid);
>>   }
>>   
>>   /*
>> diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
>> index 3c50c848893f..ae7f6f34462f 100644
>> --- a/drivers/iommu/intel/prq.c
>> +++ b/drivers/iommu/intel/prq.c
>> @@ -66,12 +66,8 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>>   	struct pci_dev *pdev;
>>   	int head, tail;
>>   	u16 sid, did;
>> -	int qdep;
>>   
>>   	info = dev_iommu_priv_get(dev);
>> -	if (WARN_ON(!info || !dev_is_pci(dev)))
> You do this on purpose because
> 1. It is not possible to go through this path without an iommu set in
>     the device
> 2. PRI within the intel driver is only relevant for PCI
> 3. There will always be a struct device_domain_info related to the
>     device iommu when doing PRI.
> right?

Yes.

--
baolu

Re: [PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[Yi Liu]

On 2024/10/31 17:51, Lu Baolu wrote:
> As this iommu driver now supports page faults for requests without
> PASID, page requests should be drained when a domain is removed from
> the RID2PASID entry.
> 
> This results in the intel_iommu_drain_pasid_prq() call being moved to
> intel_pasid_tear_down_entry(). This indicates that when a translation
> is removed from any PASID entry and the PRI has been enabled on the
> device, page requests are flushed in the domain detachment path.

s/flushed/drained/. 'drained' might suit the spec more than 'flushed'.

Per this change, the callers of the intel_pasid_tear_down_entry() will
get a mandatory PRQ drain. I think this makes sense. But most of the
callers do not follow a prq drain in the before. Will it introduce
regression (especially performance regression)?

> 
> The intel_iommu_drain_pasid_prq() helper has been modified to support
> sending device TLB invalidation requests for both PASID and non-PASID
> cases.

would it be better to rename it to be intel_iommu_drain_prq()? Also, it
makes much sense to separate the PRQ related code into a prq.c. :) It
might be done later though.

> 
> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
> ---
>   drivers/iommu/intel/iommu.c |  1 -
>   drivers/iommu/intel/pasid.c |  1 +
>   drivers/iommu/intel/prq.c   | 22 +++++++---------------
>   3 files changed, 8 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> index 87a3563dfe54..3878f35be09d 100644
> --- a/drivers/iommu/intel/iommu.c
> +++ b/drivers/iommu/intel/iommu.c
> @@ -4069,7 +4069,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
>   	intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
>   	kfree(dev_pasid);
>   	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
> -	intel_iommu_drain_pasid_prq(dev, pasid);
>   }
>   
>   static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
> diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
> index 7e76062a7ad2..31665fb62e1c 100644
> --- a/drivers/iommu/intel/pasid.c
> +++ b/drivers/iommu/intel/pasid.c
> @@ -265,6 +265,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
>   		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>   
>   	devtlb_invalidation_with_pasid(iommu, dev, pasid);
> +	intel_iommu_drain_pasid_prq(dev, pasid);
>   }
>   
>   /*
> diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
> index 3c50c848893f..ae7f6f34462f 100644
> --- a/drivers/iommu/intel/prq.c
> +++ b/drivers/iommu/intel/prq.c
> @@ -66,12 +66,8 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>   	struct pci_dev *pdev;
>   	int head, tail;
>   	u16 sid, did;
> -	int qdep;
>   
>   	info = dev_iommu_priv_get(dev);
> -	if (WARN_ON(!info || !dev_is_pci(dev)))
> -		return;
> -
>   	if (!info->pri_enabled)
>   		return;
>   
> @@ -81,8 +77,6 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>   	sid = PCI_DEVID(info->bus, info->devfn);
>   	did = domain ? domain_id_iommu(domain, iommu) : FLPT_DEFAULT_DID;
>   
> -	qdep = pci_ats_queue_depth(pdev);
> -
>   	/*
>   	 * Check and wait until all pending page requests in the queue are
>   	 * handled by the prq handling thread.
> @@ -114,15 +108,13 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>   	desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
>   			QI_IWD_FENCE |
>   			QI_IWD_TYPE;
> -	desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
> -			QI_EIOTLB_DID(did) |
> -			QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
> -			QI_EIOTLB_TYPE;
> -	desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
> -			QI_DEV_EIOTLB_SID(sid) |
> -			QI_DEV_EIOTLB_QDEP(qdep) |
> -			QI_DEIOTLB_TYPE |
> -			QI_DEV_IOTLB_PFSID(info->pfsid);
> +	qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);

Does it make more sense to submit iotlb desc when @pasid == IOMMU_NO_PASID
just like you've done to the devtlb desc? Spec looks to be unclear on this
part, but it sounds reasonable to use iotlb desc for the
@pasid == IOMMU_NO_PASID case.

> +	if (pasid == IOMMU_NO_PASID)
> +		qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
> +				  MAX_AGAW_PFN_WIDTH, &desc[2]);
> +	else
> +		qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
> +					0, MAX_AGAW_PFN_WIDTH, &desc[2]);

I noticed devtlb_invalidation_with_pasid() passes '64 - VTD_PAGE_SHIFT'
which is equal to MAX_AGAW_PFN_WIDTH. Might be good to consolidate it. :)

>   qi_retry:
>   	reinit_completion(&iommu->prq_complete);
>   	qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);

-- 
Regards,
Yi Liu

Re: [PATCH 1/1] iommu/vt-d: Drain PRQs when domain removed from RID

[Baolu Lu]

On 11/1/24 10:52, Yi Liu wrote:
> On 2024/10/31 17:51, Lu Baolu wrote:
>> As this iommu driver now supports page faults for requests without
>> PASID, page requests should be drained when a domain is removed from
>> the RID2PASID entry.
>>
>> This results in the intel_iommu_drain_pasid_prq() call being moved to
>> intel_pasid_tear_down_entry(). This indicates that when a translation
>> is removed from any PASID entry and the PRI has been enabled on the
>> device, page requests are flushed in the domain detachment path.
> 
> s/flushed/drained/. 'drained' might suit the spec more than 'flushed'.

Done.

> 
> Per this change, the callers of the intel_pasid_tear_down_entry() will
> get a mandatory PRQ drain. I think this makes sense. But most of the
> callers do not follow a prq drain in the before. Will it introduce
> regression (especially performance regression)?

intel_pasid_tear_down_entry() is called only in slow paths, so it should
have minimal performance impact.

>>
>> The intel_iommu_drain_pasid_prq() helper has been modified to support
>> sending device TLB invalidation requests for both PASID and non-PASID
>> cases.
> 
> would it be better to rename it to be intel_iommu_drain_prq()? Also, it
> makes much sense to separate the PRQ related code into a prq.c. :) It
> might be done later though.

Yes, this has already been addressed in another series.

> 
>>
>> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
>> ---
>>   drivers/iommu/intel/iommu.c |  1 -
>>   drivers/iommu/intel/pasid.c |  1 +
>>   drivers/iommu/intel/prq.c   | 22 +++++++---------------
>>   3 files changed, 8 insertions(+), 16 deletions(-)
>>
>> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
>> index 87a3563dfe54..3878f35be09d 100644
>> --- a/drivers/iommu/intel/iommu.c
>> +++ b/drivers/iommu/intel/iommu.c
>> @@ -4069,7 +4069,6 @@ static void intel_iommu_remove_dev_pasid(struct 
>> device *dev, ioasid_t pasid,
>>       intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
>>       kfree(dev_pasid);
>>       intel_pasid_tear_down_entry(iommu, dev, pasid, false);
>> -    intel_iommu_drain_pasid_prq(dev, pasid);
>>   }
>>   static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
>> diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
>> index 7e76062a7ad2..31665fb62e1c 100644
>> --- a/drivers/iommu/intel/pasid.c
>> +++ b/drivers/iommu/intel/pasid.c
>> @@ -265,6 +265,7 @@ void intel_pasid_tear_down_entry(struct 
>> intel_iommu *iommu, struct device *dev,
>>           iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>>       devtlb_invalidation_with_pasid(iommu, dev, pasid);
>> +    intel_iommu_drain_pasid_prq(dev, pasid);
>>   }
>>   /*
>> diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
>> index 3c50c848893f..ae7f6f34462f 100644
>> --- a/drivers/iommu/intel/prq.c
>> +++ b/drivers/iommu/intel/prq.c
>> @@ -66,12 +66,8 @@ void intel_iommu_drain_pasid_prq(struct device 
>> *dev, u32 pasid)
>>       struct pci_dev *pdev;
>>       int head, tail;
>>       u16 sid, did;
>> -    int qdep;
>>       info = dev_iommu_priv_get(dev);
>> -    if (WARN_ON(!info || !dev_is_pci(dev)))
>> -        return;
>> -
>>       if (!info->pri_enabled)
>>           return;
>> @@ -81,8 +77,6 @@ void intel_iommu_drain_pasid_prq(struct device *dev, 
>> u32 pasid)
>>       sid = PCI_DEVID(info->bus, info->devfn);
>>       did = domain ? domain_id_iommu(domain, iommu) : FLPT_DEFAULT_DID;
>> -    qdep = pci_ats_queue_depth(pdev);
>> -
>>       /*
>>        * Check and wait until all pending page requests in the queue are
>>        * handled by the prq handling thread.
>> @@ -114,15 +108,13 @@ void intel_iommu_drain_pasid_prq(struct device 
>> *dev, u32 pasid)
>>       desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
>>               QI_IWD_FENCE |
>>               QI_IWD_TYPE;
>> -    desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
>> -            QI_EIOTLB_DID(did) |
>> -            QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
>> -            QI_EIOTLB_TYPE;
>> -    desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
>> -            QI_DEV_EIOTLB_SID(sid) |
>> -            QI_DEV_EIOTLB_QDEP(qdep) |
>> -            QI_DEIOTLB_TYPE |
>> -            QI_DEV_IOTLB_PFSID(info->pfsid);
>> +    qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
> 
> Does it make more sense to submit iotlb desc when @pasid == IOMMU_NO_PASID
> just like you've done to the devtlb desc? Spec looks to be unclear on this
> part, but it sounds reasonable to use iotlb desc for the
> @pasid == IOMMU_NO_PASID case.

You are right. I will make it like below:

         if (pasid == IOMMU_NO_PASID) {
                 qi_desc_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH, 
&desc[1]);
                 qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
                                   MAX_AGAW_PFN_WIDTH, &desc[2]);
         } else {
                 qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
                 qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, 
info->ats_qdep,
                                         0, MAX_AGAW_PFN_WIDTH, &desc[2]);
         }


> 
>> +    if (pasid == IOMMU_NO_PASID)
>> +        qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
>> +                  MAX_AGAW_PFN_WIDTH, &desc[2]);
>> +    else
>> +        qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
>> +                    0, MAX_AGAW_PFN_WIDTH, &desc[2]);
> 
> I noticed devtlb_invalidation_with_pasid() passes '64 - VTD_PAGE_SHIFT'
> which is equal to MAX_AGAW_PFN_WIDTH. Might be good to consolidate it. :)

Probably cleanup it in a separated patch.

>>   qi_retry:
>>       reinit_completion(&iommu->prq_complete);
>>       qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);
> 

0day reported a compiling issue:

drivers/iommu/intel/prq.c: In function 'intel_iommu_drain_pasid_prq':
 >> drivers/iommu/intel/prq.c:66:25: warning: variable 'pdev' set but 
not used [-Wunused-but-set-variable]
       66 |         struct pci_dev *pdev;
          |                         ^~~~

I will fix it in a v2.

Thanks,
baolu