Re: [PATCH v3 2/3] PCI: Allow ATS to be always on for pre-CXL devices

[Nirmoy Das <nirmoyd@nvidia.com>]

On 08.03.26 21:50, Nirmoy Das wrote:
>
> On 07.03.26 00:41, Nicolin Chen wrote:
>> Some NVIDIA GPU/NIC devices, although don't implement the CXL config 
>> space,
>> they have many CXL-like properties. Call this kind "pre-CXL".
>>
>> Similar to CXL.cache capaiblity, these pre-CXL devices also require 
>> the ATS
>> function even when their RIDs are IOMMU bypassed, i.e. keep ATS 
>> "always on"
>> v.s. "on demand" when a non-zero PASID line gets enabled in SVA use 
>> cases.
>>
>> Introduce pci_dev_specific_ats_always_on() quirk function to scan a 
>> list of
>> IDs for these device. Then, include it pci_ats_always_on().
>>
>> Suggested-by: Jason Gunthorpe <jgg@nvidia.com>
>> Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
>
> Tested-by: Nirmoy Das <nirmoyd@nvidia.com>
>
> Reviewed-by: Nirmoy Das <nirmoy@nvidia.com>


Sent with wrong email address

Reviewed-by: Nirmoy Das <nirmoyd@nvidia.com>
>
>> ---
>>   drivers/pci/pci.h    |  9 +++++++++
>>   drivers/pci/ats.c    |  3 ++-
>>   drivers/pci/quirks.c | 26 ++++++++++++++++++++++++++
>>   3 files changed, 37 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
>> index 13d998fbacce6..13fa71f965900 100644
>> --- a/drivers/pci/pci.h
>> +++ b/drivers/pci/pci.h
>> @@ -1150,6 +1150,15 @@ static inline int 
>> pci_dev_specific_reset(struct pci_dev *dev, bool probe)
>>   }
>>   #endif
>>   +#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_PCI_ATS)
>> +bool pci_dev_specific_ats_always_on(struct pci_dev *dev);
>> +#else
>> +static inline bool pci_dev_specific_ats_always_on(struct pci_dev *dev)
>> +{
>> +    return false;
>> +}
>> +#endif
>> +
>>   #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64)
>>   int acpi_get_rc_resources(struct device *dev, const char *hid, u16 
>> segment,
>>                 struct resource *res);
>> diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c
>> index cf262eb6e6890..e6995b536ad4c 100644
>> --- a/drivers/pci/ats.c
>> +++ b/drivers/pci/ats.c
>> @@ -243,7 +243,8 @@ bool pci_ats_always_on(struct pci_dev *pdev)
>>       if (pdev->is_virtfn)
>>           pdev = pci_physfn(pdev);
>>   -    return pci_cxl_ats_always_on(pdev);
>> +    return pci_cxl_ats_always_on(pdev) ||
>> +           pci_dev_specific_ats_always_on(pdev);
>>   }
>>   EXPORT_SYMBOL_GPL(pci_ats_always_on);
>>   diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
>> index 48946cca4be72..21451e62f284e 100644
>> --- a/drivers/pci/quirks.c
>> +++ b/drivers/pci/quirks.c
>> @@ -5714,6 +5714,32 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 
>> 0x1457, quirk_intel_e2000_no_ats);
>>   DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, 
>> quirk_intel_e2000_no_ats);
>>   DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, 
>> quirk_intel_e2000_no_ats);
>>   DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, 
>> quirk_intel_e2000_no_ats);
>> +
>> +static const struct pci_dev_ats_always_on {
>> +    u16 vendor;
>> +    u16 device;
>> +} pci_dev_ats_always_on[] = {
>> +    /* NVIDIA GPUs */
>> +    { PCI_VENDOR_ID_NVIDIA, 0x2e12, },
>> +    { PCI_VENDOR_ID_NVIDIA, 0x2e2a, },
>> +    { PCI_VENDOR_ID_NVIDIA, 0x2e2b, },
>> +    /* NVIDIA CX10 Family NVlink-C2C */
>> +    { PCI_VENDOR_ID_MELLANOX, 0x2101, },
>> +    { 0 }
>> +};
>> +
>> +/* Some pre-CXL devices require ATS on the RID when it is 
>> IOMMU-bypassed */
>> +bool pci_dev_specific_ats_always_on(struct pci_dev *pdev)
>> +{
>> +    const struct pci_dev_ats_always_on *i;
>> +
>> +    for (i = pci_dev_ats_always_on; i->vendor; i++) {
>> +        if (i->vendor == pdev->vendor && i->device == pdev->device)
>> +            return true;
>> +    }
>> +
>> +    return false;
>> +}
>>   #endif /* CONFIG_PCI_ATS */
>>     /* Freescale PCIe doesn't support MSI in RC mode */
>