[PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Lu Baolu]

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to
probe path") changed the PCI ATS enablement logic to run earlier,
specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing
the kernel to panic with the following message and call trace:

 Kernel panic - not syncing: DMAR hardware is malfunctioning
 CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
 Call Trace:
  <TASK>
  dump_stack_lvl+0x6f/0xb0
  dump_stack+0x10/0x16
  panic+0x10a/0x2b7
  iommu_enable_translation.cold+0xc/0xc
  intel_iommu_init+0xe39/0xec0
  ? trace_hardirqs_on+0x1e/0xd0
  ? __pfx_pci_iommu_init+0x10/0x10
  pci_iommu_init+0xd/0x40
  do_one_initcall+0x5b/0x390
  kernel_init_freeable+0x26d/0x2b0
  ? __pfx_kernel_init+0x10/0x10
  kernel_init+0x15/0x120
  ret_from_fork+0x35/0x60
  ? __pfx_kernel_init+0x10/0x10
  ret_from_fork_asm+0x1a/0x30
 RIP: 1f0f:0x0
 Code: Unable to access opcode bytes at 0xffffffffffffffd6.
 RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
      1f0f2e6600000000
 RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
      2e66000000000084
 RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
      00841f0f2e660000
 RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
      000000841f0f2e66
 R10: 0000000000841f0f R11: 2e66000000000084 R12:
      000000841f0f2e66
 R13: 0000000000841f0f R14: 2e66000000000084 R15:
      1f0f2e6600000000
  </TASK>
 ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---

Fix this by reverting the timing change for ATS enablement introduced by
the offending commit and restoring the previous behavior.

Fixes: 5518f239aff1 ("iommu/vt-d: Move scalable mode ATS enablement to probe path")
Reported-by: Jarkko Nikula <jarkko.nikula@linux.intel.com>
Closes: https://lore.kernel.org/linux-iommu/01b9c72f-460d-4f77-b696-54c6825babc9@linux.intel.com/
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
 drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index b29da2d96d0b..e60b699e7b8c 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -3785,6 +3785,22 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 
 	intel_iommu_debugfs_create_dev(info);
 
+	return &iommu->iommu;
+free_table:
+	intel_pasid_free_table(dev);
+clear_rbtree:
+	device_rbtree_remove(info);
+free:
+	kfree(info);
+
+	return ERR_PTR(ret);
+}
+
+static void intel_iommu_probe_finalize(struct device *dev)
+{
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct intel_iommu *iommu = info->iommu;
+
 	/*
 	 * The PCIe spec, in its wisdom, declares that the behaviour of the
 	 * device is undefined if you enable PASID support after ATS support.
@@ -3792,22 +3808,12 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 	 * we can't yet know if we're ever going to use it.
 	 */
 	if (info->pasid_supported &&
-	    !pci_enable_pasid(pdev, info->pasid_supported & ~1))
+	    !pci_enable_pasid(to_pci_dev(dev), info->pasid_supported & ~1))
 		info->pasid_enabled = 1;
 
-	if (sm_supported(iommu))
+	if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev))
 		iommu_enable_pci_ats(info);
 	iommu_enable_pci_pri(info);
-
-	return &iommu->iommu;
-free_table:
-	intel_pasid_free_table(dev);
-clear_rbtree:
-	device_rbtree_remove(info);
-free:
-	kfree(info);
-
-	return ERR_PTR(ret);
 }
 
 static void intel_iommu_release_device(struct device *dev)
@@ -4391,6 +4397,7 @@ const struct iommu_ops intel_iommu_ops = {
 	.domain_alloc_sva	= intel_svm_domain_alloc,
 	.domain_alloc_nested	= intel_iommu_domain_alloc_nested,
 	.probe_device		= intel_iommu_probe_device,
+	.probe_finalize		= intel_iommu_probe_finalize,
 	.release_device		= intel_iommu_release_device,
 	.get_resv_regions	= intel_iommu_get_resv_regions,
 	.device_group		= intel_iommu_device_group,
-- 
2.43.0

Re: [PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Jarkko Nikula]

On 4/16/25 10:36 AM, Lu Baolu wrote:
> Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to
> probe path") changed the PCI ATS enablement logic to run earlier,
> specifically before the default domain attachment.
> 
> On some client platforms, this change resulted in boot failures, causing
> the kernel to panic with the following message and call trace:
> 
>   Kernel panic - not syncing: DMAR hardware is malfunctioning
>   CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
>   Call Trace:
>    <TASK>
>    dump_stack_lvl+0x6f/0xb0
>    dump_stack+0x10/0x16
>    panic+0x10a/0x2b7
>    iommu_enable_translation.cold+0xc/0xc
>    intel_iommu_init+0xe39/0xec0
>    ? trace_hardirqs_on+0x1e/0xd0
>    ? __pfx_pci_iommu_init+0x10/0x10
>    pci_iommu_init+0xd/0x40
>    do_one_initcall+0x5b/0x390
>    kernel_init_freeable+0x26d/0x2b0
>    ? __pfx_kernel_init+0x10/0x10
>    kernel_init+0x15/0x120
>    ret_from_fork+0x35/0x60
>    ? __pfx_kernel_init+0x10/0x10
>    ret_from_fork_asm+0x1a/0x30
>   RIP: 1f0f:0x0
>   Code: Unable to access opcode bytes at 0xffffffffffffffd6.
>   RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
>        1f0f2e6600000000
>   RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
>        2e66000000000084
>   RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
>        00841f0f2e660000
>   RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
>        000000841f0f2e66
>   R10: 0000000000841f0f R11: 2e66000000000084 R12:
>        000000841f0f2e66
>   R13: 0000000000841f0f R14: 2e66000000000084 R15:
>        1f0f2e6600000000
>    </TASK>
>   ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
> 
> Fix this by reverting the timing change for ATS enablement introduced by
> the offending commit and restoring the previous behavior.
> 
> Fixes: 5518f239aff1 ("iommu/vt-d: Move scalable mode ATS enablement to probe path")
> Reported-by: Jarkko Nikula <jarkko.nikula@linux.intel.com>
> Closes: https://lore.kernel.org/linux-iommu/01b9c72f-460d-4f77-b696-54c6825babc9@linux.intel.com/
> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
> ---
>   drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++------------
>   1 file changed, 19 insertions(+), 12 deletions(-)
> 
Tested-by: Jarkko Nikula <jarkko.nikula@linux.intel.com>

RE: [PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Tian, Kevin]

> From: Lu Baolu <baolu.lu@linux.intel.com>
> Sent: Wednesday, April 16, 2025 3:36 PM
> 
> Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement
> to
> probe path") changed the PCI ATS enablement logic to run earlier,
> specifically before the default domain attachment.
> 
> On some client platforms, this change resulted in boot failures, causing
> the kernel to panic with the following message and call trace:
> 
>  Kernel panic - not syncing: DMAR hardware is malfunctioning
>  CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
>  Call Trace:
>   <TASK>
>   dump_stack_lvl+0x6f/0xb0
>   dump_stack+0x10/0x16
>   panic+0x10a/0x2b7
>   iommu_enable_translation.cold+0xc/0xc
>   intel_iommu_init+0xe39/0xec0
>   ? trace_hardirqs_on+0x1e/0xd0
>   ? __pfx_pci_iommu_init+0x10/0x10
>   pci_iommu_init+0xd/0x40
>   do_one_initcall+0x5b/0x390
>   kernel_init_freeable+0x26d/0x2b0
>   ? __pfx_kernel_init+0x10/0x10
>   kernel_init+0x15/0x120
>   ret_from_fork+0x35/0x60
>   ? __pfx_kernel_init+0x10/0x10
>   ret_from_fork_asm+0x1a/0x30
>  RIP: 1f0f:0x0
>  Code: Unable to access opcode bytes at 0xffffffffffffffd6.
>  RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
>       1f0f2e6600000000
>  RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
>       2e66000000000084
>  RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
>       00841f0f2e660000
>  RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
>       000000841f0f2e66
>  R10: 0000000000841f0f R11: 2e66000000000084 R12:
>       000000841f0f2e66
>  R13: 0000000000841f0f R14: 2e66000000000084 R15:
>       1f0f2e6600000000
>   </TASK>
>  ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
> 
> Fix this by reverting the timing change for ATS enablement introduced by
> the offending commit and restoring the previous behavior.
> 

it's unclear how this timing is related to the dumped stack. Is there
more detail how they are related?

Re: [PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Baolu Lu]

On 4/17/25 10:23, Tian, Kevin wrote:
>> From: Lu Baolu <baolu.lu@linux.intel.com>
>> Sent: Wednesday, April 16, 2025 3:36 PM
>>
>> Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement
>> to
>> probe path") changed the PCI ATS enablement logic to run earlier,
>> specifically before the default domain attachment.
>>
>> On some client platforms, this change resulted in boot failures, causing
>> the kernel to panic with the following message and call trace:
>>
>>   Kernel panic - not syncing: DMAR hardware is malfunctioning
>>   CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
>>   Call Trace:
>>    <TASK>
>>    dump_stack_lvl+0x6f/0xb0
>>    dump_stack+0x10/0x16
>>    panic+0x10a/0x2b7
>>    iommu_enable_translation.cold+0xc/0xc
>>    intel_iommu_init+0xe39/0xec0
>>    ? trace_hardirqs_on+0x1e/0xd0
>>    ? __pfx_pci_iommu_init+0x10/0x10
>>    pci_iommu_init+0xd/0x40
>>    do_one_initcall+0x5b/0x390
>>    kernel_init_freeable+0x26d/0x2b0
>>    ? __pfx_kernel_init+0x10/0x10
>>    kernel_init+0x15/0x120
>>    ret_from_fork+0x35/0x60
>>    ? __pfx_kernel_init+0x10/0x10
>>    ret_from_fork_asm+0x1a/0x30
>>   RIP: 1f0f:0x0
>>   Code: Unable to access opcode bytes at 0xffffffffffffffd6.
>>   RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
>>        1f0f2e6600000000
>>   RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
>>        2e66000000000084
>>   RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
>>        00841f0f2e660000
>>   RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
>>        000000841f0f2e66
>>   R10: 0000000000841f0f R11: 2e66000000000084 R12:
>>        000000841f0f2e66
>>   R13: 0000000000841f0f R14: 2e66000000000084 R15:
>>        1f0f2e6600000000
>>    </TASK>
>>   ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
>>
>> Fix this by reverting the timing change for ATS enablement introduced by
>> the offending commit and restoring the previous behavior.
>>
> 
> it's unclear how this timing is related to the dumped stack. Is there
> more detail how they are related?
> 

I'm not sure, but I'm trying to find a machine and get more information.
Anyway, let's revert the change and remove the boot regression first.

Thanks,
baolu

RE: [PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Tian, Kevin]

> From: Baolu Lu <baolu.lu@linux.intel.com>
> Sent: Thursday, April 17, 2025 10:46 AM
> 
> On 4/17/25 10:23, Tian, Kevin wrote:
> >> From: Lu Baolu <baolu.lu@linux.intel.com>
> >> Sent: Wednesday, April 16, 2025 3:36 PM
> >>
> >> Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS
> enablement
> >> to
> >> probe path") changed the PCI ATS enablement logic to run earlier,
> >> specifically before the default domain attachment.
> >>
> >> On some client platforms, this change resulted in boot failures, causing
> >> the kernel to panic with the following message and call trace:
> >>
> >>   Kernel panic - not syncing: DMAR hardware is malfunctioning
> >>   CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
> >>   Call Trace:
> >>    <TASK>
> >>    dump_stack_lvl+0x6f/0xb0
> >>    dump_stack+0x10/0x16
> >>    panic+0x10a/0x2b7
> >>    iommu_enable_translation.cold+0xc/0xc
> >>    intel_iommu_init+0xe39/0xec0
> >>    ? trace_hardirqs_on+0x1e/0xd0
> >>    ? __pfx_pci_iommu_init+0x10/0x10
> >>    pci_iommu_init+0xd/0x40
> >>    do_one_initcall+0x5b/0x390
> >>    kernel_init_freeable+0x26d/0x2b0
> >>    ? __pfx_kernel_init+0x10/0x10
> >>    kernel_init+0x15/0x120
> >>    ret_from_fork+0x35/0x60
> >>    ? __pfx_kernel_init+0x10/0x10
> >>    ret_from_fork_asm+0x1a/0x30
> >>   RIP: 1f0f:0x0
> >>   Code: Unable to access opcode bytes at 0xffffffffffffffd6.
> >>   RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
> >>        1f0f2e6600000000
> >>   RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
> >>        2e66000000000084
> >>   RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
> >>        00841f0f2e660000
> >>   RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
> >>        000000841f0f2e66
> >>   R10: 0000000000841f0f R11: 2e66000000000084 R12:
> >>        000000841f0f2e66
> >>   R13: 0000000000841f0f R14: 2e66000000000084 R15:
> >>        1f0f2e6600000000
> >>    </TASK>
> >>   ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
> >>
> >> Fix this by reverting the timing change for ATS enablement introduced by
> >> the offending commit and restoring the previous behavior.
> >>
> >
> > it's unclear how this timing is related to the dumped stack. Is there
> > more detail how they are related?
> >
> 
> I'm not sure, but I'm trying to find a machine and get more information.
> Anyway, let's revert the change and remove the boot regression first.
> 

I'm fine with this fix for regression but let's do investigate more.

Reviewed-by: Kevin Tian <kevin.tian@intel.com>

Re: [PATCH 1/1] iommu/vt-d: Revert ATS timing change to fix boot failure

[Joerg Roedel]

On Wed, Apr 16, 2025 at 03:36:08PM +0800, Lu Baolu wrote:
> Fixes: 5518f239aff1 ("iommu/vt-d: Move scalable mode ATS enablement to probe path")
> Reported-by: Jarkko Nikula <jarkko.nikula@linux.intel.com>
> Closes: https://lore.kernel.org/linux-iommu/01b9c72f-460d-4f77-b696-54c6825babc9@linux.intel.com/
> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
> ---
>  drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++------------
>  1 file changed, 19 insertions(+), 12 deletions(-)

Applied for -rc, thanks.