Re: [PATCH 1/4] iommu/amd: Check SNP support before enabling IOMMU

["Kalra, Ashish" <ashish.kalra@amd.com>]

Hello Sean,

On 1/24/2025 6:39 PM, Sean Christopherson wrote:
> On Fri, Jan 24, 2025, Ashish Kalra wrote:
>> With discussions with the AMD IOMMU team, here is the AMD IOMMU
>> initialization flow:
> 
> ..
> 
>> IOMMU SNP check
>>   Core IOMMU subsystem init is done during iommu_subsys_init() via
>>   subsys_initcall.  This function does change the DMA mode depending on
>>   kernel config.  Hence, SNP check should be done after subsys_initcall.
>>   That's why its done currently during IOMMU PCI init (IOMMU_PCI_INIT stage).
>>   And for that reason snp_rmptable_init() is currently invoked via
>>   device_initcall().
>>  
>> The summary is that we cannot move snp_rmptable_init() to subsys_initcall as
>> core IOMMU subsystem gets initialized via subsys_initcall.
> 
> Just explicitly invoke RMP initialization during IOMMU SNP setup.  Pretending
> there's no connection when snp_rmptable_init() checks amd_iommu_snp_en and has
> a comment saying it needs to come after IOMMU SNP setup is ridiculous.
> 

Thanks for the suggestion and the patch, i have tested it works for all cases
and scenarios. I will post the next version of the patch-set based on this
patch.

Ashish

> Compile tested only.
> 
> ---
> From: Sean Christopherson <seanjc@google.com>
> Date: Fri, 24 Jan 2025 16:25:58 -0800
> Subject: [PATCH] x86/sev: iommu/amd: Explicitly init SNP's RMP table during
>  IOMMU SNP setup
> 
> Explicitly initialize the RMP table during IOMMU SNP setup, as there is a
> hard dependency on the IOMMU being configured first, and dancing around
> the dependency with initcall shenanigans and a comment is all kinds of
> stupid.
> 
> The RMP is blatantly not a device; initializing it via a device_initcall()
> is confusing and "works" only because of dumb luck: due to kernel build
> order, when the the PSP driver is built-in, its effective device_initcall()
> just so happens to be invoked after snp_rmptable_init().
> 
> That all falls apart if the order is changed in any way.  E.g. if KVM
> is built-in and attempts to access the RMP during its device_initcall(),
> chaos ensues.
> 
> Signed-off-by: Sean Christopherson <seanjc@google.com>
> ---
>  arch/x86/include/asm/sev.h |  1 +
>  arch/x86/virt/svm/sev.c    | 25 ++++++++-----------------
>  drivers/iommu/amd/init.c   |  7 ++++++-
>  3 files changed, 15 insertions(+), 18 deletions(-)
> 
> diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
> index 91f08af31078..30da0fc15923 100644
> --- a/arch/x86/include/asm/sev.h
> +++ b/arch/x86/include/asm/sev.h
> @@ -503,6 +503,7 @@ static inline void snp_kexec_begin(void) { }
>  
>  #ifdef CONFIG_KVM_AMD_SEV
>  bool snp_probe_rmptable_info(void);
> +int __init snp_rmptable_init(void);
>  int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level);
>  void snp_dump_hva_rmpentry(unsigned long address);
>  int psmash(u64 pfn);
> diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
> index 9a6a943d8e41..d932aa21340b 100644
> --- a/arch/x86/virt/svm/sev.c
> +++ b/arch/x86/virt/svm/sev.c
> @@ -189,19 +189,19 @@ void __init snp_fixup_e820_tables(void)
>   * described in the SNP_INIT_EX firmware command description in the SNP
>   * firmware ABI spec.
>   */
> -static int __init snp_rmptable_init(void)
> +int __init snp_rmptable_init(void)
>  {
>  	u64 max_rmp_pfn, calc_rmp_sz, rmptable_size, rmp_end, val;
>  	void *rmptable_start;
>  
> -	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
> -		return 0;
> +	if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP)))
> +		return -ENOSYS;
>  
> -	if (!amd_iommu_snp_en)
> -		goto nosnp;
> +	if (WARN_ON_ONCE(!amd_iommu_snp_en))
> +		return -ENOSYS;
>  
>  	if (!probed_rmp_size)
> -		goto nosnp;
> +		return -ENOSYS;
>  
>  	rmp_end = probed_rmp_base + probed_rmp_size - 1;
>  
> @@ -218,13 +218,13 @@ static int __init snp_rmptable_init(void)
>  	if (calc_rmp_sz > probed_rmp_size) {
>  		pr_err("Memory reserved for the RMP table does not cover full system RAM (expected 0x%llx got 0x%llx)\n",
>  		       calc_rmp_sz, probed_rmp_size);
> -		goto nosnp;
> +		return -ENOSYS;
>  	}
>  
>  	rmptable_start = memremap(probed_rmp_base, probed_rmp_size, MEMREMAP_WB);
>  	if (!rmptable_start) {
>  		pr_err("Failed to map RMP table\n");
> -		goto nosnp;
> +		return -ENOMEM;
>  	}
>  
>  	/*
> @@ -261,17 +261,8 @@ static int __init snp_rmptable_init(void)
>  	crash_kexec_post_notifiers = true;
>  
>  	return 0;
> -
> -nosnp:
> -	cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
> -	return -ENOSYS;
>  }
>  
> -/*
> - * This must be called after the IOMMU has been initialized.
> - */
> -device_initcall(snp_rmptable_init);
> -
>  static struct rmpentry *get_rmpentry(u64 pfn)
>  {
>  	if (WARN_ON_ONCE(pfn > rmptable_max_pfn))
> diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
> index 0e0a531042ac..d00530156a72 100644
> --- a/drivers/iommu/amd/init.c
> +++ b/drivers/iommu/amd/init.c
> @@ -3171,7 +3171,7 @@ static bool __init detect_ivrs(void)
>  	return true;
>  }
>  
> -static void iommu_snp_enable(void)
> +static __init void iommu_snp_enable(void)
>  {
>  #ifdef CONFIG_KVM_AMD_SEV
>  	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
> @@ -3196,6 +3196,11 @@ static void iommu_snp_enable(void)
>  		goto disable_snp;
>  	}
>  
> +	if (snp_rmptable_init()) {
> +		pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n");
> +		goto disable_snp;
> +	}
> +
>  	pr_info("IOMMU SNP support enabled.\n");
>  	return;
>  
> 
> base-commit: ac80076177131f6e3291737c851a6fe32cc03fd3