Re: [PATCH] arm64: mm: Support memory hotplug in a Realm

[Will Deacon <will@kernel.org>]

On Mon, Oct 13, 2025 at 04:09:39PM +0100, Jean-Philippe Brucker wrote:
> Call the RIPAS change functions when adding and removing memory to/from
> a Realm.
> 
> The host initiates hotplug and the guest uses a range of predefined
> guest-physical addresses to install the new memory blocks. This patch
> then switches the RIPAS of those blocks from EMPTY to RAM. The guest can
> then online the blocks and use them.
> 
> Unplug is initiated by the host as well. After offlining the blocks
> the guest removes the mapping, and this patch switches the RIPAS from
> RAM to EMPTY.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> ---
> To test this with a CCA-enabled host and virtio-mem:
>     $ qemu-system-aarch64
>         -m 1G,maxmem=1T
>         -M confidential-guest-support=rme0 -object rme-guest,id=rme0
>         -object memory-backend-ram,id=mem0,size=1G
>         -device virtio-mem-pci,id=hpm0,memdev=mem0,node=0
>         ...
> 
> Make offlining more likely to succeed:
>     # echo online_movable > /sys/devices/system/memory/auto_online_blocks
> 
> Plug and unplug some memory (2M granule)
>     (qemu) qom-set hpm0 requested-size 1G
>     (qemu) qom-set hpm0 requested-size 256M
> 
> Or via ACPI hotplug (requires EDK2 with Realm support)
>     (qemu) object_add memory-backend-ram,id=mem1,size=1G
>     (qemu) device_add pc-dimm,id=dimm1,memdev=mem1
>     (qemu) device_del dimm1
>     (qemu) object_del mem1
> ---
>  arch/arm64/include/asm/rsi.h |  3 +++
>  arch/arm64/kernel/rsi.c      | 23 +++++++++++++++++++++++
>  arch/arm64/mm/mmu.c          | 11 +++++++++--
>  3 files changed, 35 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/rsi.h b/arch/arm64/include/asm/rsi.h
> index 88b50d660e85a..5ffc7e40a1f32 100644
> --- a/arch/arm64/include/asm/rsi.h
> +++ b/arch/arm64/include/asm/rsi.h
> @@ -18,6 +18,9 @@ void __init arm64_rsi_init(void);
>  
>  bool arm64_rsi_is_protected(phys_addr_t base, size_t size);
>  
> +int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end);
> +int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end);
> +
>  static inline bool is_realm_world(void)
>  {
>  	return static_branch_unlikely(&rsi_present);
> diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
> index c64a06f58c0bc..b983b85f03dfb 100644
> --- a/arch/arm64/kernel/rsi.c
> +++ b/arch/arm64/kernel/rsi.c
> @@ -138,6 +138,29 @@ static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot)
>  	return 0;
>  }
>  
> +/*
> + * Convert a range of IPAs from EMPTY to RAM
> + */
> +int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end)
> +{
> +	if (!is_realm_world())
> +		return 0;
> +
> +	return rsi_set_memory_range_protected(start, end);
> +}
> +
> +/*
> + * Convert a range of IPAs from RAM to EMPTY. The pages will be wiped by
> + * UNDELEGATE before being returned to the host.
> + */
> +int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end)
> +{
> +	if (!is_realm_world())
> +		return 0;
> +
> +	return rsi_set_memory_range_shared(start, end);
> +}

Probably worth having this in a header so that the static key can be
inlined for the common case where we're not in a realm.

> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
> index b8d37eb037fcc..095b094e8a82f 100644
> --- a/arch/arm64/mm/mmu.c
> +++ b/arch/arm64/mm/mmu.c
> @@ -44,6 +44,7 @@
>  #include <asm/tlbflush.h>
>  #include <asm/pgalloc.h>
>  #include <asm/kfence.h>
> +#include <asm/rsi.h>
>  
>  #define NO_BLOCK_MAPPINGS	BIT(0)
>  #define NO_CONT_MAPPINGS	BIT(1)
> @@ -1874,6 +1875,10 @@ int arch_add_memory(int nid, u64 start, u64 size,
>  
>  	VM_BUG_ON(!mhp_range_allowed(start, size, true));
>  
> +	ret = arm64_rsi_add_memory(start, start + size);
> +	if (ret)
> +		return ret;
> +
>  	if (force_pte_mapping())
>  		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
>  
> @@ -1885,10 +1890,11 @@ int arch_add_memory(int nid, u64 start, u64 size,
>  
>  	ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
>  			   params);
> -	if (ret)
> +	if (ret) {
>  		__remove_pgd_mapping(swapper_pg_dir,
>  				     __phys_to_virt(start), size);
> -	else {
> +		WARN_ON(arm64_rsi_remove_memory(start, start + size));
> +	} else {
>  		/* Address of hotplugged memory can be smaller */
>  		max_pfn = max(max_pfn, PFN_UP(start + size));
>  		max_low_pfn = max_pfn;
> @@ -1904,6 +1910,7 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
>  
>  	__remove_pages(start_pfn, nr_pages, altmap);
>  	__remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
> +	WARN_ON(arm64_rsi_remove_memory(start, start + size));

Hmm. This looks correct to me, but I wonder whether you've pushed the
rsi calls too far down? That is, I can understand this code being called
via virtio-mem and needing to transition the physical pages between
EMPTY/RAM, but these functions have other callers (e.g. via
memunmap_pages()) where it's not at all clear that the RMM should be
called.

Will