[PATCH] arm64: mm: Support memory hotplug in a Realm

[PATCH] arm64: mm: Support memory hotplug in a Realm

[Jean-Philippe Brucker]

Call the RIPAS change functions when adding and removing memory to/from
a Realm.

The host initiates hotplug and the guest uses a range of predefined
guest-physical addresses to install the new memory blocks. This patch
then switches the RIPAS of those blocks from EMPTY to RAM. The guest can
then online the blocks and use them.

Unplug is initiated by the host as well. After offlining the blocks
the guest removes the mapping, and this patch switches the RIPAS from
RAM to EMPTY.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
To test this with a CCA-enabled host and virtio-mem:
    $ qemu-system-aarch64
        -m 1G,maxmem=1T
        -M confidential-guest-support=rme0 -object rme-guest,id=rme0
        -object memory-backend-ram,id=mem0,size=1G
        -device virtio-mem-pci,id=hpm0,memdev=mem0,node=0
        ...

Make offlining more likely to succeed:
    # echo online_movable > /sys/devices/system/memory/auto_online_blocks

Plug and unplug some memory (2M granule)
    (qemu) qom-set hpm0 requested-size 1G
    (qemu) qom-set hpm0 requested-size 256M

Or via ACPI hotplug (requires EDK2 with Realm support)
    (qemu) object_add memory-backend-ram,id=mem1,size=1G
    (qemu) device_add pc-dimm,id=dimm1,memdev=mem1
    (qemu) device_del dimm1
    (qemu) object_del mem1
---
 arch/arm64/include/asm/rsi.h |  3 +++
 arch/arm64/kernel/rsi.c      | 23 +++++++++++++++++++++++
 arch/arm64/mm/mmu.c          | 11 +++++++++--
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/rsi.h b/arch/arm64/include/asm/rsi.h
index 88b50d660e85a..5ffc7e40a1f32 100644
--- a/arch/arm64/include/asm/rsi.h
+++ b/arch/arm64/include/asm/rsi.h
@@ -18,6 +18,9 @@ void __init arm64_rsi_init(void);
 
 bool arm64_rsi_is_protected(phys_addr_t base, size_t size);
 
+int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end);
+int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end);
+
 static inline bool is_realm_world(void)
 {
 	return static_branch_unlikely(&rsi_present);
diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
index c64a06f58c0bc..b983b85f03dfb 100644
--- a/arch/arm64/kernel/rsi.c
+++ b/arch/arm64/kernel/rsi.c
@@ -138,6 +138,29 @@ static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot)
 	return 0;
 }
 
+/*
+ * Convert a range of IPAs from EMPTY to RAM
+ */
+int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end)
+{
+	if (!is_realm_world())
+		return 0;
+
+	return rsi_set_memory_range_protected(start, end);
+}
+
+/*
+ * Convert a range of IPAs from RAM to EMPTY. The pages will be wiped by
+ * UNDELEGATE before being returned to the host.
+ */
+int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end)
+{
+	if (!is_realm_world())
+		return 0;
+
+	return rsi_set_memory_range_shared(start, end);
+}
+
 void __init arm64_rsi_init(void)
 {
 	if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index b8d37eb037fcc..095b094e8a82f 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -44,6 +44,7 @@
 #include <asm/tlbflush.h>
 #include <asm/pgalloc.h>
 #include <asm/kfence.h>
+#include <asm/rsi.h>
 
 #define NO_BLOCK_MAPPINGS	BIT(0)
 #define NO_CONT_MAPPINGS	BIT(1)
@@ -1874,6 +1875,10 @@ int arch_add_memory(int nid, u64 start, u64 size,
 
 	VM_BUG_ON(!mhp_range_allowed(start, size, true));
 
+	ret = arm64_rsi_add_memory(start, start + size);
+	if (ret)
+		return ret;
+
 	if (force_pte_mapping())
 		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
 
@@ -1885,10 +1890,11 @@ int arch_add_memory(int nid, u64 start, u64 size,
 
 	ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
 			   params);
-	if (ret)
+	if (ret) {
 		__remove_pgd_mapping(swapper_pg_dir,
 				     __phys_to_virt(start), size);
-	else {
+		WARN_ON(arm64_rsi_remove_memory(start, start + size));
+	} else {
 		/* Address of hotplugged memory can be smaller */
 		max_pfn = max(max_pfn, PFN_UP(start + size));
 		max_low_pfn = max_pfn;
@@ -1904,6 +1910,7 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 
 	__remove_pages(start_pfn, nr_pages, altmap);
 	__remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
+	WARN_ON(arm64_rsi_remove_memory(start, start + size));
 }
 
 /*

base-commit: 3a8660878839faadb4f1a6dd72c3179c1df56787
-- 
2.51.0

Re: [PATCH] arm64: mm: Support memory hotplug in a Realm

[Will Deacon]

On Mon, Oct 13, 2025 at 04:09:39PM +0100, Jean-Philippe Brucker wrote:
> Call the RIPAS change functions when adding and removing memory to/from
> a Realm.
> 
> The host initiates hotplug and the guest uses a range of predefined
> guest-physical addresses to install the new memory blocks. This patch
> then switches the RIPAS of those blocks from EMPTY to RAM. The guest can
> then online the blocks and use them.
> 
> Unplug is initiated by the host as well. After offlining the blocks
> the guest removes the mapping, and this patch switches the RIPAS from
> RAM to EMPTY.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> ---
> To test this with a CCA-enabled host and virtio-mem:
>     $ qemu-system-aarch64
>         -m 1G,maxmem=1T
>         -M confidential-guest-support=rme0 -object rme-guest,id=rme0
>         -object memory-backend-ram,id=mem0,size=1G
>         -device virtio-mem-pci,id=hpm0,memdev=mem0,node=0
>         ...
> 
> Make offlining more likely to succeed:
>     # echo online_movable > /sys/devices/system/memory/auto_online_blocks
> 
> Plug and unplug some memory (2M granule)
>     (qemu) qom-set hpm0 requested-size 1G
>     (qemu) qom-set hpm0 requested-size 256M
> 
> Or via ACPI hotplug (requires EDK2 with Realm support)
>     (qemu) object_add memory-backend-ram,id=mem1,size=1G
>     (qemu) device_add pc-dimm,id=dimm1,memdev=mem1
>     (qemu) device_del dimm1
>     (qemu) object_del mem1
> ---
>  arch/arm64/include/asm/rsi.h |  3 +++
>  arch/arm64/kernel/rsi.c      | 23 +++++++++++++++++++++++
>  arch/arm64/mm/mmu.c          | 11 +++++++++--
>  3 files changed, 35 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/rsi.h b/arch/arm64/include/asm/rsi.h
> index 88b50d660e85a..5ffc7e40a1f32 100644
> --- a/arch/arm64/include/asm/rsi.h
> +++ b/arch/arm64/include/asm/rsi.h
> @@ -18,6 +18,9 @@ void __init arm64_rsi_init(void);
>  
>  bool arm64_rsi_is_protected(phys_addr_t base, size_t size);
>  
> +int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end);
> +int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end);
> +
>  static inline bool is_realm_world(void)
>  {
>  	return static_branch_unlikely(&rsi_present);
> diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
> index c64a06f58c0bc..b983b85f03dfb 100644
> --- a/arch/arm64/kernel/rsi.c
> +++ b/arch/arm64/kernel/rsi.c
> @@ -138,6 +138,29 @@ static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot)
>  	return 0;
>  }
>  
> +/*
> + * Convert a range of IPAs from EMPTY to RAM
> + */
> +int arm64_rsi_add_memory(phys_addr_t start, phys_addr_t end)
> +{
> +	if (!is_realm_world())
> +		return 0;
> +
> +	return rsi_set_memory_range_protected(start, end);
> +}
> +
> +/*
> + * Convert a range of IPAs from RAM to EMPTY. The pages will be wiped by
> + * UNDELEGATE before being returned to the host.
> + */
> +int arm64_rsi_remove_memory(phys_addr_t start, phys_addr_t end)
> +{
> +	if (!is_realm_world())
> +		return 0;
> +
> +	return rsi_set_memory_range_shared(start, end);
> +}

Probably worth having this in a header so that the static key can be
inlined for the common case where we're not in a realm.

> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
> index b8d37eb037fcc..095b094e8a82f 100644
> --- a/arch/arm64/mm/mmu.c
> +++ b/arch/arm64/mm/mmu.c
> @@ -44,6 +44,7 @@
>  #include <asm/tlbflush.h>
>  #include <asm/pgalloc.h>
>  #include <asm/kfence.h>
> +#include <asm/rsi.h>
>  
>  #define NO_BLOCK_MAPPINGS	BIT(0)
>  #define NO_CONT_MAPPINGS	BIT(1)
> @@ -1874,6 +1875,10 @@ int arch_add_memory(int nid, u64 start, u64 size,
>  
>  	VM_BUG_ON(!mhp_range_allowed(start, size, true));
>  
> +	ret = arm64_rsi_add_memory(start, start + size);
> +	if (ret)
> +		return ret;
> +
>  	if (force_pte_mapping())
>  		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
>  
> @@ -1885,10 +1890,11 @@ int arch_add_memory(int nid, u64 start, u64 size,
>  
>  	ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
>  			   params);
> -	if (ret)
> +	if (ret) {
>  		__remove_pgd_mapping(swapper_pg_dir,
>  				     __phys_to_virt(start), size);
> -	else {
> +		WARN_ON(arm64_rsi_remove_memory(start, start + size));
> +	} else {
>  		/* Address of hotplugged memory can be smaller */
>  		max_pfn = max(max_pfn, PFN_UP(start + size));
>  		max_low_pfn = max_pfn;
> @@ -1904,6 +1910,7 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
>  
>  	__remove_pages(start_pfn, nr_pages, altmap);
>  	__remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
> +	WARN_ON(arm64_rsi_remove_memory(start, start + size));

Hmm. This looks correct to me, but I wonder whether you've pushed the
rsi calls too far down? That is, I can understand this code being called
via virtio-mem and needing to transition the physical pages between
EMPTY/RAM, but these functions have other callers (e.g. via
memunmap_pages()) where it's not at all clear that the RMM should be
called.

Will