Re: [PATCH v6 03/14] vdso/datastore: Allocate data pages dynamically

["Christophe Leroy (CS GROUP)" <chleroy@kernel.org>]

Le 04/03/2026 à 08:49, Thomas Weißschuh a écrit :
> Allocating the data pages as part of the kernel image does not work on
> SPARC. The MMU will through a fault when userspace tries to access them.
> 
> Allocate the data pages through the page allocator instead.
> Unused pages in the vDSO VMA are still allocated to keep the virtual
> addresses aligned. Switch the mapping from PFNs to 'struct page' as that
> is required for dynamically allocated pages.
> This also aligns the allocation of the datapages with the code
> pages and is a prerequisite for mlockall() support.
> 
> VM_MIXEDMAP is necessary for the call to vmf_insert_page() in the timens
> prefault path to work.
> 
> The data pages need to be order-0, non-compound pages so that the
> mapping to userspace and the different orderings work.
> 
> These pages are also used by the timekeeping, random pool and
> architecture initialization code. Some of these are running before the
> page allocator is available. To keep these subsytems working without
> changes, introduce early, statically data storage which will then
> replaced by the real one as soon as that is available.
> 
> Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>

Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>

> ---
>   include/linux/vdso_datastore.h |  6 +++
>   init/main.c                    |  2 +
>   lib/vdso/datastore.c           | 92 +++++++++++++++++++++++++++---------------
>   3 files changed, 68 insertions(+), 32 deletions(-)
> 
> diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
> index a91fa24b06e0..0b530428db71 100644
> --- a/include/linux/vdso_datastore.h
> +++ b/include/linux/vdso_datastore.h
> @@ -2,9 +2,15 @@
>   #ifndef _LINUX_VDSO_DATASTORE_H
>   #define _LINUX_VDSO_DATASTORE_H
>   
> +#ifdef CONFIG_HAVE_GENERIC_VDSO
>   #include <linux/mm_types.h>
>   
>   extern const struct vm_special_mapping vdso_vvar_mapping;
>   struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
>   
> +void __init vdso_setup_data_pages(void);
> +#else /* !CONFIG_HAVE_GENERIC_VDSO */
> +static inline void vdso_setup_data_pages(void) { }
> +#endif /* CONFIG_HAVE_GENERIC_VDSO */
> +
>   #endif /* _LINUX_VDSO_DATASTORE_H */
> diff --git a/init/main.c b/init/main.c
> index 1cb395dd94e4..de867b2693d2 100644
> --- a/init/main.c
> +++ b/init/main.c
> @@ -105,6 +105,7 @@
>   #include <linux/ptdump.h>
>   #include <linux/time_namespace.h>
>   #include <linux/unaligned.h>
> +#include <linux/vdso_datastore.h>
>   #include <net/net_namespace.h>
>   
>   #include <asm/io.h>
> @@ -1119,6 +1120,7 @@ void start_kernel(void)
>   	srcu_init();
>   	hrtimers_init();
>   	softirq_init();
> +	vdso_setup_data_pages();
>   	timekeeping_init();
>   	time_init();
>   
> diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
> index 7377fcb6e1df..faebf5b7cd6e 100644
> --- a/lib/vdso/datastore.c
> +++ b/lib/vdso/datastore.c
> @@ -1,52 +1,79 @@
>   // SPDX-License-Identifier: GPL-2.0-only
>   
> -#include <linux/linkage.h>
> +#include <linux/gfp.h>
> +#include <linux/init.h>
>   #include <linux/mm.h>
>   #include <linux/time_namespace.h>
>   #include <linux/types.h>
>   #include <linux/vdso_datastore.h>
>   #include <vdso/datapage.h>
>   
> -/*
> - * The vDSO data page.
> - */
> +static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
> +
>   #ifdef CONFIG_GENERIC_GETTIMEOFDAY
> -static union {
> -	struct vdso_time_data	data;
> -	u8			page[PAGE_SIZE];
> -} vdso_time_data_store __page_aligned_data;
> -struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
> -static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
> +struct vdso_time_data *vdso_k_time_data __refdata =
> +	(void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
> +
> +static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
>   #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
>   
>   #ifdef CONFIG_VDSO_GETRANDOM
> -static union {
> -	struct vdso_rng_data	data;
> -	u8			page[PAGE_SIZE];
> -} vdso_rng_data_store __page_aligned_data;
> -struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
> -static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
> +struct vdso_rng_data *vdso_k_rng_data __refdata =
> +	(void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
> +
> +static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
>   #endif /* CONFIG_VDSO_GETRANDOM */
>   
>   #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
> -static union {
> -	struct vdso_arch_data	data;
> -	u8			page[VDSO_ARCH_DATA_SIZE];
> -} vdso_arch_data_store __page_aligned_data;
> -struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
> +struct vdso_arch_data *vdso_k_arch_data __refdata =
> +	(void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
>   #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
>   
> +void __init vdso_setup_data_pages(void)
> +{
> +	unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
> +	struct page *pages;
> +
> +	/*
> +	 * Allocate the data pages dynamically. SPARC does not support mapping
> +	 * static pages to be mapped into userspace.
> +	 * It is also a requirement for mlockall() support.
> +	 *
> +	 * Do not use folios. In time namespaces the pages are mapped in a different order
> +	 * to userspace, which is not handled by the folio optimizations in finish_fault().
> +	 */
> +	pages = alloc_pages(GFP_KERNEL, order);
> +	if (!pages)
> +		panic("Unable to allocate VDSO storage pages");
> +
> +	/* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
> +	split_page(pages, order);
> +
> +	/* Move the data already written by other subsystems to the new pages */
> +	memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
> +
> +	if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
> +		vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
> +
> +	if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
> +		vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
> +
> +	if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
> +		vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
> +}
> +
>   static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
>   			     struct vm_area_struct *vma, struct vm_fault *vmf)
>   {
> -	struct page *timens_page = find_timens_vvar_page(vma);
> -	unsigned long pfn;
> +	struct page *page, *timens_page;
> +
> +	timens_page = find_timens_vvar_page(vma);
>   
>   	switch (vmf->pgoff) {
>   	case VDSO_TIME_PAGE_OFFSET:
>   		if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
>   			return VM_FAULT_SIGBUS;
> -		pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
> +		page = virt_to_page(vdso_k_time_data);
>   		if (timens_page) {
>   			/*
>   			 * Fault in VVAR page too, since it will be accessed
> @@ -56,10 +83,10 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
>   			vm_fault_t err;
>   
>   			addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
> -			err = vmf_insert_pfn(vma, addr, pfn);
> +			err = vmf_insert_page(vma, addr, page);
>   			if (unlikely(err & VM_FAULT_ERROR))
>   				return err;
> -			pfn = page_to_pfn(timens_page);
> +			page = timens_page;
>   		}
>   		break;
>   	case VDSO_TIMENS_PAGE_OFFSET:
> @@ -72,24 +99,25 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
>   		 */
>   		if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
>   			return VM_FAULT_SIGBUS;
> -		pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
> +		page = virt_to_page(vdso_k_time_data);
>   		break;
>   	case VDSO_RNG_PAGE_OFFSET:
>   		if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
>   			return VM_FAULT_SIGBUS;
> -		pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data));
> +		page = virt_to_page(vdso_k_rng_data);
>   		break;
>   	case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
>   		if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
>   			return VM_FAULT_SIGBUS;
> -		pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) +
> -			vmf->pgoff - VDSO_ARCH_PAGES_START;
> +		page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
>   		break;
>   	default:
>   		return VM_FAULT_SIGBUS;
>   	}
>   
> -	return vmf_insert_pfn(vma, vmf->address, pfn);
> +	get_page(page);
> +	vmf->page = page;
> +	return 0;
>   }
>   
>   const struct vm_special_mapping vdso_vvar_mapping = {
> @@ -101,7 +129,7 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned
>   {
>   	return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
>   					VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
> -					VM_PFNMAP | VM_SEALED_SYSMAP,
> +					VM_MIXEDMAP | VM_SEALED_SYSMAP,
>   					&vdso_vvar_mapping);
>   }
>   
>