Re: [PATCH v6 03/14] vdso/datastore: Allocate data pages dynamically

Christophe Leroy (CS GROUP)
2026-03-04T08:49:17Z

Le 04/03/2026 à 08:49, Thomas Weißschuh a écrit :
> Allocating the data pages as part of the kernel image does not work on
> SPARC. The MMU will through a fault when userspace tries to access them.
>
> Allocate the data pages through the page allocator instead.
> Unused pages in the vDSO VMA are still allocated to keep the virtual
> addresses aligned. Switch the mapping from PFNs to 'struct page' as that
> is required for dynamically allocated pages.
> This also aligns the allocation of the datapages with the code
> pages and is a prerequisite for mlockall() support.
>
> VM_MIXEDMAP is necessary for the call to vmf_insert_page() in the timens
> prefault path to work.
>
> The data pages need to be order-0, non-compound pages so that the
> mapping to userspace and the different orderings work.
>
> These pages are also used by the timekeeping, random pool and
> architecture initialization code. Some of these are running before the
> page allocator is available. To keep these subsytems working without
> changes, introduce early, statically data storage which will then
> replaced by the real one as soon as that is available.
>
> Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>

Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>

> ---
> include/linux/vdso_datastore.h | 6 +++
> init/main.c | 2 +
> lib/vdso/datastore.c | 92 +++++++++++++++++++++++++++---------------
> 3 files changed, 68 insertions(+), 32 deletions(-)
>
> diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
> index a91fa24b06e0..0b530428db71 100644
> --- a/include/linux/vdso_datastore.h
> +++ b/include/linux/vdso_datastore.h
> @@ -2,9 +2,15 @@
> #ifndef _LINUX_VDSO_DATASTORE_H
> #define _LINUX_VDSO_DATASTORE_H
>
> +#ifdef CONFIG_HAVE_GENERIC_VDSO
> #include <linux/mm_types.h>
>
> extern const struct vm_special_mapping vdso_vvar_mapping;
> struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
>
> +void __init vdso_setup_data_pages(void);
> +#else /* !CONFIG_HAVE_GENERIC_VDSO */
> +static inline void vdso_setup_data_pages(void) { }
> +#endif /* CONFIG_HAVE_GENERIC_VDSO */
> +
> #endif /* _LINUX_VDSO_DATASTORE_H */
> diff --git a/init/main.c b/init/main.c
> index 1cb395dd94e4..de867b2693d2 100644
> --- a/init/main.c
> +++ b/init/main.c
> @@ -105,6 +105,7 @@
> #include <linux/ptdump.h>
> #include <linux/time_namespace.h>
> #include <linux/unaligned.h>
> +#include <linux/vdso_datastore.h>
> #include <net/net_namespace.h>
>
> #include <asm/io.h>
> @@ -1119,6 +1120,7 @@ void start_kernel(void)
> srcu_init();
> hrtimers_init();
> softirq_init();
> + vdso_setup_data_pages();
> timekeeping_init();
> time_init();
>
> diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
> index 7377fcb6e1df..faebf5b7cd6e 100644
> --- a/lib/vdso/datastore.c
> +++ b/lib/vdso/datastore.c
> @@ -1,52 +1,79 @@
> // SPDX-License-Identifier: GPL-2.0-only
>
> -#include <linux/linkage.h>
> +#include <linux/gfp.h>
> +#include <linux/init.h>
> #include <linux/mm.h>
> #include <linux/time_namespace.h>
> #include <linux/types.h>
> #include <linux/vdso_datastore.h>
> #include <vdso/datapage.h>
>
> -/*
> - * The vDSO data page.
> - */
> +static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
> +
> #ifdef CONFIG_GENERIC_GETTIMEOFDAY
> -static union {
> - struct vdso_time_data data;
> - u8 page[PAGE_SIZE];
> -} vdso_time_data_store __page_aligned_data;
> -struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
> -static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
> +struct vdso_time_data *vdso_k_time_data __refdata =
> + (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
> +
> +static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
> #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
>
> #ifdef CONFIG_VDSO_GETRANDOM
> -static union {
> - struct vdso_rng_data data;
> - u8 page[PAGE_SIZE];
> -} vdso_rng_data_store __page_aligned_data;
> -struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
> -static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
> +struct vdso_rng_data *vdso_k_rng_data __refdata =
> + (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
> +
> +static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
> #endif /* CONFIG_VDSO_GETRANDOM */
>
> #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
> -static union {
> - struct vdso_arch_data data;
> - u8 page[VDSO_ARCH_DATA_SIZE];
> -} vdso_arch_data_store __page_aligned_data;
> -struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
> +struct vdso_arch_data *vdso_k_arch_data __refdata =
> + (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
> #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
>
> +void __init vdso_setup_data_pages(void)
> +{
> + unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
> + struct page *pages;
> +
> + /*
> + * Allocate the data pages dynamically. SPARC does not support mapping
> + * static pages to be mapped into userspace.
> + * It is also a requirement for mlockall() support.
> + *
> + * Do not use folios. In time namespaces the pages are mapped in a different order
> + * to userspace, which is not handled by the folio optimizations in finish_fault().
> + */
> + pages = alloc_pages(GFP_KERNEL, order);
> + if (!pages)
> + panic("Unable to allocate VDSO storage pages");
> +
> + /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
> + split_page(pages, order);
> +
> + /* Move the data already written by other subsystems to the new pages */
> + memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
> +
> + if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
> + vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
> +
> + if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
> + vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
> +
> + if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
> + vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
> +}
> +
> static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
> struct vm_area_struct *vma, struct vm_fault *vmf)
> {
> - struct page *timens_page = find_timens_vvar_page(vma);
> - unsigned long pfn;
> + struct page *page, *timens_page;
> +
> + timens_page = find_timens_vvar_page(vma);
>
> switch (vmf->pgoff) {
> case VDSO_TIME_PAGE_OFFSET:
> if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
> return VM_FAULT_SIGBUS;
> - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
> + page = virt_to_page(vdso_k_time_data);
> if (timens_page) {
> /*
> * Fault in VVAR page too, since it will be accessed
> @@ -56,10 +83,10 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
> vm_fault_t err;
>
> addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
> - err = vmf_insert_pfn(vma, addr, pfn);
> + err = vmf_insert_page(vma, addr, page);
> if (unlikely(err & VM_FAULT_ERROR))
> return err;
> - pfn = page_to_pfn(timens_page);
> + page = timens_page;
> }
> break;
> case VDSO_TIMENS_PAGE_OFFSET:
> @@ -72,24 +99,25 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
> */
> if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
> return VM_FAULT_SIGBUS;
> - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
> + page = virt_to_page(vdso_k_time_data);
> break;
> case VDSO_RNG_PAGE_OFFSET:
> if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
> return VM_FAULT_SIGBUS;
> - pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data));
> + page = virt_to_page(vdso_k_rng_data);
> break;
> case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
> if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
> return VM_FAULT_SIGBUS;
> - pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) +
> - vmf->pgoff - VDSO_ARCH_PAGES_START;
> + page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
> break;
> default:
> return VM_FAULT_SIGBUS;
> }
>
> - return vmf_insert_pfn(vma, vmf->address, pfn);
> + get_page(page);
> + vmf->page = page;
> + return 0;
> }
>
> const struct vm_special_mapping vdso_vvar_mapping = {
> @@ -101,7 +129,7 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned
> {
> return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
> VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
> - VM_PFNMAP | VM_SEALED_SYSMAP,
> + VM_MIXEDMAP | VM_SEALED_SYSMAP,
> &vdso_vvar_mapping);
> }
>
>