Re: [PATCH 01/11] dma-mapping: add dma_alloc_attrs_node() and dma_alloc_coherent_node()

[Marek Szyprowski <m.szyprowski@samsung.com>]

On 06.11.2026 19:44, Jakub Kicinski wrote:
> Coherent DMA memory is always allocated on the NUMA node of the device.
> That's the right choice for memory mostly accessed by the device, but
> some coherent allocations are accessed by the CPU on every operation.
> For example a NIC's descriptor and completion rings are written and
> read by the CPU servicing the queue, which may well be on a different
> node than the device. Placing such rings on the device's node means
> the CPU accesses remote memory in the fast path.
>
> Drivers work around this today by temporarily changing the node
> of the device with set_dev_node() around dma_alloc_coherent()
> (mlx4, mlx5, ixgbe, xgbe). This is racy against anything else
> looking at the device's node at the same time.
>
> Add dma_alloc_attrs_node() which takes the node explicitly, and
> a dma_alloc_coherent_node() wrapper. dma_alloc_attrs() now passes
> dev_to_node().
>
> The node is only a preference, the same as for kmalloc_node().
> Nothing takes it into account yet, following patches wire it up
> in dma-direct and iommu-dma. Allocations from fixed pools and from
> dma_map_ops->alloc implementations will keep ignoring it.
>
> Add an explicit new export instead of using static inline wrappers
> because we want the new API to be GPL-only.
>
> Reviewed-by: Björn Töpel <bjorn@kernel.org>
> Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>


> ---
> v1:
>  - move before the patches wiring up the node in dma-direct, iommu-dma
>    and CMA (Robin)
>  - dma-api.rst: drop the NUMA_NO_NODE suggestion (Björn)
>  - docs: describe how the node is handled in the patches wiring it up,
>    drop the mention of dma_free_coherent()
>  - explain the separate GPL-only export in the commit message (Marek)
> rfc: https://lore.kernel.org/20261027022528.633359-5-kuba@kernel.org
> ---
>  Documentation/core-api/dma-api.rst | 13 +++++++++++++
>  include/linux/dma-mapping.h        | 14 ++++++++++++++
>  kernel/dma/mapping.c               | 26 ++++++++++++++++++++++++--
>  3 files changed, 51 insertions(+), 2 deletions(-)
>
> diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
> index 69d4ebe4f96d..5d079507ffa1 100644
> --- a/Documentation/core-api/dma-api.rst
> +++ b/Documentation/core-api/dma-api.rst
> @@ -57,6 +57,19 @@ The flag parameter allows the caller to specify the ``GFP_`` flags (see
>  kmalloc()) for the allocation (the implementation may ignore flags that affect
>  the location of the returned memory, like GFP_DMA).
>  
> +::
> +
> +	void *
> +	dma_alloc_coherent_node(struct device *dev, size_t size,
> +				dma_addr_t *dma_handle, gfp_t flag, int node)
> +
> +Same as dma_alloc_coherent(), but places the memory on the given NUMA node
> +rather than on the node of the device. This is useful for memory accessed
> +frequently by both the device and a CPU on a different node, for example
> +descriptor rings serviced by that CPU.
> +
> +Like with kmalloc_node() the node is only a preference.
> +
>  ::
>  
>  	void
> diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
> index efde0e5a58dc..f8529f761823 100644
> --- a/include/linux/dma-mapping.h
> +++ b/include/linux/dma-mapping.h
> @@ -188,6 +188,8 @@ void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
>  		enum dma_data_direction dir, unsigned long attrs);
>  void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
>  		gfp_t flag, unsigned long attrs);
> +void *dma_alloc_attrs_node(struct device *dev, size_t size,
> +		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node);
>  void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
>  		dma_addr_t dma_handle, unsigned long attrs);
>  void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
> @@ -273,6 +275,11 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
>  {
>  	return NULL;
>  }
> +static inline void *dma_alloc_attrs_node(struct device *dev, size_t size,
> +		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node)
> +{
> +	return NULL;
> +}
>  static inline void dma_free_attrs(struct device *dev, size_t size,
>  		void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
>  {
> @@ -626,6 +633,13 @@ static inline void *dma_alloc_coherent(struct device *dev, size_t size,
>  			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
>  }
>  
> +static inline void *dma_alloc_coherent_node(struct device *dev, size_t size,
> +		dma_addr_t *dma_handle, gfp_t gfp, int node)
> +{
> +	return dma_alloc_attrs_node(dev, size, dma_handle, gfp,
> +			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0, node);
> +}
> +
>  static inline void dma_free_coherent(struct device *dev, size_t size,
>  		void *cpu_addr, dma_addr_t dma_handle)
>  {
> diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
> index 82e701648d60..1b1cd3bc4a40 100644
> --- a/kernel/dma/mapping.c
> +++ b/kernel/dma/mapping.c
> @@ -630,8 +630,22 @@ u64 dma_get_required_mask(struct device *dev)
>  }
>  EXPORT_SYMBOL_GPL(dma_get_required_mask);
>  
> -void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
> -		gfp_t flag, unsigned long attrs)
> +/**
> + * dma_alloc_attrs_node - allocate coherent memory on a given NUMA node
> + * @dev: device to allocate the memory for
> + * @size: size of the allocation
> + * @dma_handle: returns the DMA address of the allocation
> + * @flag: GFP flags for the allocation
> + * @attrs: DMA_ATTR_* flags for the allocation
> + * @node: NUMA node to allocate the memory on, NUMA_NO_NODE for no preference
> + *
> + * Same as dma_alloc_attrs() but places the memory on @node rather than on the
> + * node of @dev. Like with kmalloc_node() the node is only a preference.
> + *
> + * Return: the CPU address of the allocation, or NULL on failure.
> + */
> +void *dma_alloc_attrs_node(struct device *dev, size_t size,
> +		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node)
>  {
>  	const struct dma_map_ops *ops = get_dma_ops(dev);
>  	void *cpu_addr;
> @@ -681,6 +695,14 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
>  	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);
>  	return cpu_addr;
>  }
> +EXPORT_SYMBOL_GPL(dma_alloc_attrs_node);
> +
> +void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
> +		gfp_t flag, unsigned long attrs)
> +{
> +	return dma_alloc_attrs_node(dev, size, dma_handle, flag, attrs,
> +				    dev_to_node(dev));
> +}
>  EXPORT_SYMBOL(dma_alloc_attrs);
>  
>  void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,

Best regards
-- 
Marek Szyprowski, PhD
Samsung R&D Institute Poland