[PATCH 01/11] dma-mapping: add dma_alloc_attrs_node() and dma_alloc_coherent_node()

[Jakub Kicinski <kuba@kernel.org>]

Coherent DMA memory is always allocated on the NUMA node of the device.
That's the right choice for memory mostly accessed by the device, but
some coherent allocations are accessed by the CPU on every operation.
For example a NIC's descriptor and completion rings are written and
read by the CPU servicing the queue, which may well be on a different
node than the device. Placing such rings on the device's node means
the CPU accesses remote memory in the fast path.

Drivers work around this today by temporarily changing the node
of the device with set_dev_node() around dma_alloc_coherent()
(mlx4, mlx5, ixgbe, xgbe). This is racy against anything else
looking at the device's node at the same time.

Add dma_alloc_attrs_node() which takes the node explicitly, and
a dma_alloc_coherent_node() wrapper. dma_alloc_attrs() now passes
dev_to_node().

The node is only a preference, the same as for kmalloc_node().
Nothing takes it into account yet, following patches wire it up
in dma-direct and iommu-dma. Allocations from fixed pools and from
dma_map_ops->alloc implementations will keep ignoring it.

Add an explicit new export instead of using static inline wrappers
because we want the new API to be GPL-only.

Reviewed-by: Björn Töpel <bjorn@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
---
v1:
 - move before the patches wiring up the node in dma-direct, iommu-dma
   and CMA (Robin)
 - dma-api.rst: drop the NUMA_NO_NODE suggestion (Björn)
 - docs: describe how the node is handled in the patches wiring it up,
   drop the mention of dma_free_coherent()
 - explain the separate GPL-only export in the commit message (Marek)
rfc: https://lore.kernel.org/20261027022528.633359-5-kuba@kernel.org
---
 Documentation/core-api/dma-api.rst | 13 +++++++++++++
 include/linux/dma-mapping.h        | 14 ++++++++++++++
 kernel/dma/mapping.c               | 26 ++++++++++++++++++++++++--
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index 69d4ebe4f96d..5d079507ffa1 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -57,6 +57,19 @@ The flag parameter allows the caller to specify the ``GFP_`` flags (see
 kmalloc()) for the allocation (the implementation may ignore flags that affect
 the location of the returned memory, like GFP_DMA).
 
+::
+
+	void *
+	dma_alloc_coherent_node(struct device *dev, size_t size,
+				dma_addr_t *dma_handle, gfp_t flag, int node)
+
+Same as dma_alloc_coherent(), but places the memory on the given NUMA node
+rather than on the node of the device. This is useful for memory accessed
+frequently by both the device and a CPU on a different node, for example
+descriptor rings serviced by that CPU.
+
+Like with kmalloc_node() the node is only a preference.
+
 ::
 
 	void
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index efde0e5a58dc..f8529f761823 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -188,6 +188,8 @@ void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
 		enum dma_data_direction dir, unsigned long attrs);
 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		gfp_t flag, unsigned long attrs);
+void *dma_alloc_attrs_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node);
 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
 		dma_addr_t dma_handle, unsigned long attrs);
 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
@@ -273,6 +275,11 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
 {
 	return NULL;
 }
+static inline void *dma_alloc_attrs_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node)
+{
+	return NULL;
+}
 static inline void dma_free_attrs(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
 {
@@ -626,6 +633,13 @@ static inline void *dma_alloc_coherent(struct device *dev, size_t size,
 			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
 }
 
+static inline void *dma_alloc_coherent_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t gfp, int node)
+{
+	return dma_alloc_attrs_node(dev, size, dma_handle, gfp,
+			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0, node);
+}
+
 static inline void dma_free_coherent(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_handle)
 {
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 82e701648d60..1b1cd3bc4a40 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -630,8 +630,22 @@ u64 dma_get_required_mask(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dma_get_required_mask);
 
-void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
-		gfp_t flag, unsigned long attrs)
+/**
+ * dma_alloc_attrs_node - allocate coherent memory on a given NUMA node
+ * @dev: device to allocate the memory for
+ * @size: size of the allocation
+ * @dma_handle: returns the DMA address of the allocation
+ * @flag: GFP flags for the allocation
+ * @attrs: DMA_ATTR_* flags for the allocation
+ * @node: NUMA node to allocate the memory on, NUMA_NO_NODE for no preference
+ *
+ * Same as dma_alloc_attrs() but places the memory on @node rather than on the
+ * node of @dev. Like with kmalloc_node() the node is only a preference.
+ *
+ * Return: the CPU address of the allocation, or NULL on failure.
+ */
+void *dma_alloc_attrs_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs, int node)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	void *cpu_addr;
@@ -681,6 +695,14 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);
 	return cpu_addr;
 }
+EXPORT_SYMBOL_GPL(dma_alloc_attrs_node);
+
+void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
+		gfp_t flag, unsigned long attrs)
+{
+	return dma_alloc_attrs_node(dev, size, dma_handle, flag, attrs,
+				    dev_to_node(dev));
+}
 EXPORT_SYMBOL(dma_alloc_attrs);
 
 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
-- 
2.56.0