[PATCH 05/26] dma-buf: Let the exporter select mapping type at pin/map

[Alistair Popple <apopple@nvidia.com>]

The match set negotiated at attach determines which types an attachment
could possibly be served with. Which one should actually be served
depends on where the buffer is and other attachments when an importer
first pins or maps. So far the first entry was used unconditionally,
even if it cannot be served to some importers.

Add dma_buf_ops.select_mapping() under dma_resv from both pin and map
paths before the type's map callback to let the exporter choose which to
serve based on the current placement and list of attachments.

The choice is recorded in attach->selected and does not call
select_mapping again across any number of unmaps and remaps, until
dma_buf_invalidate_mappings() has been delivered to the attachment.

This prevents importer remap from bouncing a buffer between placements
- the type cannot change without the exporter explicitly invalidating
first. In other words attachments that are never invalidated keep the
initially selected type for the life of the attachment.

A failed select or map establishes nothing and leaves the choice open.

Importers never select. The importer's list order is a preference the
exporter may weigh or ignore, and an importer must accept any type it
listed.

Assisted-by: LLM
Signed-off-by: Alistair Popple <apopple@nvidia.com>
---
 drivers/dma-buf/dma-buf-mapping.c | 49 ++++++++++++++-
 drivers/dma-buf/dma-buf.c         | 99 +++++++++++++++++++++----------
 include/linux/dma-buf-mapping.h   |  6 ++
 include/linux/dma-buf.h           | 39 +++++++++++-
 4 files changed, 156 insertions(+), 37 deletions(-)

diff --git a/drivers/dma-buf/dma-buf-mapping.c b/drivers/dma-buf/dma-buf-mapping.c
index d30c0e9ef038..dbcd83b6dcff 100644
--- a/drivers/dma-buf/dma-buf-mapping.c
+++ b/drivers/dma-buf/dma-buf-mapping.c
@@ -270,8 +270,9 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_free_sgt, "DMA_BUF");
  * Walk the exporter's list in order and, for each entry, find the first
  * importer entry of the same type that the type's match() callback accepts.
  * Every such pair is recorded in @args->attach->matches, so the match set
- * preserves the exporter's priority. The first entry becomes the default in
- * @args->attach->map_type.
+ * preserves the exporter's priority. The first entry becomes the default
+ * selection in @args->attach->map_type until &dma_buf_ops.select_mapping
+ * chooses otherwise at pin or map time.
  *
  * Because the exporter list is walked in order, the exporter controls the
  * priority of mapping types.
@@ -321,6 +322,50 @@ int dma_buf_match_mapping(struct dma_buf_match_args *args,
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_match_mapping, "DMA_BUF");
 
+/**
+ * dma_buf_mapping_select - Let the exporter choose the type to map with
+ * @attach: Attachment about to be pinned or mapped
+ *
+ * Called with the dma_resv lock held by dma_buf_pin() and the map functions.
+ * The first call after attach, and after each dma_buf_invalidate_mappings(),
+ * asks &dma_buf_ops.select_mapping to pick an entry of the match set given
+ * the buffer's placement and the other live mappings; exporters without the
+ * callback get the first entry. Every other call returns the previous choice
+ * unchanged, so the mapping type cannot change between unmap and remap without
+ * the exporter having explicitly invalidated.
+ *
+ * Returns: 0 on success or the exporter's error, typically -EBUSY when the
+ * attachment cannot be served without disturbing a mapping that cannot be
+ * invalidated.
+ */
+int dma_buf_mapping_select(struct dma_buf_attachment *attach)
+{
+	const struct dma_buf_mapping_match *match;
+
+	dma_resv_assert_held(attach->dmabuf->resv);
+
+	if (attach->selected)
+		return 0;
+
+	if (WARN_ON(!attach->nr_matches))
+		return -EINVAL;
+
+	if (attach->dmabuf->ops->select_mapping) {
+		match = attach->dmabuf->ops->select_mapping(attach);
+		if (IS_ERR(match))
+			return PTR_ERR(match);
+		if (WARN_ON(match < attach->matches ||
+			    match >= attach->matches + attach->nr_matches))
+			return -EINVAL;
+	} else {
+		match = &attach->matches[0];
+	}
+
+	attach->map_type = *match;
+	attach->selected = true;
+	return 0;
+}
+
 static int dma_buf_sgt_match(struct dma_buf *dmabuf,
 			     const struct dma_buf_mapping_match *exp,
 			     const struct dma_buf_mapping_match *imp)
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 348b47f266b4..03f06c4b957c 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -1201,6 +1201,10 @@ int dma_buf_pin(struct dma_buf_attachment *attach)
 
 	dma_resv_assert_held(dmabuf->resv);
 
+	ret = dma_buf_mapping_select(attach);
+	if (ret)
+		return ret;
+
 	if (dmabuf->ops->pin)
 		ret = dmabuf->ops->pin(attach);
 
@@ -1229,42 +1233,19 @@ void dma_buf_unpin(struct dma_buf_attachment *attach)
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, "DMA_BUF");
 
-/**
- * dma_buf_map_attachment - Returns the scatterlist table of the attachment;
- * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
- * dma_buf_ops.
- * @attach:	[in]	attachment whose scatterlist is to be returned
- * @direction:	[in]	direction of DMA transfer
- *
- * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
- * on error. May return -EINTR if it is interrupted by a signal.
- *
- * On success, the DMA addresses and lengths in the returned scatterlist are
- * PAGE_SIZE aligned.
- *
- * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
- * the underlying backing storage is pinned for as long as a mapping exists,
- * therefore users/importers should not hold onto a mapping for undue amounts of
- * time.
- *
- * Important: Dynamic importers must wait for the exclusive fence of the struct
- * dma_resv attached to the DMA-BUF first.
- */
-struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
-					enum dma_data_direction direction)
+/* The SGT type must have been selected into attach->map_type already */
+static struct sg_table *
+__dma_buf_sgt_map_attachment(struct dma_buf_attachment *attach,
+			     enum dma_data_direction direction)
 {
 	const struct dma_buf_mapping_sgt_exp_ops *sgt_exp_ops =
 		dma_buf_get_sgt_ops(attach);
 	struct sg_table *sg_table;
 	signed long ret;
 
-	might_sleep();
-
-	if (WARN_ON(!attach || !attach->dmabuf || !sgt_exp_ops))
+	if (WARN_ON(!sgt_exp_ops))
 		return ERR_PTR(-EINVAL);
 
-	dma_resv_assert_held(attach->dmabuf->resv);
-
 	if (dma_buf_pin_on_map(attach)) {
 		ret = attach->dmabuf->ops->pin(attach);
 		/*
@@ -1324,6 +1305,52 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
 
 	return sg_table;
 }
+
+/**
+ * dma_buf_map_attachment - Returns the scatterlist table of the attachment;
+ * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
+ * dma_buf_ops.
+ * @attach:	[in]	attachment whose scatterlist is to be returned
+ * @direction:	[in]	direction of DMA transfer
+ *
+ * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
+ * on error. May return -EINTR if it is interrupted by a signal.
+ *
+ * On success, the DMA addresses and lengths in the returned scatterlist are
+ * PAGE_SIZE aligned.
+ *
+ * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
+ * the underlying backing storage is pinned for as long as a mapping exists,
+ * therefore users/importers should not hold onto a mapping for undue amounts of
+ * time.
+ *
+ * Important: Dynamic importers must wait for the exclusive fence of the struct
+ * dma_resv attached to the DMA-BUF first.
+ */
+struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
+					enum dma_data_direction direction)
+{
+	struct sg_table *sg_table;
+	int ret;
+
+	might_sleep();
+
+	if (WARN_ON(!attach || !attach->dmabuf))
+		return ERR_PTR(-EINVAL);
+
+	dma_resv_assert_held(attach->dmabuf->resv);
+
+	ret = dma_buf_mapping_select(attach);
+	if (ret)
+		return ERR_PTR(ret);
+
+	sg_table = __dma_buf_sgt_map_attachment(attach, direction);
+	if (IS_ERR(sg_table))
+		/* Nothing was established, so the choice may be made afresh */
+		attach->selected = false;
+
+	return sg_table;
+}
 EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, "DMA_BUF");
 
 /**
@@ -1465,6 +1492,10 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_attach_revocable, "DMA_BUF");
  *
  * Importers are free to request a new mapping in parallel as this function
  * returns.
+ *
+ * This is also the only point at which an attachment's mapping type
+ * may change after a pin/map: the core keeps the type chosen by
+ * &dma_buf_ops.select_mapping until the attachment has been invalidated here.
  */
 void dma_buf_invalidate_mappings(struct dma_buf *dmabuf)
 {
@@ -1472,10 +1503,14 @@ void dma_buf_invalidate_mappings(struct dma_buf *dmabuf)
 
 	dma_resv_assert_held(dmabuf->resv);
 
-	list_for_each_entry(attach, &dmabuf->attachments, node)
-		if (attach->importer_ops &&
-		    attach->importer_ops->invalidate_mappings)
-			attach->importer_ops->invalidate_mappings(attach);
+	list_for_each_entry(attach, &dmabuf->attachments, node) {
+		if (!attach->importer_ops ||
+		    !attach->importer_ops->invalidate_mappings)
+			continue;
+
+		attach->selected = false;
+		attach->importer_ops->invalidate_mappings(attach);
+	}
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_invalidate_mappings, "DMA_BUF");
 
diff --git a/include/linux/dma-buf-mapping.h b/include/linux/dma-buf-mapping.h
index f9b749af362b..c04fac71726a 100644
--- a/include/linux/dma-buf-mapping.h
+++ b/include/linux/dma-buf-mapping.h
@@ -96,6 +96,12 @@ int dma_buf_match_mapping(struct dma_buf_match_args *args,
 			  const struct dma_buf_mapping_match *exp_mappings,
 			  size_t exp_len);
 
+/*
+ * Internal: Selection is made by exporters through &dma_buf_ops.select_mapping
+ * in dma_buf_pin() or map paths. Importers never call this.
+ */
+int dma_buf_mapping_select(struct dma_buf_attachment *attach);
+
 /*
  * DMA Mapped Scatterlist Type
  *
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 83ab2902a23b..8bd5026eb642 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -337,6 +337,34 @@ struct dma_buf_ops {
 	 * any time until the final location is resolved during map/pin.
 	 */
 	int (*match_mapping)(struct dma_buf_match_args *args);
+
+	/**
+	 * @select_mapping:
+	 *
+	 * Called with the dma_resv lock held from dma_buf_pin() and from the
+	 * map functions, before the type's map callback, to choose which
+	 * entry of &dma_buf_attachment.matches this attachment is mapped
+	 * with. The exporter settles the buffer's placement here, taking all
+	 * live mappings and pinned attachments into account, and returns the
+	 * entry that placement can serve.
+	 *
+	 * The core records the choice and does not ask again, across any
+	 * number of unmaps and remaps, until the exporter has delivered
+	 * &dma_buf_attach_ops.invalidate_mappings to this attachment. The
+	 * exporter must therefore keep the recorded type servable: moving the
+	 * buffer somewhere that type cannot be produced from requires
+	 * invalidating first. Attachments that cannot be invalidated keep
+	 * their first choice for life.
+	 *
+	 * Optional. Without it the first entry of the match set, the
+	 * exporter's most preferred, is used.
+	 *
+	 * Returns: a pointer into &dma_buf_attachment.matches, or an ERR_PTR.
+	 * -EBUSY means the attachment cannot be served without disturbing a
+	 * mapping that cannot be invalidated.
+	 */
+	const struct dma_buf_mapping_match *(*select_mapping)(
+		struct dma_buf_attachment *attach);
 };
 
 /**
@@ -532,12 +560,16 @@ struct dma_buf_attach_ops {
  * @importer_ops: importer operations for this attachment, if provided
  * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
  * @importer_priv: importer specific attachment data.
- * @map_type: The match this attachment is mapped with. Until the exporter
- *            selects otherwise it holds the exporter's default, the first
- *            entry of @matches.
+ * @map_type: The match this attachment is mapped with. Chosen by
+ *            &dma_buf_ops.select_mapping at the first pin or map and kept,
+ *            enforced by the core, until &dma_buf_attach_ops.invalidate_mappings
+ *            is delivered to this attachment. Before the first selection it
+ *            holds the exporter's default, the first entry of @matches.
  * @matches: The match set negotiated at attach time: every mapping type both
  *           sides accept, in the exporter's order of preference.
  * @nr_matches: Number of valid entries in @matches.
+ * @selected: True once @map_type has been chosen by the exporter; cleared
+ *            only by dma_buf_invalidate_mappings().
  *
  * This structure holds the attachment information between the dma_buf buffer
  * and its user device(s). The list contains one attachment struct per device
@@ -559,6 +591,7 @@ struct dma_buf_attachment {
 	struct dma_buf_mapping_match map_type;
 	struct dma_buf_mapping_match matches[DMA_BUF_MAX_MATCHES];
 	unsigned int nr_matches;
+	bool selected;
 };
 
 /**
-- 
2.54.0