[PATCH 03/26] dma-buf: Add dma_buf_mapping_attach()

[Alistair Popple <apopple@nvidia.com>]

From: Jason Gunthorpe <jgg@nvidia.com>

Introduce a new attach function that accepts the list of importer
supported mapping types from the caller. Turn dma_buf_dynamic_attach() and
dma_buf_attach() into simple wrappers calling this new function with a
compatibility mapping type list that only includes sgt.

dma_buf_mapping_attach() checks if the exporter is mapping aware and calls
its ops->match_mapping() function to pick up the exporter match list and
call dma_buf_match_mapping().

If unaware it will use dma_buf_sgt_exp_compat_match as a compatibility
matchlist that uses the unaware exporter's dma_buf_ops callbacks.

The resulting match set is stored in attach->matches with the first entry,
the exporter's preferred type, in attach->map_type.

In effect attach->map_type is always available and always makes sense
regardless of what combination of aware/unaware importer/exporter is used.

The attachment's importer side (dmabuf, importer_ops, importer_priv) is
initialised before match_mapping() runs so that exporters can consult it,
for example to recognise an importer driven by the same driver. The
dma_resv lock is not held at this point, so match_mapping() may only look
at properties that cannot change for the lifetime of the attachment.

For compatibility with unaware drivers copy the sgt matching data into the
attach->dev and peer2peer.

If the exporter sets exporter_requires_p2p then only the following are
allowed:
 - dma_buf_dynamic_attach() with importer_ops->allow_peer2peer = true
 - dma_buf_mapping_attach() with a
    DMA_BUF_IMAPPING_SGT(xx, exporter_requires_p2p=true)

Other combinations are blocked.

Exporters that want to behave differently based on the importer's
capability can declare exporter_requires_p2p=false and check
attach->map_type.sgt_data.importer_accepts_p2p. Or they can declare two
SGT exporters with different map/unmap functions.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
[ rebase context change due to upstream removing a WARN_ON - apopple ]
[ rebase context change due to upstream switching to kzalloc_obj() - apopple ]
[ use dma_buf_sgt_dma_device(attach) for trace points - apopple ]
[ initialise the attachment before calling match_mapping(), reported by Karl Mehltretter - apopple ]
[ kdoc updates - apopple ]
Signed-off-by: Alistair Popple <apopple@nvidia.com>
---
 drivers/dma-buf/dma-buf.c      | 103 +++++++++++++++++++++++++++------
 include/linux/dma-buf.h        |  27 +++++++++
 include/trace/events/dma_buf.h |   2 +-
 3 files changed, 112 insertions(+), 20 deletions(-)

diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 4c9add51f9ef..16a7fbb8aaab 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -14,6 +14,7 @@
 #include <linux/fs.h>
 #include <linux/slab.h>
 #include <linux/dma-buf.h>
+#include <linux/dma-buf-mapping.h>
 #include <linux/dma-fence.h>
 #include <linux/dma-fence-unwrap.h>
 #include <linux/anon_inodes.h>
@@ -714,11 +715,19 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
 	int ret;
 
 	if (WARN_ON(!exp_info->priv || !exp_info->ops
-		    || !exp_info->ops->map_dma_buf
-		    || !exp_info->ops->unmap_dma_buf
 		    || !exp_info->ops->release))
 		return ERR_PTR(-EINVAL);
 
+	if (exp_info->ops->match_mapping) {
+		if (WARN_ON(exp_info->ops->map_dma_buf ||
+			    exp_info->ops->unmap_dma_buf))
+			return ERR_PTR(-EINVAL);
+	} else {
+		if (WARN_ON(!exp_info->ops->map_dma_buf ||
+			    !exp_info->ops->unmap_dma_buf))
+			return ERR_PTR(-EINVAL);
+	}
+
 	if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin))
 		return ERR_PTR(-EINVAL);
 
@@ -1007,9 +1016,10 @@ dma_buf_pin_on_map(struct dma_buf_attachment *attach)
  */
 
 /**
- * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list
+ * dma_buf_mapping_attach - Add the device to dma_buf's attachments list
  * @dmabuf:		[in]	buffer to attach device to.
- * @dev:		[in]	device to be attached.
+ * @importer_matches:	[in]	mapping types supported by the importer
+ * @match_len:		[in]	length of @importer_matches
  * @importer_ops:	[in]	importer operations for the attachment
  * @importer_priv:	[in]	importer private pointer for the attachment
  *
@@ -1025,31 +1035,55 @@ dma_buf_pin_on_map(struct dma_buf_attachment *attach)
  * error code wrapped into a pointer on failure.
  *
  * Note that this can fail if the backing storage of @dmabuf is in a place not
- * accessible to @dev, and cannot be moved to a more suitable place. This is
- * indicated with the error code -EBUSY.
+ * accessible to any importers, and cannot be moved to a more suitable place.
+ * This is indicated with the error code -EBUSY.
+ *
+ * This records the attachment's match set, i.e. the intersection of types
+ * supported by both importer and exporter, in &dma_buf_attachment.matches.
+ * Later attachments by other importers never change the set but may affect
+ * what type the exporter selects when the attachment is first mapped or
+ * pinned.
  */
-struct dma_buf_attachment *
-dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
-		       const struct dma_buf_attach_ops *importer_ops,
-		       void *importer_priv)
+struct dma_buf_attachment *dma_buf_mapping_attach(
+	struct dma_buf *dmabuf, struct dma_buf_mapping_match *importer_matches,
+	size_t match_len, const struct dma_buf_attach_ops *importer_ops,
+	void *importer_priv)
 {
+	struct dma_buf_match_args match_args = {
+		.dmabuf = dmabuf,
+		.imp_matches = importer_matches,
+		.imp_len = match_len,
+	};
 	struct dma_buf_attachment *attach;
 	int ret;
 
-	if (WARN_ON(!dmabuf || !dev))
+	if (WARN_ON(!dmabuf))
 		return ERR_PTR(-EINVAL);
 
 	attach = kzalloc_obj(*attach);
 	if (!attach)
 		return ERR_PTR(-ENOMEM);
 
-	attach->dev = dev;
 	attach->dmabuf = dmabuf;
-	if (importer_ops)
-		attach->peer2peer = importer_ops->allow_peer2peer;
 	attach->importer_ops = importer_ops;
 	attach->importer_priv = importer_priv;
 
+	/*
+	 * The exporter's match_mapping() may look at the importer side of the
+	 * attachment, so initialise it before the match.
+	 */
+	match_args.attach = attach;
+	if (dmabuf->ops->match_mapping) {
+		ret = dmabuf->ops->match_mapping(&match_args);
+		if (ret)
+			goto err_attach;
+	} else {
+		ret = dma_buf_match_mapping(&match_args,
+					    &dma_buf_sgt_exp_compat_match, 1);
+		if (ret)
+			goto err_attach;
+	}
+
 	if (dmabuf->ops->attach) {
 		ret = dmabuf->ops->attach(dmabuf, attach);
 		if (ret)
@@ -1060,7 +1094,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 	dma_resv_unlock(dmabuf->resv);
 
 	DMA_BUF_TRACE(trace_dma_buf_dynamic_attach, dmabuf, attach,
-		dma_buf_attachment_is_dynamic(attach), dev);
+		dma_buf_attachment_is_dynamic(attach),
+		dma_buf_sgt_dma_device(attach));
 
 	return attach;
 
@@ -1068,23 +1103,53 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 	kfree(attach);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, "DMA_BUF");
+EXPORT_SYMBOL_NS_GPL(dma_buf_mapping_attach, "DMA_BUF");
 
 /**
- * dma_buf_attach - Wrapper for dma_buf_dynamic_attach
+ * dma_buf_attach - Wrapper for dma_buf_mapping_attach
  * @dmabuf:	[in]	buffer to attach device to.
  * @dev:	[in]	device to be attached.
  *
- * Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static
+ * Wrapper to call dma_buf_mapping_attach() for drivers which still use a static
  * mapping.
  */
 struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
 					  struct device *dev)
 {
-	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);
+	struct dma_buf_mapping_match sgt_match[] = {
+		DMA_BUF_IMAPPING_SGT(dev, DMA_SGT_NO_P2P),
+	};
+
+	return dma_buf_mapping_attach(dmabuf, sgt_match, ARRAY_SIZE(sgt_match),
+				      NULL, NULL);
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_attach, "DMA_BUF");
 
+/**
+ * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list
+ * @dmabuf:		[in]	buffer to attach device to.
+ * @dev:		[in]	device to be attached.
+ * @importer_ops:	[in]	importer operations for the attachment
+ * @importer_priv:	[in]	importer private pointer for the attachment
+ *
+ * Wrapper to call dma_buf_mapping_attach() for drivers which only support SGT.
+ */
+struct dma_buf_attachment *
+dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
+		       const struct dma_buf_attach_ops *importer_ops,
+		       void *importer_priv)
+{
+	struct dma_buf_mapping_match sgt_match[] = {
+		DMA_BUF_IMAPPING_SGT(dev, importer_ops->allow_peer2peer ?
+						  DMA_SGT_IMPORTER_ACCEPTS_P2P :
+						  DMA_SGT_NO_P2P),
+	};
+
+	return dma_buf_mapping_attach(dmabuf, sgt_match, ARRAY_SIZE(sgt_match),
+				      importer_ops, importer_priv);
+}
+EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, "DMA_BUF");
+
 /**
  * dma_buf_detach - Remove the given attachment from dmabuf's attachments list
  * @dmabuf:	[in]	buffer to detach from.
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 3e82ec59d5ce..83ab2902a23b 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -29,6 +29,7 @@ struct dma_buf;
 struct dma_buf_attachment;
 struct dma_buf_mapping_type;
 struct dma_buf_mapping_exp_ops;
+struct dma_buf_match_args;
 
 
 /*
@@ -315,6 +316,27 @@ struct dma_buf_ops {
 
 	int (*vmap)(struct dma_buf *dmabuf, struct iosys_map *map);
 	void (*vunmap)(struct dma_buf *dmabuf, struct iosys_map *map);
+
+	/**
+	 * @match_mapping:
+	 *
+	 * Called during attach, without the dma_resv lock held. Allows the
+	 * exporter to build its own exporter struct dma_buf_mapping_match[]
+	 * and call dma_buf_match_mapping() to record every mutually
+	 * acceptable type in &dma_buf_attachment.matches in the exporter's
+	 * order of preference.
+	 *
+	 * The match set is fixed for the lifetime of the attachment, so every
+	 * input to the decision must hold for that long: either static for the
+	 * dma_buf (the importer's list, PCI P2P topology, buffer creation
+	 * flags) or runtime state the exporter pins until @detach, for example
+	 * a runtime PM reference.
+	 *
+	 * It must not depend on the set of attachments or on current buffer
+	 * placement as matches are static and buffer placement can change at
+	 * any time until the final location is resolved during map/pin.
+	 */
+	int (*match_mapping)(struct dma_buf_match_args *args);
 };
 
 /**
@@ -606,6 +628,11 @@ struct dma_buf_attachment *
 dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 		       const struct dma_buf_attach_ops *importer_ops,
 		       void *importer_priv);
+struct dma_buf_attachment *dma_buf_mapping_attach(
+	struct dma_buf *dmabuf, struct dma_buf_mapping_match *importer_matches,
+	size_t match_len, const struct dma_buf_attach_ops *importer_ops,
+	void *importer_priv);
+
 void dma_buf_detach(struct dma_buf *dmabuf,
 		    struct dma_buf_attachment *attach);
 int dma_buf_pin(struct dma_buf_attachment *attach);
diff --git a/include/trace/events/dma_buf.h b/include/trace/events/dma_buf.h
index 3bb88d05bcc8..13d3a82856b6 100644
--- a/include/trace/events/dma_buf.h
+++ b/include/trace/events/dma_buf.h
@@ -40,7 +40,7 @@ DECLARE_EVENT_CLASS(dma_buf_attach_dev,
 	TP_ARGS(dmabuf, attach, is_dynamic, dev),
 
 	TP_STRUCT__entry(
-		__string(	dev_name,			dev_name(dev))
+		__string(	dev_name,			dev ? dev_name(dev) : "none")
 		__string(	exp_name,			dmabuf->exp_name)
 		__field(	size_t,				size)
 		__field(	ino_t,				ino)
-- 
2.54.0