[RFC PATCH 8/8] drm/xe: Allow mapping DMA-BUF buffer objects connected through IAF

[Maarten Lankhorst <dev@lankhorst.se>]

IAF allows multiple xe devices to import VRAM from each other.
It's mapped as local VRAM, but outside the local device range.

To support this, Xe is changed to allow a special case of
same-implementation importing with direction = DMA_NONE.

If this DMA direction is used, the physical VRAM addresses are exported,
and can be imported into a xe_vm with the device memory bit set.

In order to prevent bugs, we ensure that this is only allowed for
xe <-> xe imports/exports, and only when the VRAM flag is set.

If any of the steps required fails for IAF-import, the default
non-IAF import path is attempted.

Signed-off-by: Maarten Lankhorst <dev@lankhorst.se>
---
 drivers/gpu/drm/xe/xe_bo.c           | 85 ++++++++++++++++++++++++----
 drivers/gpu/drm/xe/xe_bo.h           |  2 +
 drivers/gpu/drm/xe/xe_dma_buf.c      | 27 ++++++++-
 drivers/gpu/drm/xe/xe_dma_buf.h      |  1 +
 drivers/gpu/drm/xe/xe_ggtt.c         |  4 +-
 drivers/gpu/drm/xe/xe_iaf.c          | 42 ++++++++++++++
 drivers/gpu/drm/xe/xe_iaf.h          | 16 ++++++
 drivers/gpu/drm/xe/xe_pt.c           |  4 ++
 drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 23 +++++---
 drivers/gpu/drm/xe/xe_vm.c           |  2 +-
 10 files changed, 184 insertions(+), 22 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 4e39188a021ab..13f0c685a012f 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -24,6 +24,7 @@
 #include "xe_drm_client.h"
 #include "xe_ggtt.h"
 #include "xe_gt.h"
+#include "xe_iaf.h"
 #include "xe_map.h"
 #include "xe_migrate.h"
 #include "xe_pm.h"
@@ -82,6 +83,11 @@ bool mem_type_is_vram(u32 mem_type)
 	return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
 }
 
+bool xe_bo_is_iaf(struct xe_bo *bo)
+{
+	return bo->flags & XE_BO_FLAG_IAF;
+}
+
 static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
 {
 	return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
@@ -656,6 +662,68 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
 	return ret;
 }
 
+static void bo_unmap_dma_buf(struct xe_bo *bo)
+{
+	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
+	struct dma_buf_attachment *attach = bo->ttm.base.import_attach;
+	struct dma_buf *dmabuf;
+
+	if (xe_bo_is_iaf(bo))
+		dir = DMA_NONE;
+
+	dma_buf_unmap_attachment(attach, bo->ttm.sg, dir);
+	bo->ttm.sg = NULL;
+
+	if (xe_bo_is_iaf(bo)) {
+		/* Cleanup IAF bindings */
+		dmabuf = attach->dmabuf;
+		xe_iaf_mapping_put(xe_bo_device(gem_to_xe_bo(dmabuf->priv))->iaf);
+		xe_iaf_mapping_put(xe_bo_device(bo)->iaf);
+		bo->flags &= ~XE_BO_FLAG_IAF;
+	}
+}
+
+static struct sg_table *bo_map_dma_buf(struct xe_bo *bo)
+{
+	struct dma_buf_attachment *attach = bo->ttm.base.import_attach;
+	struct dma_buf *dmabuf = attach->dmabuf;
+	struct sg_table *sg;
+
+	if (is_xe_dma_buf(dmabuf)) {
+		struct xe_device *src = xe_bo_device(gem_to_xe_bo(dmabuf->priv));
+		struct xe_device *dst = xe_bo_device(bo);
+		int err;
+
+		if (!xe_iaf_fabrics_connected(dst->iaf, src->iaf))
+			goto no_iaf;
+
+		err = xe_iaf_mapping_get(dst->iaf);
+		if (err) {
+			drm_warn(&dst->drm, "Getting DST IAF mapping ref failed with %pe", ERR_PTR(err));
+			goto no_iaf;
+		}
+
+		err = xe_iaf_mapping_get(src->iaf);
+		if (err) {
+			drm_warn(&src->drm, "Getting SRC IAF mapping ref failed with %pe", ERR_PTR(err));
+			goto err_put_dst;
+		}
+
+		sg = dma_buf_map_attachment(attach, DMA_NONE);
+		if (!IS_ERR(sg)) {
+			bo->flags |= XE_BO_FLAG_IAF;
+			return sg;
+		}
+
+		xe_iaf_mapping_put(src->iaf);
+err_put_dst:
+		xe_iaf_mapping_put(dst->iaf);
+	}
+
+no_iaf:
+	return dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+}
+
 /*
  * The dma-buf map_attachment() / unmap_attachment() is hooked up here.
  * Note that unmapping the attachment is deferred to the next
@@ -665,9 +733,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
  * backing store out. Should that assumption not hold, then we will be able
  * to unconditionally call unmap_attachment() when moving out to system.
  */
-static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
-			     struct ttm_resource *new_res)
+static int xe_bo_move_dmabuf(struct xe_bo *bo, struct ttm_resource *new_res)
 {
+	struct ttm_buffer_object *ttm_bo = &bo->ttm;
 	struct dma_buf_attachment *attach = ttm_bo->base.import_attach;
 	struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,
 					       ttm);
@@ -682,19 +750,16 @@ static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
 	    ttm_bo->sg) {
 		dma_resv_wait_timeout(ttm_bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
 				      false, MAX_SCHEDULE_TIMEOUT);
-		dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
-		ttm_bo->sg = NULL;
+		bo_unmap_dma_buf(bo);
 	}
 
 	if (new_res->mem_type == XE_PL_SYSTEM)
 		goto out;
 
-	if (ttm_bo->sg) {
-		dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
-		ttm_bo->sg = NULL;
-	}
+	if (ttm_bo->sg)
+		bo_unmap_dma_buf(bo);
 
-	sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+	sg = bo_map_dma_buf(bo);
 	if (IS_ERR(sg))
 		return PTR_ERR(sg);
 
@@ -797,7 +862,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 	if (ttm_bo->type == ttm_bo_type_sg) {
 		ret = xe_bo_move_notify(bo, ctx);
 		if (!ret)
-			ret = xe_bo_move_dmabuf(ttm_bo, new_mem);
+			ret = xe_bo_move_dmabuf(bo, new_mem);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index ecb5fe3b1c1dc..210ee49d6a04d 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -44,6 +44,7 @@
 #define XE_BO_FLAG_GGTT2		BIT(22)
 #define XE_BO_FLAG_GGTT3		BIT(23)
 #define XE_BO_FLAG_CPU_ADDR_MIRROR	BIT(24)
+#define XE_BO_FLAG_IAF			BIT(25)
 
 /* this one is trigger internally only */
 #define XE_BO_FLAG_INTERNAL_TEST	BIT(30)
@@ -262,6 +263,7 @@ void xe_bo_vunmap(struct xe_bo *bo);
 int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size);
 
 bool mem_type_is_vram(u32 mem_type);
+bool xe_bo_is_iaf(struct xe_bo *bo);
 bool xe_bo_is_vram(struct xe_bo *bo);
 bool xe_bo_is_stolen(struct xe_bo *bo);
 bool xe_bo_is_stolen_devmem(struct xe_bo *bo);
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 346f857f38374..2e8aca5d572ae 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -86,6 +86,8 @@ static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
 	xe_bo_unpin_external(bo);
 }
 
+static const struct dma_buf_attach_ops xe_dma_buf_attach_ops;
+
 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 				       enum dma_data_direction dir)
 {
@@ -94,12 +96,21 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	struct sg_table *sgt;
 	int r = 0;
+	bool iaf = false;
+
+	if (dir == DMA_NONE) {
+		if (attach->importer_ops != &xe_dma_buf_attach_ops)
+			return ERR_PTR(-EOPNOTSUPP);
+
+		iaf = true;
+	}
 
-	if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
+	if (!attach->peer2peer && !iaf &&
+	    !xe_bo_can_migrate(bo, XE_PL_TT))
 		return ERR_PTR(-EOPNOTSUPP);
 
 	if (!xe_bo_is_pinned(bo)) {
-		if (!attach->peer2peer)
+		if (!attach->peer2peer && !iaf)
 			r = xe_bo_migrate(bo, XE_PL_TT);
 		else
 			r = xe_bo_validate(bo, NULL, false);
@@ -109,6 +120,10 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 
 	switch (bo->ttm.resource->mem_type) {
 	case XE_PL_TT:
+		/* IAF only available for VRAM */
+		if (iaf)
+			return ERR_PTR(-EOPNOTSUPP);
+
 		sgt = drm_prime_pages_to_sg(obj->dev,
 					    bo->ttm.ttm->pages,
 					    bo->ttm.ttm->num_pages);
@@ -146,7 +161,8 @@ static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
 			     enum dma_data_direction dir)
 {
 	if (sg_page(sgt->sgl)) {
-		dma_unmap_sgtable(attach->dev, sgt, dir, 0);
+		if (dir != DMA_NONE)
+			dma_unmap_sgtable(attach->dev, sgt, dir, 0);
 		sg_free_table(sgt);
 		kfree(sgt);
 	} else {
@@ -187,6 +203,11 @@ static const struct dma_buf_ops xe_dmabuf_ops = {
 	.vunmap = drm_gem_dmabuf_vunmap,
 };
 
+bool is_xe_dma_buf(struct dma_buf *dma_buf)
+{
+	return dma_buf->ops == &xe_dmabuf_ops;
+}
+
 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
 {
 	struct xe_bo *bo = gem_to_xe_bo(obj);
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.h b/drivers/gpu/drm/xe/xe_dma_buf.h
index 861dd28a862c7..80d12c624b921 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.h
+++ b/drivers/gpu/drm/xe/xe_dma_buf.h
@@ -11,5 +11,6 @@
 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags);
 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
 					   struct dma_buf *dma_buf);
+bool is_xe_dma_buf(struct dma_buf *dma_buf);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index b8e1b44452e4d..c6ff18deb9db1 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -122,7 +122,9 @@ static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
 {
 	u64 pte = XE_PAGE_PRESENT;
 
-	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
+	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo) ||
+	    /* It's likely legal to map IAF to GGTT, but it should be impossible? */
+	    drm_WARN_ON(bo->ttm.base.dev, xe_bo_is_iaf(bo)))
 		pte |= XE_GGTT_PTE_DM;
 
 	return pte;
diff --git a/drivers/gpu/drm/xe/xe_iaf.c b/drivers/gpu/drm/xe/xe_iaf.c
index 431d20fea5369..af2a8d045b593 100644
--- a/drivers/gpu/drm/xe/xe_iaf.c
+++ b/drivers/gpu/drm/xe/xe_iaf.c
@@ -416,3 +416,45 @@ struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id)
 
 	return iaf->ops->connectivity_query(iaf->handle, fabric_id);
 }
+
+bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst)
+{
+	struct query_info *qi;
+	bool connected;
+
+	if (!src || !dst || !src->ops || !dst->ops)
+		return false;
+
+	qi = xe_iaf_connectivity_query(src, dst->fabric_id);
+	if (IS_ERR(qi))
+		return false;
+
+	if (WARN_ON_ONCE(!qi))
+		return true;
+
+	connected = true;
+	for (int i = 0, n = qi->src_cnt * qi->dst_cnt; i < n; i++)
+		if (!qi->sd2sd[i].bandwidth) {
+			connected = false;
+			break;
+		}
+
+	kfree(qi);
+	return connected;
+}
+
+int xe_iaf_mapping_get(struct xe_iaf *iaf)
+{
+	if (!iaf || !iaf->ops)
+		return -ENODEV;
+
+	return iaf->ops->parent_event(iaf->handle, IAF_PARENT_MAPPING_GET);
+}
+
+void xe_iaf_mapping_put(struct xe_iaf *iaf)
+{
+	if (!iaf || !iaf->ops)
+		return;
+
+	iaf->ops->parent_event(iaf->handle, IAF_PARENT_MAPPING_PUT);
+}
diff --git a/drivers/gpu/drm/xe/xe_iaf.h b/drivers/gpu/drm/xe/xe_iaf.h
index df6b8f9f2bc5f..b8e75640e8967 100644
--- a/drivers/gpu/drm/xe/xe_iaf.h
+++ b/drivers/gpu/drm/xe/xe_iaf.h
@@ -38,6 +38,9 @@ int xe_iaf_init(struct xe_device *xe);
 int xe_iaf_init_aux(struct xe_device *xe);
 u64 xe_iaf_dpa_base(struct xe_device *xe);
 struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id);
+bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst);
+int xe_iaf_mapping_get(struct xe_iaf *iaf);
+void xe_iaf_mapping_put(struct xe_iaf *iaf);
 
 #else
 
@@ -61,6 +64,19 @@ struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id)
 	return ERR_PTR(-ENODEV);
 }
 
+static inline int xe_iaf_mapping_get(struct xe_iaf *iaf)
+{
+	return -ENODEV;
+}
+
+static inline void xe_iaf_mapping_put(struct xe_iaf *iaf)
+{}
+
+static inline bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst)
+{
+	return false;
+}
+
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index f39d5cc9f411e..2231882b2bc45 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -764,6 +764,10 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 		else
 			xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
 					xe_vma_size(vma), &curs);
+
+		/* When IAF is used, the bo requires the DM flag */
+		if (xe_bo_is_iaf(bo))
+			curs.mem_type = XE_PL_VRAM0;
 	} else if (!range) {
 		curs.size = xe_vma_size(vma);
 	}
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 9e375a40aee90..94a572693e139 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -396,11 +396,16 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 		size_t size = min_t(u64, cursor.size, SZ_2G);
 		dma_addr_t addr;
 
-		addr = dma_map_resource(dev, phys, size, dir,
-					DMA_ATTR_SKIP_CPU_SYNC);
-		r = dma_mapping_error(dev, addr);
-		if (r)
-			goto error_unmap;
+		if (valid_dma_direction(dir)) {
+			addr = dma_map_resource(dev, phys, size, dir,
+						DMA_ATTR_SKIP_CPU_SYNC);
+			r = dma_mapping_error(dev, addr);
+			if (r)
+				goto error_unmap;
+		} else {
+			/* Only want the SG table for fabric */
+			addr = cursor.start + tile->mem.vram.dpa_base;
+		}
 
 		sg_set_page(sg, NULL, size, 0);
 		sg_dma_address(sg) = addr;
@@ -413,7 +418,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 
 error_unmap:
 	for_each_sgtable_sg((*sgt), sg, i) {
-		if (!sg->length)
+		if (!sg->length || !valid_dma_direction(dir))
 			continue;
 
 		dma_unmap_resource(dev, sg->dma_address,
@@ -433,10 +438,14 @@ void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
 	struct scatterlist *sg;
 	int i;
 
-	for_each_sgtable_sg(sgt, sg, i)
+	for_each_sgtable_sg(sgt, sg, i) {
+		if (!valid_dma_direction(dir))
+			continue;
+
 		dma_unmap_resource(dev, sg->dma_address,
 				   sg->length, dir,
 				   DMA_ATTR_SKIP_CPU_SYNC);
+	}
 	sg_free_table(sgt);
 	kfree(sgt);
 }
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 18f967ce1f1a6..f3dd38c95deb5 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1534,7 +1534,7 @@ static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
 	pte |= pte_encode_pat_index(pat_index, pt_level);
 	pte |= pte_encode_ps(pt_level);
 
-	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
+	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo) || xe_bo_is_iaf(bo))
 		pte |= XE_PPGTT_PTE_DM;
 
 	return pte;
-- 
2.45.2