[PATCH 07/11] drm/xe: Suballocate BO for page reclaim

[Brian Nguyen <brian3.nguyen@intel.com>]

Page reclamation feature needs the PRL to be suballocated into a
GGTT-mapped BO. On allocation failure, fallback to default tlb
invalidation with full PPC flush.

PRL's BO allocation is managed in separate pool to ensure 4K alignment
for proper GGTT address.

With BO, pass into TLB invalidation backend and modify fence to
accomadate accordingly.

Signed-off-by: Brian Nguyen <brian3.nguyen@intel.com>
Suggested-by: Matthew Brost <matthew.brost@intel.com>
---
 drivers/gpu/drm/xe/xe_device_types.h    |  7 ++++++
 drivers/gpu/drm/xe/xe_page_reclaim.c    | 33 +++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_page_reclaim.h    |  4 +++
 drivers/gpu/drm/xe/xe_tile.c            |  5 ++++
 drivers/gpu/drm/xe/xe_tlb_inval.c       | 18 ++++++++++++--
 drivers/gpu/drm/xe/xe_tlb_inval_types.h |  5 ++++
 6 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 268c8e28601a..057df3f9dc1d 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -184,6 +184,13 @@ struct xe_tile {
 		 * Media GT shares a pool with its primary GT.
 		 */
 		struct xe_sa_manager *kernel_bb_pool;
+
+		/**
+		 * @mem.reclaim_pool: Pool for PRLs allocated.
+		 *
+		 * Only main GT has page reclaim list allocations.
+		 */
+		struct xe_sa_manager *reclaim_pool;
 	} mem;
 
 	/** @sriov: tile level virtualization data */
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c
index a0d15efff58c..801a7f1731c0 100644
--- a/drivers/gpu/drm/xe/xe_page_reclaim.c
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.c
@@ -13,6 +13,39 @@
 #include "regs/xe_gt_regs.h"
 #include "xe_assert.h"
 #include "xe_macros.h"
+#include "xe_sa.h"
+#include "xe_tlb_inval_types.h"
+
+/**
+ * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO
+ * @tlb_inval: TLB invalidation frontend associated with the request
+ * @fence: Fence carrying the PRL metadata
+ *
+ * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU
+ * copy into the BO and queues the buffer for release when @fence signals.
+ *
+ * Return: 0 on success or -ENOMEM if the suballocation fails.
+ */
+int xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence)
+{
+	struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval);
+	struct xe_tile *tile = gt_to_tile(gt);
+
+	/* Maximum size of PRL is 1 4K-page */
+	fence->prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool,
+				       XE_PAGE_RECLAIM_LIST_MAX_SIZE, GFP_ATOMIC);
+	if (IS_ERR(fence->prl_sa))
+		return -ENOMEM;
+
+	memcpy(xe_sa_bo_cpu_addr(fence->prl_sa), fence->reclaim_entries,
+	       XE_PAGE_RECLAIM_LIST_MAX_SIZE);
+	xe_sa_bo_flush_write(fence->prl_sa);
+
+	/* Queue up sa_bo_free on fence signal */
+	xe_sa_bo_free(fence->prl_sa, &fence->base);
+
+	return 0;
+}
 
 /**
  * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h
index d066d7d97f79..f82b4d0865e0 100644
--- a/drivers/gpu/drm/xe/xe_page_reclaim.h
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.h
@@ -15,6 +15,9 @@
 #define XE_PAGE_RECLAIM_MAX_ENTRIES	512
 #define XE_PAGE_RECLAIM_LIST_MAX_SIZE	SZ_4K
 
+struct xe_tlb_inval;
+struct xe_tlb_inval_fence;
+
 struct xe_guc_page_reclaim_entry {
 	u32 valid:1;
 	u32 reclamation_size:6;
@@ -32,6 +35,7 @@ struct xe_page_reclaim_list {
 #define XE_PAGE_RECLAIM_INVALID_LIST	-1
 };
 
+int xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence);
 void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl);
 int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl);
 static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries)
diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c
index 4f4f9a5c43af..63c060c2ea5c 100644
--- a/drivers/gpu/drm/xe/xe_tile.c
+++ b/drivers/gpu/drm/xe/xe_tile.c
@@ -209,6 +209,11 @@ int xe_tile_init(struct xe_tile *tile)
 	if (IS_ERR(tile->mem.kernel_bb_pool))
 		return PTR_ERR(tile->mem.kernel_bb_pool);
 
+	/* Optimistically anticipate at most 256 TLB fences with PRL */
+	tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE);
+	if (IS_ERR(tile->mem.reclaim_pool))
+		return PTR_ERR(tile->mem.reclaim_pool);
+
 	return 0;
 }
 void xe_tile_migrate_wait(struct xe_tile *tile)
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
index de275759743c..67a047521165 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
@@ -15,6 +15,7 @@
 #include "xe_guc_ct.h"
 #include "xe_guc_tlb_inval.h"
 #include "xe_mmio.h"
+#include "xe_page_reclaim.h"
 #include "xe_pm.h"
 #include "xe_tlb_inval.h"
 #include "xe_trace.h"
@@ -326,8 +327,19 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
 		       struct xe_tlb_inval_fence *fence, u64 start, u64 end,
 		       u32 asid, bool flush_cache)
 {
-	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
-				  start, end, asid, flush_cache);
+	int err;
+
+	if (fence->reclaim_entries) {
+		err = xe_page_reclaim_create_prl_bo(tlb_inval, fence);
+		if (err) {
+			flush_cache = true;
+			fence->prl_sa = NULL;
+		}
+	}
+	err = xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
+				 start, end, asid, flush_cache);
+
+	return err;
 }
 
 /**
@@ -461,4 +473,6 @@ void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
 		dma_fence_get(&fence->base);
 	fence->tlb_inval = tlb_inval;
 	fence->flush_cache = true;
+	fence->reclaim_entries = NULL;
+	fence->prl_sa = NULL;
 }
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
index c3c3943fb07e..7cf741e6a0c7 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
@@ -9,6 +9,7 @@
 #include <linux/workqueue.h>
 #include <linux/dma-fence.h>
 
+struct xe_guc_page_reclaim_entry;
 struct xe_tlb_inval;
 
 /** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */
@@ -129,6 +130,10 @@ struct xe_tlb_inval_fence {
 	ktime_t inval_time;
 	/** @flush_cache: bool for PPC flush, default is true */
 	bool flush_cache;
+	/** @reclaim_entries: list of pages to reclaim */
+	struct xe_guc_page_reclaim_entry *reclaim_entries;
+	/** @prl_sa: BO allocation for page reclaim list */
+	struct drm_suballoc *prl_sa;
 };
 
 #endif
-- 
2.51.2