[PATCH v3 2/3] drm/xe: Split the bo shrinker into purgeable and non-purgeable shrinkers

["Thomas Hellström" <thomas.hellstrom@linux.intel.com>]

Purgeable objects are reclaimed by simply discarding their pages, with
no writeback required. They are therefore much cheaper to shrink than
regular shrinkable (non-purgeable) objects, whose reclaim may need to
wait for the GPU or write dirty compressed contents back to a backup
store, and whose contents need to be read back on next use.

Split the single xe_shrinker into two independent struct shrinker
instances registered against the shrinker core, sharing the same
underlying accounting (shrinkable_pages, purgeable_pages,
populated_tts) and locking:

 - A purgeable shrinker, whose count_objects() reports only
   purgeable_pages, and whose scan_objects() only purges objects.

 - A non-purgeable (shrinkable) shrinker, whose count_objects()
   reports the shrinkable page count (capped by backup availability)
   and whose scan_objects() shrinks non-purgeable objects, waking the
   device via runtime PM as needed.

Express the cost difference to the shrinker core through the seeks
value: the purgeable shrinker keeps DEFAULT_SEEKS, while the
non-purgeable shrinker uses a higher value,
XE_SHRINKER_NON_PURGEABLE_SEEKS (8). The core scales each shrinker's
scan target inversely with seeks, so purgeable objects receive
proportionally more reclaim pressure than non-purgeable ones, without
one shrinker's behaviour depending on the other's state.

v3:
- Patch introduced. (Matt Brost)

Assisted-by: LLM
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
 drivers/gpu/drm/xe/xe_shrinker.c | 109 +++++++++++++++++++------------
 1 file changed, 68 insertions(+), 41 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c
index baf3229f8160..eb684b289679 100644
--- a/drivers/gpu/drm/xe/xe_shrinker.c
+++ b/drivers/gpu/drm/xe/xe_shrinker.c
@@ -14,6 +14,15 @@
 #include "xe_pm.h"
 #include "xe_shrinker.h"
 
+/*
+ * Reclaiming non-purgeable objects requires GPU idling and writeback to a
+ * backup store, and restoring them requires reading the contents back.
+ * Tell the shrinker core they're costlier to recreate than purgeable
+ * objects, which use DEFAULT_SEEKS, so that purgeable objects are
+ * preferentially reclaimed.
+ */
+#define XE_SHRINKER_NON_PURGEABLE_SEEKS 8
+
 /**
  * struct xe_shrinker - per-device shrinker
  * @xe: Back pointer to the device.
@@ -21,7 +30,8 @@
  * @shrinkable_pages: Number of pages that are currently shrinkable.
  * @purgeable_pages: Number of pages that are currently purgeable.
  * @populated_tts: Number of populated ttm_tts currently shrinkable or purgeable.
- * @shrink: Pointer to the mm shrinker.
+ * @shrink: Pointer to the mm shrinker handling non-purgeable (shrinkable) objects.
+ * @purge_shrink: Pointer to the mm shrinker handling purgeable objects.
  * @pm_worker: Worker to wake up the device if required.
  */
 struct xe_shrinker {
@@ -31,6 +41,7 @@ struct xe_shrinker {
 	long purgeable_pages;
 	long populated_tts;
 	struct shrinker *shrink;
+	struct shrinker *purge_shrink;
 	struct work_struct pm_worker;
 };
 
@@ -176,6 +187,19 @@ static int xe_shrinker_walk(struct xe_shrinker *shrinker,
 	return ret;
 }
 
+static unsigned long
+xe_shrinker_count_purgeable(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
+	unsigned long num_pages;
+
+	read_lock(&shrinker->lock);
+	num_pages = shrinker->purgeable_pages;
+	read_unlock(&shrinker->lock);
+
+	return num_pages ? num_pages : SHRINK_EMPTY;
+}
+
 static unsigned long
 xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
 {
@@ -191,7 +215,6 @@ xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
 	else
 		num_pages = 0;
 
-	num_pages += shrinker->purgeable_pages;
 	read_unlock(&shrinker->lock);
 
 	return num_pages ? num_pages : SHRINK_EMPTY;
@@ -204,8 +227,7 @@ xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
  * If bo shrinking needs an rpm reference and we don't have it (yet),
  * that bo will be skipped anyway.
  */
-static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force,
-				       unsigned long nr_to_scan, bool can_backup)
+static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker)
 {
 	struct xe_device *xe = shrinker->xe;
 
@@ -213,63 +235,52 @@ static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force,
 	    !ttm_backup_bytes_avail())
 		return false;
 
-	if (!force) {
-		read_lock(&shrinker->lock);
-		force = (nr_to_scan > shrinker->purgeable_pages && can_backup);
-		read_unlock(&shrinker->lock);
-		if (!force)
-			return false;
-	}
-
 	return __xe_shrinker_runtime_pm_get(shrinker);
 }
 
-static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+static unsigned long xe_shrinker_scan_purgeable(struct shrinker *shrink,
+						struct shrink_control *sc)
 {
 	struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
 	struct ttm_operation_ctx ctx = {
 		.interruptible = false,
 		.no_wait_gpu = ttm_bo_shrink_avoid_wait(),
 	};
-	unsigned long nr_to_scan, nr_scanned = 0, freed = 0;
+	unsigned long nr_to_scan = sc->nr_to_scan, nr_scanned = 0, freed = 0;
 	struct xe_bo_shrink_flags shrink_flags = {
 		.purge = true,
-		/* Don't request writeback without __GFP_IO. */
-		.writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO),
 	};
-	bool runtime_pm;
-	bool purgeable;
-	bool can_backup = !!(sc->gfp_mask & __GFP_FS);
-
-	nr_to_scan = sc->nr_to_scan;
-
-	read_lock(&shrinker->lock);
-	purgeable = !!shrinker->purgeable_pages;
-	read_unlock(&shrinker->lock);
 
-	/* Might need runtime PM. Try to wake early if it looks like it. */
-	runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup);
-
-	if (purgeable && nr_scanned < nr_to_scan)
-		xe_shrinker_walk(shrinker, &ctx, shrink_flags,
-				 nr_to_scan, &nr_scanned, &freed);
+	xe_shrinker_walk(shrinker, &ctx, shrink_flags, nr_to_scan, &nr_scanned, &freed);
 
 	sc->nr_scanned = nr_scanned;
-	if (nr_scanned >= nr_to_scan || !can_backup)
-		goto out;
 
-	/* If we didn't wake before, try to do it now if needed. */
-	if (!runtime_pm)
-		runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup);
+	return nr_scanned ? freed : SHRINK_STOP;
+}
 
-	shrink_flags.purge = false;
+static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = ttm_bo_shrink_avoid_wait(),
+	};
+	unsigned long nr_to_scan = sc->nr_to_scan, nr_scanned = 0, freed = 0;
+	struct xe_bo_shrink_flags shrink_flags = {
+		.purge = false,
+		/* Don't request writeback without __GFP_IO. */
+		.writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO),
+	};
+	bool runtime_pm;
+
+	/* Might need runtime PM to access backed-up or CCS-compressed data. */
+	runtime_pm = xe_shrinker_runtime_pm_get(shrinker);
 
-	xe_shrinker_walk(shrinker, &ctx, shrink_flags,
-			 nr_to_scan, &nr_scanned, &freed);
+	xe_shrinker_walk(shrinker, &ctx, shrink_flags, nr_to_scan, &nr_scanned, &freed);
 
 	sc->nr_scanned = nr_scanned;
-out:
 	xe_shrinker_runtime_pm_put(shrinker, runtime_pm);
+
 	return nr_scanned ? freed : SHRINK_STOP;
 }
 
@@ -291,6 +302,7 @@ static void xe_shrinker_fini(struct drm_device *drm, void *arg)
 	xe_assert(shrinker->xe, !shrinker->purgeable_pages);
 	xe_assert(shrinker->xe, !shrinker->populated_tts);
 	shrinker_free(shrinker->shrink);
+	shrinker_free(shrinker->purge_shrink);
 	flush_work(&shrinker->pm_worker);
 	kfree(shrinker);
 }
@@ -308,8 +320,15 @@ int xe_shrinker_create(struct xe_device *xe)
 	if (!shrinker)
 		return -ENOMEM;
 
-	shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique);
+	shrinker->purge_shrink = shrinker_alloc(0, "drm-xe_gem-purgeable:%s", xe->drm.unique);
+	if (!shrinker->purge_shrink) {
+		kfree(shrinker);
+		return -ENOMEM;
+	}
+
+	shrinker->shrink = shrinker_alloc(0, "drm-xe_gem-shrinkable:%s", xe->drm.unique);
 	if (!shrinker->shrink) {
+		shrinker_free(shrinker->purge_shrink);
 		kfree(shrinker);
 		return -ENOMEM;
 	}
@@ -317,10 +336,18 @@ int xe_shrinker_create(struct xe_device *xe)
 	INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm);
 	shrinker->xe = xe;
 	rwlock_init(&shrinker->lock);
+
+	shrinker->purge_shrink->count_objects = xe_shrinker_count_purgeable;
+	shrinker->purge_shrink->scan_objects = xe_shrinker_scan_purgeable;
+	shrinker->purge_shrink->private_data = shrinker;
+	shrinker_register(shrinker->purge_shrink);
+
 	shrinker->shrink->count_objects = xe_shrinker_count;
 	shrinker->shrink->scan_objects = xe_shrinker_scan;
 	shrinker->shrink->private_data = shrinker;
+	shrinker->shrink->seeks = XE_SHRINKER_NON_PURGEABLE_SEEKS;
 	shrinker_register(shrinker->shrink);
+
 	xe->mem.shrinker = shrinker;
 
 	return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker);
-- 
2.56.0