[PATCH v3 2/3] drm/ttm: Allow skipping to the next priority in the bo LRU cursor

[Matthew Brost <matthew.brost@intel.com>]

Drivers walking a manager's LRU with ttm_bo_lru_for_each_reserved_guarded()
may decide that the rest of the current priority isn't worth walking,
for example when the priority is roughly age ordered and the walk hit
a BO that is still too hot to evict. Today the only option is to stop
the walk altogether, which also gives up on the remaining priorities.

Add ttm_resource_manager_next_priority() to move a resource cursor to
the start of the next priority, and ttm_bo_lru_cursor_skip_priority()
to make the next ttm_bo_lru_cursor_next() use it.

While at it, stop incrementing the cursor priority past the last valid
priority once the iteration is done.

Assisted-by: LLM
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
 drivers/gpu/drm/ttm/ttm_bo_util.c  |  3 +++
 drivers/gpu/drm/ttm/ttm_resource.c | 35 ++++++++++++++++++++++++++----
 include/drm/ttm/ttm_bo.h           | 19 ++++++++++++++++
 include/drm/ttm/ttm_resource.h     |  2 ++
 4 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index f9458a4d5a09..d626f014e61f 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -997,6 +997,9 @@ __ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs)
 		if (first) {
 			res = ttm_resource_manager_first(&curs->res_curs);
 			first = false;
+		} else if (curs->skip_priority) {
+			res = ttm_resource_manager_next_priority(&curs->res_curs);
+			curs->skip_priority = false;
 		} else {
 			res = ttm_resource_manager_next(&curs->res_curs);
 		}
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 4e6d3d658d85..e631507d7aa1 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -674,6 +674,17 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
 	}
 }
 
+static bool ttm_resource_cursor_next_priority(struct ttm_resource_cursor *cursor)
+{
+	if (cursor->priority + 1 >= TTM_MAX_BO_PRIORITY)
+		return false;
+
+	list_move(&cursor->hitch.link, &cursor->man->lru[++cursor->priority]);
+	ttm_resource_cursor_clear_bulk(cursor);
+
+	return true;
+}
+
 /**
  * ttm_resource_manager_first() - Start iterating over the resources
  * of a resource manager
@@ -723,16 +734,32 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
 			}
 		}
 
-		if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
+		if (!ttm_resource_cursor_next_priority(cursor))
 			break;
-
-		list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
-		ttm_resource_cursor_clear_bulk(cursor);
 	}
 
 	return NULL;
 }
 
+/**
+ * ttm_resource_manager_next_priority() - Skip the rest of the current
+ * priority and continue iterating at the next one
+ * @cursor: cursor to record the position
+ *
+ * Return: the first resource of the next non-empty priority, or NULL if
+ * there is none.
+ */
+struct ttm_resource *
+ttm_resource_manager_next_priority(struct ttm_resource_cursor *cursor)
+{
+	lockdep_assert_held(&cursor->man->bdev->lru_lock);
+
+	if (!ttm_resource_cursor_next_priority(cursor))
+		return NULL;
+
+	return ttm_resource_manager_next(cursor);
+}
+
 /**
  * ttm_lru_first_res_or_null() - Return the first resource on an lru list
  * @head: The list head of the lru list.
diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h
index 32791c4db2a9..2fc517dbf03d 100644
--- a/include/drm/ttm/ttm_bo.h
+++ b/include/drm/ttm/ttm_bo.h
@@ -507,10 +507,29 @@ struct ttm_bo_lru_cursor {
 	 * unlock before the next iteration or after loop exit.
 	 */
 	bool needs_unlock;
+	/**
+	 * @skip_priority: Continue at the next LRU priority on the next
+	 * iteration. See ttm_bo_lru_cursor_skip_priority().
+	 */
+	bool skip_priority;
 	/** @arg: Pointer to common BO LRU walk arguments. */
 	struct ttm_lru_walk_arg *arg;
 };
 
+/**
+ * ttm_bo_lru_cursor_skip_priority() - Skip the rest of the current priority
+ * @curs: The cursor.
+ *
+ * Make the next ttm_bo_lru_cursor_next() skip the remaining buffer objects
+ * of the current LRU priority and continue at the next priority, for
+ * example when the caller decides the rest of the priority isn't worth
+ * walking.
+ */
+static inline void ttm_bo_lru_cursor_skip_priority(struct ttm_bo_lru_cursor *curs)
+{
+	curs->skip_priority = true;
+}
+
 void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs);
 
 struct ttm_bo_lru_cursor *
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 714c533622c2..8cd70d1e5481 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -499,6 +499,8 @@ struct ttm_resource *
 ttm_resource_manager_first(struct ttm_resource_cursor *cursor);
 struct ttm_resource *
 ttm_resource_manager_next(struct ttm_resource_cursor *cursor);
+struct ttm_resource *
+ttm_resource_manager_next_priority(struct ttm_resource_cursor *cursor);
 
 struct ttm_resource *
 ttm_lru_first_res_or_null(struct list_head *head);
-- 
2.34.1