* [CI v2 1/8] drm/ttm: Allow TTM LRU list nodes of different types
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 2/8] drm/ttm: Use LRU hitches Thomas Hellström
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
To be able to handle list unlocking while traversing the LRU
list, we want the iterators not only to point to the next
position of the list traversal, but to insert themselves as
list nodes at that point to work around the fact that the
next node might otherwise disappear from the list while
the iterator is pointing to it.
These list nodes need to be easily distinguishable from other
list nodes so that others traversing the list can skip
over them.
So declare a struct ttm_lru_item, with a struct list_head member
and a type enum. This will slightly increase the size of a
struct ttm_resource.
v2:
- Update enum ttm_lru_item_type documentation.
Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_device.c | 13 ++++--
drivers/gpu/drm/ttm/ttm_resource.c | 70 ++++++++++++++++++++++--------
include/drm/ttm/ttm_resource.h | 51 +++++++++++++++++++++-
3 files changed, 110 insertions(+), 24 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 76027960054f..f27406e851e5 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -270,17 +270,22 @@ EXPORT_SYMBOL(ttm_device_fini);
static void ttm_device_clear_lru_dma_mappings(struct ttm_device *bdev,
struct list_head *list)
{
- struct ttm_resource *res;
+ struct ttm_lru_item *lru;
spin_lock(&bdev->lru_lock);
- while ((res = list_first_entry_or_null(list, typeof(*res), lru))) {
- struct ttm_buffer_object *bo = res->bo;
+ while ((lru = list_first_entry_or_null(list, typeof(*lru), link))) {
+ struct ttm_buffer_object *bo;
+
+ if (!ttm_lru_item_is_res(lru))
+ continue;
+
+ bo = ttm_lru_item_to_res(lru)->bo;
/* Take ref against racing releases once lru_lock is unlocked */
if (!ttm_bo_get_unless_zero(bo))
continue;
- list_del_init(&res->lru);
+ list_del_init(&bo->resource->lru.link);
spin_unlock(&bdev->lru_lock);
if (bo->ttm)
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index be8d286513f9..7aa5ca5c0e33 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -69,8 +69,8 @@ void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
dma_resv_assert_held(pos->last->bo->base.resv);
man = ttm_manager_type(pos->first->bo->bdev, i);
- list_bulk_move_tail(&man->lru[j], &pos->first->lru,
- &pos->last->lru);
+ list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,
+ &pos->last->lru.link);
}
}
}
@@ -83,14 +83,38 @@ ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
return &bulk->pos[res->mem_type][res->bo->priority];
}
+/* Return the previous resource on the list (skip over non-resource list items) */
+static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
+{
+ struct ttm_lru_item *lru = &cur->lru;
+
+ do {
+ lru = list_prev_entry(lru, link);
+ } while (!ttm_lru_item_is_res(lru));
+
+ return ttm_lru_item_to_res(lru);
+}
+
+/* Return the next resource on the list (skip over non-resource list items) */
+static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
+{
+ struct ttm_lru_item *lru = &cur->lru;
+
+ do {
+ lru = list_next_entry(lru, link);
+ } while (!ttm_lru_item_is_res(lru));
+
+ return ttm_lru_item_to_res(lru);
+}
+
/* Move the resource to the tail of the bulk move range */
static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
struct ttm_resource *res)
{
if (pos->last != res) {
if (pos->first == res)
- pos->first = list_next_entry(res, lru);
- list_move(&res->lru, &pos->last->lru);
+ pos->first = ttm_lru_next_res(res);
+ list_move(&res->lru.link, &pos->last->lru.link);
pos->last = res;
}
}
@@ -121,11 +145,11 @@ static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
pos->first = NULL;
pos->last = NULL;
} else if (pos->first == res) {
- pos->first = list_next_entry(res, lru);
+ pos->first = ttm_lru_next_res(res);
} else if (pos->last == res) {
- pos->last = list_prev_entry(res, lru);
+ pos->last = ttm_lru_prev_res(res);
} else {
- list_move(&res->lru, &pos->last->lru);
+ list_move(&res->lru.link, &pos->last->lru.link);
}
}
@@ -154,7 +178,7 @@ void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
lockdep_assert_held(&bo->bdev->lru_lock);
if (bo->pin_count) {
- list_move_tail(&res->lru, &bdev->pinned);
+ list_move_tail(&res->lru.link, &bdev->pinned);
} else if (bo->bulk_move) {
struct ttm_lru_bulk_move_pos *pos =
@@ -165,7 +189,7 @@ void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
struct ttm_resource_manager *man;
man = ttm_manager_type(bdev, res->mem_type);
- list_move_tail(&res->lru, &man->lru[bo->priority]);
+ list_move_tail(&res->lru.link, &man->lru[bo->priority]);
}
}
@@ -196,9 +220,9 @@ void ttm_resource_init(struct ttm_buffer_object *bo,
man = ttm_manager_type(bo->bdev, place->mem_type);
spin_lock(&bo->bdev->lru_lock);
if (bo->pin_count)
- list_add_tail(&res->lru, &bo->bdev->pinned);
+ list_add_tail(&res->lru.link, &bo->bdev->pinned);
else
- list_add_tail(&res->lru, &man->lru[bo->priority]);
+ list_add_tail(&res->lru.link, &man->lru[bo->priority]);
man->usage += res->size;
spin_unlock(&bo->bdev->lru_lock);
}
@@ -220,7 +244,7 @@ void ttm_resource_fini(struct ttm_resource_manager *man,
struct ttm_device *bdev = man->bdev;
spin_lock(&bdev->lru_lock);
- list_del_init(&res->lru);
+ list_del_init(&res->lru.link);
man->usage -= res->size;
spin_unlock(&bdev->lru_lock);
}
@@ -471,14 +495,16 @@ struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager *man,
struct ttm_resource_cursor *cursor)
{
- struct ttm_resource *res;
+ struct ttm_lru_item *lru;
lockdep_assert_held(&man->bdev->lru_lock);
for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
++cursor->priority)
- list_for_each_entry(res, &man->lru[cursor->priority], lru)
- return res;
+ list_for_each_entry(lru, &man->lru[cursor->priority], link) {
+ if (ttm_lru_item_is_res(lru))
+ return ttm_lru_item_to_res(lru);
+ }
return NULL;
}
@@ -497,15 +523,21 @@ ttm_resource_manager_next(struct ttm_resource_manager *man,
struct ttm_resource_cursor *cursor,
struct ttm_resource *res)
{
+ struct ttm_lru_item *lru = &res->lru;
+
lockdep_assert_held(&man->bdev->lru_lock);
- list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
- return res;
+ list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
+ if (ttm_lru_item_is_res(lru))
+ return ttm_lru_item_to_res(lru);
+ }
for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
++cursor->priority)
- list_for_each_entry(res, &man->lru[cursor->priority], lru)
- return res;
+ list_for_each_entry(lru, &man->lru[cursor->priority], link) {
+ if (ttm_lru_item_is_res(lru))
+ ttm_lru_item_to_res(lru);
+ }
return NULL;
}
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 69769355139f..4babc4ff10b0 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -49,6 +49,43 @@ struct io_mapping;
struct sg_table;
struct scatterlist;
+/**
+ * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses
+ */
+enum ttm_lru_item_type {
+ /** @TTM_LRU_RESOURCE: The resource subclass */
+ TTM_LRU_RESOURCE,
+ /** @TTM_LRU_HITCH: The iterator hitch subclass */
+ TTM_LRU_HITCH
+};
+
+/**
+ * struct ttm_lru_item - The TTM lru list node base class
+ * @link: The list link
+ * @type: The subclass type
+ */
+struct ttm_lru_item {
+ struct list_head link;
+ enum ttm_lru_item_type type;
+};
+
+/**
+ * ttm_lru_item_init() - initialize a struct ttm_lru_item
+ * @item: The item to initialize
+ * @type: The subclass type
+ */
+static inline void ttm_lru_item_init(struct ttm_lru_item *item,
+ enum ttm_lru_item_type type)
+{
+ item->type = type;
+ INIT_LIST_HEAD(&item->link);
+}
+
+static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)
+{
+ return item->type == TTM_LRU_RESOURCE;
+}
+
struct ttm_resource_manager_func {
/**
* struct ttm_resource_manager_func member alloc
@@ -217,9 +254,21 @@ struct ttm_resource {
/**
* @lru: Least recently used list, see &ttm_resource_manager.lru
*/
- struct list_head lru;
+ struct ttm_lru_item lru;
};
+/**
+ * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource
+ * @item: The struct ttm_lru_item to downcast
+ *
+ * Return: Pointer to the embedding struct ttm_resource
+ */
+static inline struct ttm_resource *
+ttm_lru_item_to_res(struct ttm_lru_item *item)
+{
+ return container_of(item, struct ttm_resource, lru);
+}
+
/**
* struct ttm_resource_cursor
*
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 2/8] drm/ttm: Use LRU hitches
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
2024-03-28 8:47 ` [CI v2 1/8] drm/ttm: Allow TTM LRU list nodes of different types Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 3/8] drm/ttm, drm/amdgpu, drm/xe: Consider hitch moves within bulk sublist moves Thomas Hellström
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
Have iterators insert themselves into the list they are iterating
over using hitch list nodes. Since only the iterator owner
can remove these list nodes from the list, it's safe to unlock
the list and when continuing, use them as a starting point. Due to
the way LRU bumping works in TTM, newly added items will not be
missed, and bumped items will be iterated over a second time before
reaching the end of the list.
The exception is list with bulk move sublists. When bumping a
sublist, a hitch that is part of that sublist will also be moved
and we might miss items if restarting from it. This will be
addressed in a later patch.
v2:
- Updated ttm_resource_cursor_fini() documentation.
Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_bo.c | 1 +
drivers/gpu/drm/ttm/ttm_device.c | 9 ++-
drivers/gpu/drm/ttm/ttm_resource.c | 94 ++++++++++++++++++++----------
include/drm/ttm/ttm_resource.h | 16 +++--
4 files changed, 82 insertions(+), 38 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index e059b1e1b13b..b6f75a0ff2e5 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -622,6 +622,7 @@ int ttm_mem_evict_first(struct ttm_device *bdev,
if (locked)
dma_resv_unlock(res->bo->base.resv);
}
+ ttm_resource_cursor_fini_locked(&cursor);
if (!bo) {
if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index f27406e851e5..e8a6a1dab669 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -169,12 +169,17 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
num_pages = PFN_UP(bo->base.size);
ret = ttm_bo_swapout(bo, ctx, gfp_flags);
/* ttm_bo_swapout has dropped the lru_lock */
- if (!ret)
+ if (!ret) {
+ ttm_resource_cursor_fini(&cursor);
return num_pages;
- if (ret != -EBUSY)
+ }
+ if (ret != -EBUSY) {
+ ttm_resource_cursor_fini(&cursor);
return ret;
+ }
}
}
+ ttm_resource_cursor_fini_locked(&cursor);
spin_unlock(&bdev->lru_lock);
return 0;
}
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 7aa5ca5c0e33..ccc31ad85e3b 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -32,6 +32,37 @@
#include <drm/drm_util.h>
+/**
+ * ttm_resource_cursor_fini_locked() - Finalize the LRU list cursor usage
+ * @cursor: The struct ttm_resource_cursor to finalize.
+ *
+ * The function pulls the LRU list cursor off any lists it was previusly
+ * attached to. Needs to be called with the LRU lock held. The function
+ * can be called multiple times after eachother.
+ */
+void ttm_resource_cursor_fini_locked(struct ttm_resource_cursor *cursor)
+{
+ lockdep_assert_held(&cursor->man->bdev->lru_lock);
+ list_del_init(&cursor->hitch.link);
+}
+
+/**
+ * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
+ * @cursor: The struct ttm_resource_cursor to finalize.
+ *
+ * The function pulls the LRU list cursor off any lists it was previusly
+ * attached to. Needs to be called without the LRU list lock held. The
+ * function can be called multiple times after eachother.
+ */
+void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
+{
+ spinlock_t *lru_lock = &cursor->man->bdev->lru_lock;
+
+ spin_lock(lru_lock);
+ ttm_resource_cursor_fini_locked(cursor);
+ spin_unlock(lru_lock);
+}
+
/**
* ttm_lru_bulk_move_init - initialize a bulk move structure
* @bulk: the structure to init
@@ -484,62 +515,63 @@ void ttm_resource_manager_debug(struct ttm_resource_manager *man,
EXPORT_SYMBOL(ttm_resource_manager_debug);
/**
- * ttm_resource_manager_first
- *
- * @man: resource manager to iterate over
+ * ttm_resource_manager_next() - Continue iterating over the resource manager
+ * resources
* @cursor: cursor to record the position
*
- * Returns the first resource from the resource manager.
+ * Return: The next resource from the resource manager.
*/
struct ttm_resource *
-ttm_resource_manager_first(struct ttm_resource_manager *man,
- struct ttm_resource_cursor *cursor)
+ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
{
+ struct ttm_resource_manager *man = cursor->man;
struct ttm_lru_item *lru;
lockdep_assert_held(&man->bdev->lru_lock);
- for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
- ++cursor->priority)
- list_for_each_entry(lru, &man->lru[cursor->priority], link) {
- if (ttm_lru_item_is_res(lru))
+ do {
+ lru = &cursor->hitch;
+ list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
+ if (ttm_lru_item_is_res(lru)) {
+ list_move(&cursor->hitch.link, &lru->link);
return ttm_lru_item_to_res(lru);
+ }
}
+ if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
+ break;
+
+ list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
+ } while (true);
+
+ list_del_init(&cursor->hitch.link);
+
return NULL;
}
/**
- * ttm_resource_manager_next
- *
+ * ttm_resource_manager_first() - Start iterating over the resources
+ * of a resource manager
* @man: resource manager to iterate over
* @cursor: cursor to record the position
- * @res: the current resource pointer
*
- * Returns the next resource from the resource manager.
+ * Initializes the cursor and starts iterating. When done iterating,
+ * the caller must explicitly call ttm_resource_cursor_fini().
+ *
+ * Return: The first resource from the resource manager.
*/
struct ttm_resource *
-ttm_resource_manager_next(struct ttm_resource_manager *man,
- struct ttm_resource_cursor *cursor,
- struct ttm_resource *res)
+ttm_resource_manager_first(struct ttm_resource_manager *man,
+ struct ttm_resource_cursor *cursor)
{
- struct ttm_lru_item *lru = &res->lru;
-
lockdep_assert_held(&man->bdev->lru_lock);
- list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
- if (ttm_lru_item_is_res(lru))
- return ttm_lru_item_to_res(lru);
- }
+ cursor->priority = 0;
+ cursor->man = man;
+ ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);
+ list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
- for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
- ++cursor->priority)
- list_for_each_entry(lru, &man->lru[cursor->priority], link) {
- if (ttm_lru_item_is_res(lru))
- ttm_lru_item_to_res(lru);
- }
-
- return NULL;
+ return ttm_resource_manager_next(cursor);
}
static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 4babc4ff10b0..dfc01258c981 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -271,15 +271,23 @@ ttm_lru_item_to_res(struct ttm_lru_item *item)
/**
* struct ttm_resource_cursor
- *
+ * @man: The resource manager currently being iterated over
+ * @hitch: A hitch list node inserted before the next resource
+ * to iterate over.
* @priority: the current priority
*
* Cursor to iterate over the resources in a manager.
*/
struct ttm_resource_cursor {
+ struct ttm_resource_manager *man;
+ struct ttm_lru_item hitch;
unsigned int priority;
};
+void ttm_resource_cursor_fini_locked(struct ttm_resource_cursor *cursor);
+
+void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
+
/**
* struct ttm_lru_bulk_move_pos
*
@@ -438,9 +446,7 @@ struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager *man,
struct ttm_resource_cursor *cursor);
struct ttm_resource *
-ttm_resource_manager_next(struct ttm_resource_manager *man,
- struct ttm_resource_cursor *cursor,
- struct ttm_resource *res);
+ttm_resource_manager_next(struct ttm_resource_cursor *cursor);
/**
* ttm_resource_manager_for_each_res - iterate over all resources
@@ -452,7 +458,7 @@ ttm_resource_manager_next(struct ttm_resource_manager *man,
*/
#define ttm_resource_manager_for_each_res(man, cursor, res) \
for (res = ttm_resource_manager_first(man, cursor); res; \
- res = ttm_resource_manager_next(man, cursor, res))
+ res = ttm_resource_manager_next(cursor))
struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 3/8] drm/ttm, drm/amdgpu, drm/xe: Consider hitch moves within bulk sublist moves
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
2024-03-28 8:47 ` [CI v2 1/8] drm/ttm: Allow TTM LRU list nodes of different types Thomas Hellström
2024-03-28 8:47 ` [CI v2 2/8] drm/ttm: Use LRU hitches Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 4/8] drm/ttm: Allow continued swapout after -ENOSPC falure Thomas Hellström
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
To address the problem with hitches moving when bulk move
sublists are lru-bumped, register the list cursors with the
ttm_lru_bulk_move structure when traversing its list, and
when lru-bumping the list, move the cursor hitch to the tail.
This also means it's mandatory for drivers to call
ttm_lru_bulk_move_init() and ttm_lru_bulk_move_fini() when
initializing and finalizing the bulk move structure, so add
those calls to the amdgpu- and xe driver.
Compared to v1 this is slightly more code but less fragile
and hopefully easier to understand.
v2:
- Completely rework the functionality
v3:
- Avoid a NULL pointer dereference assigning manager->mem_type
v4:
- Remove some leftover code causing build problems
Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 ++
drivers/gpu/drm/ttm/ttm_resource.c | 90 +++++++++++++++++++++++++-
drivers/gpu/drm/xe/xe_vm.c | 4 ++
include/drm/ttm/ttm_device.h | 2 +
include/drm/ttm/ttm_resource.h | 55 ++++++++++------
5 files changed, 134 insertions(+), 21 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 4299ce386322..18bf174c8d47 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2368,6 +2368,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
if (r)
return r;
+ ttm_lru_bulk_move_init(&vm->lru_bulk_move);
+
vm->is_compute_context = false;
vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
@@ -2431,6 +2433,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
error_free_delayed:
dma_fence_put(vm->last_tlb_flush);
dma_fence_put(vm->last_unlocked);
+ ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
amdgpu_vm_fini_entities(vm);
return r;
@@ -2587,6 +2590,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
}
}
+ ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
}
/**
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index ccc31ad85e3b..89f19fb6e2b3 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -32,6 +32,49 @@
#include <drm/drm_util.h>
+/* Detach the cursor from the bulk move list*/
+static void
+ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
+{
+ cursor->bulk = NULL;
+ list_del_init(&cursor->bulk_link);
+}
+
+/* Move the cursor to the end of the bulk move list it's in */
+static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,
+ struct ttm_resource_cursor *cursor)
+{
+ struct ttm_lru_bulk_move_pos *pos;
+
+ if (WARN_ON_ONCE(bulk != cursor->bulk)) {
+ list_del_init(&cursor->bulk_link);
+ return;
+ }
+
+ pos = &bulk->pos[cursor->man->mem_type][cursor->priority];
+ if (pos)
+ list_move(&cursor->hitch.link, &pos->last->lru.link);
+ ttm_resource_cursor_clear_bulk(cursor);
+}
+
+/* Move all cursors attached to a bulk move to its end */
+static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)
+{
+ struct ttm_resource_cursor *cursor, *next;
+
+ list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
+ ttm_resource_cursor_move_bulk_tail(bulk, cursor);
+}
+
+/* Remove a cursor from an empty bulk move list */
+static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)
+{
+ struct ttm_resource_cursor *cursor, *next;
+
+ list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
+ ttm_resource_cursor_clear_bulk(cursor);
+}
+
/**
* ttm_resource_cursor_fini_locked() - Finalize the LRU list cursor usage
* @cursor: The struct ttm_resource_cursor to finalize.
@@ -44,6 +87,7 @@ void ttm_resource_cursor_fini_locked(struct ttm_resource_cursor *cursor)
{
lockdep_assert_held(&cursor->man->bdev->lru_lock);
list_del_init(&cursor->hitch.link);
+ ttm_resource_cursor_clear_bulk(cursor);
}
/**
@@ -72,9 +116,27 @@ void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
{
memset(bulk, 0, sizeof(*bulk));
+ INIT_LIST_HEAD(&bulk->cursor_list);
}
EXPORT_SYMBOL(ttm_lru_bulk_move_init);
+/**
+ * ttm_lru_bulk_move_fini - finalize a bulk move structure
+ * @bdev: The struct ttm_device
+ * @bulk: the structure to finalize
+ *
+ * Sanity checks that bulk moves don't have any
+ * resources left and hence no cursors attached.
+ */
+void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
+ struct ttm_lru_bulk_move *bulk)
+{
+ spin_lock(&bdev->lru_lock);
+ ttm_bulk_move_drop_cursors(bulk);
+ spin_unlock(&bdev->lru_lock);
+}
+EXPORT_SYMBOL(ttm_lru_bulk_move_fini);
+
/**
* ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
*
@@ -87,6 +149,7 @@ void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
{
unsigned i, j;
+ ttm_bulk_move_adjust_cursors(bulk);
for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
@@ -418,6 +481,7 @@ void ttm_resource_manager_init(struct ttm_resource_manager *man,
man->bdev = bdev;
man->size = size;
man->usage = 0;
+ man->mem_type = TTM_NUM_MEM_TYPES;
for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
INIT_LIST_HEAD(&man->lru[i]);
@@ -514,6 +578,27 @@ void ttm_resource_manager_debug(struct ttm_resource_manager *man,
}
EXPORT_SYMBOL(ttm_resource_manager_debug);
+static void
+ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
+ struct ttm_lru_item *next_lru)
+{
+ struct ttm_resource *next = ttm_lru_item_to_res(next_lru);
+ struct ttm_lru_bulk_move *bulk = NULL;
+ struct ttm_buffer_object *bo = next->bo;
+
+ lockdep_assert_held(&cursor->man->bdev->lru_lock);
+ if (bo && bo->resource == next)
+ bulk = bo->bulk_move;
+
+ if (cursor->bulk != bulk) {
+ if (bulk)
+ list_move_tail(&cursor->bulk_link, &bulk->cursor_list);
+ else
+ list_del_init(&cursor->bulk_link);
+ cursor->bulk = bulk;
+ }
+}
+
/**
* ttm_resource_manager_next() - Continue iterating over the resource manager
* resources
@@ -533,6 +618,7 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
lru = &cursor->hitch;
list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
if (ttm_lru_item_is_res(lru)) {
+ ttm_resource_cursor_check_bulk(cursor, lru);
list_move(&cursor->hitch.link, &lru->link);
return ttm_lru_item_to_res(lru);
}
@@ -542,9 +628,10 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
break;
list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
+ ttm_resource_cursor_clear_bulk(cursor);
} while (true);
- list_del_init(&cursor->hitch.link);
+ ttm_resource_cursor_fini_locked(cursor);
return NULL;
}
@@ -569,6 +656,7 @@ ttm_resource_manager_first(struct ttm_resource_manager *man,
cursor->priority = 0;
cursor->man = man;
ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);
+ INIT_LIST_HEAD(&cursor->bulk_link);
list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
return ttm_resource_manager_next(cursor);
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 694fbb546372..af351c525d67 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1336,6 +1336,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
+ ttm_lru_bulk_move_init(&vm->lru_bulk_move);
+
INIT_LIST_HEAD(&vm->preempt.exec_queues);
vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */
@@ -1453,6 +1455,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
mutex_destroy(&vm->snap_mutex);
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
+ ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
kfree(vm);
if (!(flags & XE_VM_FLAG_MIGRATION))
xe_device_mem_access_put(xe);
@@ -1593,6 +1596,7 @@ static void vm_destroy_work_func(struct work_struct *w)
trace_xe_vm_free(vm);
dma_fence_put(vm->rebind_fence);
+ ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
kfree(vm);
}
diff --git a/include/drm/ttm/ttm_device.h b/include/drm/ttm/ttm_device.h
index c22f30535c84..8816d5ba8cb3 100644
--- a/include/drm/ttm/ttm_device.h
+++ b/include/drm/ttm/ttm_device.h
@@ -285,6 +285,8 @@ static inline void ttm_set_driver_manager(struct ttm_device *bdev, int type,
{
BUILD_BUG_ON(__builtin_constant_p(type) && type >= TTM_NUM_MEM_TYPES);
bdev->man_drv[type] = manager;
+ if (manager)
+ manager->mem_type = type;
}
int ttm_device_init(struct ttm_device *bdev, const struct ttm_device_funcs *funcs,
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index dfc01258c981..6b7bf4c18997 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -211,6 +211,9 @@ struct ttm_resource_manager {
* bdev->lru_lock.
*/
uint64_t usage;
+
+ /** @mem_type: The memory type used for this manager. */
+ unsigned int mem_type;
};
/**
@@ -269,25 +272,6 @@ ttm_lru_item_to_res(struct ttm_lru_item *item)
return container_of(item, struct ttm_resource, lru);
}
-/**
- * struct ttm_resource_cursor
- * @man: The resource manager currently being iterated over
- * @hitch: A hitch list node inserted before the next resource
- * to iterate over.
- * @priority: the current priority
- *
- * Cursor to iterate over the resources in a manager.
- */
-struct ttm_resource_cursor {
- struct ttm_resource_manager *man;
- struct ttm_lru_item hitch;
- unsigned int priority;
-};
-
-void ttm_resource_cursor_fini_locked(struct ttm_resource_cursor *cursor);
-
-void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
-
/**
* struct ttm_lru_bulk_move_pos
*
@@ -303,8 +287,9 @@ struct ttm_lru_bulk_move_pos {
/**
* struct ttm_lru_bulk_move
- *
* @pos: first/last lru entry for resources in the each domain/priority
+ * @cursor_list: The list of cursors currently traversing any of
+ * the sublists of @pos. Protected by the ttm device's lru_lock.
*
* Container for the current bulk move state. Should be used with
* ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
@@ -314,8 +299,36 @@ struct ttm_lru_bulk_move_pos {
*/
struct ttm_lru_bulk_move {
struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
+ struct list_head cursor_list;
+};
+
+/**
+ * struct ttm_resource_cursor
+ * @man: The resource manager currently being iterated over
+ * @hitch: A hitch list node inserted before the next resource
+ * to iterate over.
+ * @bulk_link: A list link for the list of cursors traversing the
+ * bulk sublist of @bulk. Protected by the ttm device's lru_lock.
+ * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is
+ * inserted to. NULL if none. Never dereference this pointer since
+ * the struct ttm_lru_bulk_move object pointed to might have been
+ * freed. The pointer is only for comparison.
+ * @priority: the current priority
+ *
+ * Cursor to iterate over the resources in a manager.
+ */
+struct ttm_resource_cursor {
+ struct ttm_resource_manager *man;
+ struct ttm_lru_item hitch;
+ struct list_head bulk_link;
+ struct ttm_lru_bulk_move *bulk;
+ unsigned int priority;
};
+void ttm_resource_cursor_fini_locked(struct ttm_resource_cursor *cursor);
+
+void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
+
/**
* struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
* struct sg_table backed struct ttm_resource.
@@ -404,6 +417,8 @@ ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
+void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
+ struct ttm_lru_bulk_move *bulk);
void ttm_resource_add_bulk_move(struct ttm_resource *res,
struct ttm_buffer_object *bo);
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 4/8] drm/ttm: Allow continued swapout after -ENOSPC falure
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (2 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 3/8] drm/ttm, drm/amdgpu, drm/xe: Consider hitch moves within bulk sublist moves Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 5/8] drm/ttm: Add a virtual base class for graphics memory backup Thomas Hellström
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
The -ENOSPC failure from ttm_bo_swapout() meant that the lru_lock
was dropped and simply restarting the iteration meant we'd likely
hit the same error again on the same resource. Now that we can
restart the iteration even if the lock was dropped, do that.
Cc: Christian König <christian.koenig@amd.com>
Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com>
Cc: <dri-devel@lists.freedesktop.org>
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_device.c | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index e8a6a1dab669..4a030b4bc848 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -168,15 +168,20 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
num_pages = PFN_UP(bo->base.size);
ret = ttm_bo_swapout(bo, ctx, gfp_flags);
- /* ttm_bo_swapout has dropped the lru_lock */
- if (!ret) {
- ttm_resource_cursor_fini(&cursor);
- return num_pages;
- }
- if (ret != -EBUSY) {
- ttm_resource_cursor_fini(&cursor);
- return ret;
+ /* Couldn't swap out, and retained the lru_lock */
+ if (ret == -EBUSY)
+ continue;
+ /* Couldn't swap out and dropped the lru_lock */
+ if (ret == -ENOSPC) {
+ spin_lock(&bdev->lru_lock);
+ continue;
}
+ /*
+ * Dropped the lock and either succeeded or
+ * hit an error that forces us to break.
+ */
+ ttm_resource_cursor_fini(&cursor);
+ return ret ? ret : num_pages;
}
}
ttm_resource_cursor_fini_locked(&cursor);
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 5/8] drm/ttm: Add a virtual base class for graphics memory backup
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (3 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 4/8] drm/ttm: Allow continued swapout after -ENOSPC falure Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 6/8] drm/ttm/pool: Provide a helper to shrink pages Thomas Hellström
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
Initially intended for experimenting with different backup
solutions (shmem vs direct swap cache insertion), abstract
the backup destination using a virtual base class.
Also provide a sample implementation for shmem.
While when settling on a preferred backup solution, one could
perhaps skip the abstraction, this functionality may actually
come in handy for configurable dedicated graphics memory
backup to fast nvme files or similar, whithout affecting
swap-space. Could indeed be useful for VRAM backup on S4 and
other cases.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/Makefile | 2 +-
drivers/gpu/drm/ttm/ttm_backup_shmem.c | 122 +++++++++++++++++++++++++
include/drm/ttm/ttm_backup.h | 48 ++++++++++
3 files changed, 171 insertions(+), 1 deletion(-)
create mode 100644 drivers/gpu/drm/ttm/ttm_backup_shmem.c
create mode 100644 include/drm/ttm/ttm_backup.h
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile
index dad298127226..5e980dd90e41 100644
--- a/drivers/gpu/drm/ttm/Makefile
+++ b/drivers/gpu/drm/ttm/Makefile
@@ -4,7 +4,7 @@
ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \
ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \
- ttm_device.o ttm_sys_manager.o
+ ttm_device.o ttm_sys_manager.o ttm_backup_shmem.o
ttm-$(CONFIG_AGP) += ttm_agp_backend.o
obj-$(CONFIG_DRM_TTM) += ttm.o
diff --git a/drivers/gpu/drm/ttm/ttm_backup_shmem.c b/drivers/gpu/drm/ttm/ttm_backup_shmem.c
new file mode 100644
index 000000000000..e3596dee17d7
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_backup_shmem.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <drm/ttm/ttm_backup.h>
+#include <linux/page-flags.h>
+
+struct ttm_backup_shmem {
+ struct ttm_backup backup;
+ struct file *filp;
+};
+
+static struct ttm_backup_shmem *to_backup_shmem(struct ttm_backup *backup)
+{
+ return container_of(backup, struct ttm_backup_shmem, backup);
+}
+
+static void ttm_backup_shmem_drop(struct ttm_backup *backup, unsigned long handle)
+{
+ handle -= 1;
+ shmem_truncate_range(file_inode(to_backup_shmem(backup)->filp), handle,
+ handle + 1);
+}
+
+static int ttm_backup_shmem_copy_page(struct ttm_backup *backup, struct page *dst,
+ unsigned long handle, bool killable)
+{
+ struct file *filp = to_backup_shmem(backup)->filp;
+ struct address_space *mapping = filp->f_mapping;
+ struct folio *from_folio;
+
+ handle -= 1;
+ from_folio = shmem_read_folio(mapping, handle);
+ if (IS_ERR(from_folio))
+ return PTR_ERR(from_folio);
+
+ copy_highpage(dst, folio_file_page(from_folio, handle));
+ folio_put(from_folio);
+
+ return 0;
+}
+
+static unsigned long
+ttm_backup_shmem_backup_page(struct ttm_backup *backup, struct page *page,
+ bool writeback, pgoff_t i, gfp_t page_gfp,
+ gfp_t alloc_gfp)
+{
+ struct file *filp = to_backup_shmem(backup)->filp;
+ struct address_space *mapping = filp->f_mapping;
+ unsigned long handle = 0;
+ struct folio *to_folio;
+ int ret;
+
+ to_folio = shmem_read_folio_gfp(mapping, i, alloc_gfp);
+ if (IS_ERR(to_folio))
+ return handle;
+
+ folio_mark_accessed(to_folio);
+ folio_lock(to_folio);
+ folio_mark_dirty(to_folio);
+ copy_highpage(folio_file_page(to_folio, i), page);
+ handle = i + 1;
+
+ if (writeback && !folio_mapped(to_folio) && folio_clear_dirty_for_io(to_folio)) {
+ struct writeback_control wbc = {
+ .sync_mode = WB_SYNC_NONE,
+ .nr_to_write = SWAP_CLUSTER_MAX,
+ .range_start = 0,
+ .range_end = LLONG_MAX,
+ .for_reclaim = 1,
+ };
+ folio_set_reclaim(to_folio);
+ ret = mapping->a_ops->writepage(folio_page(to_folio, 0), &wbc);
+ if (!folio_test_writeback(to_folio))
+ folio_clear_reclaim(to_folio);
+ /* If writepage succeeds, it unlocks the folio */
+ if (ret)
+ folio_unlock(to_folio);
+ } else {
+ folio_unlock(to_folio);
+ }
+
+ folio_put(to_folio);
+
+ return handle;
+}
+
+static void ttm_backup_shmem_fini(struct ttm_backup *backup)
+{
+ struct ttm_backup_shmem *sbackup = to_backup_shmem(backup);
+
+ fput(sbackup->filp);
+ kfree(sbackup);
+}
+
+static const struct ttm_backup_ops ttm_backup_shmem_ops = {
+ .drop = ttm_backup_shmem_drop,
+ .copy_backed_up_page = ttm_backup_shmem_copy_page,
+ .backup_page = ttm_backup_shmem_backup_page,
+ .fini = ttm_backup_shmem_fini,
+};
+
+struct ttm_backup *ttm_backup_shmem_create(loff_t size)
+{
+ struct ttm_backup_shmem *sbackup =
+ kzalloc(sizeof(*sbackup), GFP_KERNEL | __GFP_ACCOUNT);
+
+ if (!sbackup)
+ return ERR_PTR(-ENOMEM);
+
+ sbackup->filp = shmem_file_setup("ttm shmem backup", size, 0);
+ if (IS_ERR(sbackup->filp)) {
+ kfree(sbackup);
+ return ERR_CAST(sbackup->filp);
+ }
+
+ sbackup->backup.ops = &ttm_backup_shmem_ops;
+
+ return &sbackup->backup;
+}
+EXPORT_SYMBOL_GPL(ttm_backup_shmem_create);
diff --git a/include/drm/ttm/ttm_backup.h b/include/drm/ttm/ttm_backup.h
new file mode 100644
index 000000000000..1ba80fe0cbcf
--- /dev/null
+++ b/include/drm/ttm/ttm_backup.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _TTM_BACKUP_H_
+#define _TTM_BACKUP_H_
+
+#include <linux/mm_types.h>
+#include <linux/shmem_fs.h>
+
+struct ttm_backup;
+
+static inline struct page *
+ttm_backup_handle_to_page_ptr(unsigned long handle)
+{
+ return (struct page *)(handle << 1 | 1);
+}
+
+static inline bool ttm_backup_page_ptr_is_handle(const struct page *page)
+{
+ return (unsigned long)page & 1;
+}
+
+static inline unsigned long
+ttm_backup_page_ptr_to_handle(const struct page *page)
+{
+ WARN_ON(!ttm_backup_page_ptr_is_handle(page));
+ return (unsigned long)page >> 1;
+}
+
+struct ttm_backup_ops {
+ void (*drop)(struct ttm_backup *backup, unsigned long handle);
+ int (*copy_backed_up_page)(struct ttm_backup *backup, struct page *dst,
+ unsigned long handle, bool intr);
+ unsigned long (*backup_page)(struct ttm_backup *backup, struct page *page,
+ bool writeback, pgoff_t i, gfp_t page_gfp,
+ gfp_t alloc_gfp);
+ void (*fini)(struct ttm_backup *backup);
+};
+
+struct ttm_backup {
+ const struct ttm_backup_ops *ops;
+};
+
+struct ttm_backup *ttm_backup_shmem_create(loff_t size);
+
+#endif
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 6/8] drm/ttm/pool: Provide a helper to shrink pages.
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (4 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 5/8] drm/ttm: Add a virtual base class for graphics memory backup Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 7/8] drm/xe, drm/ttm: Provide a generic LRU walker helper Thomas Hellström
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
Provide a helper to shrink ttm_tt page-vectors on a per-page
basis. A ttm_backup backend could then in theory get away with
allocating a single temporary page for each struct ttm_tt.
This is accomplished by splitting larger pages before trying to
back them up.
In the future we could allow ttm_backup to handle backing up
large pages as well, but currently there's no benefit in
doing that, since the shmem backup backend would have to
split those anyway to avoid allocating too much temporary
memory, and if the backend instead inserts pages into the
swap-cache, those are split on reclaim by the core.
Due to potential backup- and recover errors, allow partially swapped
out struct ttm_tt's, although mark them as swapped out stopping them
from being swapped out a second time. More details in the ttm_pool.c
DOC section.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_pool.c | 388 +++++++++++++++++++++++++++++++--
drivers/gpu/drm/ttm/ttm_tt.c | 32 +++
include/drm/ttm/ttm_pool.h | 4 +
include/drm/ttm/ttm_tt.h | 12 +
4 files changed, 423 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 112438d965ff..b9cdbf2ffa00 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -41,6 +41,7 @@
#include <asm/set_memory.h>
#endif
+#include <drm/ttm/ttm_backup.h>
#include <drm/ttm/ttm_pool.h>
#include <drm/ttm/ttm_tt.h>
#include <drm/ttm/ttm_bo.h>
@@ -58,6 +59,32 @@ struct ttm_pool_dma {
unsigned long vaddr;
};
+/**
+ * struct ttm_pool_tt_restore - State representing restore from swap.
+ * @alloced_pages: Total number of already allocated pages for the ttm_tt.
+ * @restored_pages: Number of (sub) pages restored from swap for this
+ * chunk of 1 << @order pages.
+ * @first_page: The ttm page ptr representing for @old_pages[0].
+ * @caching_divide: Page pointer where subsequent pages are cached.
+ * @old_pages: Backup copy of page pointers that were replaced by the new
+ * page allocation.
+ * @pool: The pool used for page allocation while restoring.
+ * @order: The order of the last page allocated while restoring.
+ *
+ * Recovery from swap space might fail when we've recovered less than the
+ * full ttm_tt. In order not to loose any data (yet), keep information
+ * around that allows us to restart a failed ttm swap-space recovery.
+ */
+struct ttm_pool_tt_restore {
+ pgoff_t alloced_pages;
+ pgoff_t restored_pages;
+ struct page **first_page;
+ struct page **caching_divide;
+ struct ttm_pool *pool;
+ unsigned int order;
+ struct page *old_pages[];
+};
+
static unsigned long page_pool_size;
MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");
@@ -348,11 +375,101 @@ static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p)
return p->private;
}
+/*
+ * To be able to insert single pages into the swap cache directly,
+ * we need to split multi-order page allocations and make them look
+ * like single page-allocations.
+ */
+static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p)
+{
+ unsigned int order = ttm_pool_page_order(pool, p);
+ pgoff_t nr;
+
+ if (!order)
+ return;
+
+ split_page(p, order);
+ nr = 1UL << order;
+ while (nr--)
+ (p++)->private = 0;
+}
+
+/**
+ * DOC: Partial shrinking and restoration of a struct ttm_tt.
+ *
+ * Swapout using swap_backup_folio() and swapin using swap_copy_folio() may fail.
+ * The former most likely due to lack of swap-space or memory, the latter due
+ * to lack of memory or because of signal interruption during waits.
+ *
+ * Swapout failure is easily handled by using a ttm_tt pages vector that holds
+ * both swap entries and page pointers. This has to be taken into account when
+ * restoring such a ttm_tt from swap, and when freeing it while swapped out.
+ * When restoring, for simplicity, new pages are actually allocated from the
+ * pool and the contents of any old pages are copied in and then the old pages
+ * are released.
+ *
+ * For swapin failures, the struct ttm_pool_tt_restore holds sufficient state
+ * to be able to resume an interrupted restore, and that structure is freed once
+ * the restoration is complete. If the struct ttm_tt is destroyed while there
+ * is a valid struct ttm_pool_tt_restore attached, that is also properly taken
+ * care of.
+ */
+
+static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore)
+{
+ return restore && restore->restored_pages < (1 << restore->order);
+}
+
+static int ttm_pool_swapin(struct ttm_pool_tt_restore *restore,
+ struct ttm_backup *backup,
+ struct ttm_operation_ctx *ctx)
+{
+ unsigned int i, nr = 1 << restore->order;
+ int ret = 0;
+
+ if (!ttm_pool_restore_valid(restore))
+ return 0;
+
+ for (i = restore->restored_pages; i < nr; ++i) {
+ struct page *p = restore->old_pages[i];
+
+ if (ttm_backup_page_ptr_is_handle(p)) {
+ unsigned long handle = ttm_backup_page_ptr_to_handle(p);
+
+ if (handle == 0)
+ continue;
+
+ ret = backup->ops->copy_backed_up_page
+ (backup, restore->first_page[i],
+ handle, ctx->interruptible);
+ if (ret)
+ break;
+
+ backup->ops->drop(backup, handle);
+ } else if (p) {
+ /*
+ * We could probably avoid splitting the old page
+ * using clever logic, but ATM we don't care.
+ */
+ ttm_pool_split_for_swap(restore->pool, p);
+ copy_highpage(restore->first_page[i], p);
+ __free_pages(p, 0);
+ }
+
+ restore->restored_pages++;
+ restore->old_pages[i] = NULL;
+ cond_resched();
+ }
+
+ return ret;
+}
+
/* Called when we got a page, either from a pool or newly allocated */
static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
struct page *p, dma_addr_t **dma_addr,
unsigned long *num_pages,
- struct page ***pages)
+ struct page ***pages,
+ struct ttm_pool_tt_restore *restore)
{
unsigned int i;
int r;
@@ -363,6 +480,16 @@ static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
return r;
}
+ if (restore) {
+ memcpy(restore->old_pages, *pages,
+ (1 << order) * sizeof(*restore->old_pages));
+ memset(*pages, 0, (1 << order) * sizeof(**pages));
+ restore->order = order;
+ restore->restored_pages = 0;
+ restore->first_page = *pages;
+ restore->alloced_pages += 1UL << order;
+ }
+
*num_pages -= 1 << order;
for (i = 1 << order; i; --i, ++(*pages), ++p)
**pages = p;
@@ -388,22 +515,39 @@ static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,
pgoff_t start_page, pgoff_t end_page)
{
struct page **pages = &tt->pages[start_page];
+ struct ttm_backup *backup = tt->backup;
unsigned int order;
pgoff_t i, nr;
for (i = start_page; i < end_page; i += nr, pages += nr) {
struct ttm_pool_type *pt = NULL;
+ struct page *p = *pages;
+
+ if (ttm_backup_page_ptr_is_handle(p)) {
+ unsigned long handle = ttm_backup_page_ptr_to_handle(p);
+
+ nr = 1;
+ if (handle != 0)
+ backup->ops->drop(backup, handle);
+ continue;
+ }
- order = ttm_pool_page_order(pool, *pages);
- nr = (1UL << order);
- if (tt->dma_address)
- ttm_pool_unmap(pool, tt->dma_address[i], nr);
+ if (pool) {
+ order = ttm_pool_page_order(pool, p);
+ nr = (1UL << order);
+ if (tt->dma_address)
+ ttm_pool_unmap(pool, tt->dma_address[i], nr);
+
+ pt = ttm_pool_select_type(pool, caching, order);
+ } else {
+ order = p->private;
+ nr = (1UL << order);
+ }
- pt = ttm_pool_select_type(pool, caching, order);
if (pt)
- ttm_pool_type_give(pt, *pages);
+ ttm_pool_type_give(pt, p);
else
- ttm_pool_free_page(pool, caching, order, *pages);
+ ttm_pool_free_page(pool, caching, order, p);
}
}
@@ -447,9 +591,37 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
else
gfp_flags |= GFP_HIGHUSER;
- for (order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages));
- num_pages;
- order = min_t(unsigned int, order, __fls(num_pages))) {
+ order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages));
+
+ if (tt->page_flags & TTM_TT_FLAG_PRIV_SHRUNKEN) {
+ if (!tt->restore) {
+ gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
+
+ if (ctx->gfp_retry_mayfail)
+ gfp |= __GFP_RETRY_MAYFAIL;
+
+ tt->restore =
+ kvzalloc(struct_size(tt->restore, old_pages,
+ (size_t)1 << order), gfp);
+ /* RFC: Possibly loop on -ENOMEM and reduce order. */
+ if (!tt->restore)
+ return -ENOMEM;
+ } else if (ttm_pool_restore_valid(tt->restore)) {
+ struct ttm_pool_tt_restore *restore = tt->restore;
+
+ num_pages -= restore->alloced_pages;
+ order = min_t(unsigned int, order, __fls(num_pages));
+ pages += restore->alloced_pages;
+ r = ttm_pool_swapin(restore, tt->backup, ctx);
+ if (r)
+ return r;
+ caching = restore->caching_divide;
+ }
+
+ tt->restore->pool = pool;
+ }
+
+ for (; num_pages; order = min_t(unsigned int, order, __fls(num_pages))) {
struct ttm_pool_type *pt;
page_caching = tt->caching;
@@ -466,11 +638,18 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
r = ttm_pool_page_allocated(pool, order, p,
&dma_addr,
&num_pages,
- &pages);
+ &pages,
+ tt->restore);
if (r)
goto error_free_page;
caching = pages;
+ if (ttm_pool_restore_valid(tt->restore)) {
+ r = ttm_pool_swapin(tt->restore, tt->backup, ctx);
+ if (r)
+ goto error_free_all;
+ }
+
if (num_pages < (1 << order))
break;
@@ -490,9 +669,17 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
caching = pages;
}
r = ttm_pool_page_allocated(pool, order, p, &dma_addr,
- &num_pages, &pages);
+ &num_pages, &pages,
+ tt->restore);
if (r)
goto error_free_page;
+
+ if (ttm_pool_restore_valid(tt->restore)) {
+ r = ttm_pool_swapin(tt->restore, tt->backup, ctx);
+ if (r)
+ goto error_free_all;
+ }
+
if (PageHighMem(p))
caching = pages;
}
@@ -511,12 +698,26 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
if (r)
goto error_free_all;
+ if (tt->restore) {
+ kvfree(tt->restore);
+ tt->restore = NULL;
+ }
+
+ if (tt->page_flags & TTM_TT_FLAG_PRIV_SHRUNKEN)
+ tt->page_flags &= ~(TTM_TT_FLAG_PRIV_SHRUNKEN |
+ TTM_TT_FLAG_SWAPPED);
+
return 0;
error_free_page:
ttm_pool_free_page(pool, page_caching, order, p);
error_free_all:
+ if (tt->page_flags & TTM_TT_FLAG_PRIV_SHRUNKEN) {
+ tt->restore->caching_divide = caching;
+ return r;
+ }
+
num_pages = tt->num_pages - num_pages;
caching_divide = caching - tt->pages;
ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide);
@@ -543,6 +744,167 @@ void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)
}
EXPORT_SYMBOL(ttm_pool_free);
+/**
+ * ttm_pool_release_shrunken() - Release content of a swapped-out struct ttm_tt
+ * @tt: The struct ttm_tt.
+ *
+ * Release swap entries with associated content or any remaining pages of
+ * a swapped-out struct ttm_tt.
+ */
+void ttm_pool_release_shrunken(struct ttm_tt *tt)
+{
+ struct ttm_backup *backup = tt->backup;
+ struct ttm_pool_tt_restore *restore;
+ pgoff_t i, start_page = 0;
+ unsigned long handle;
+
+ if (!(tt->page_flags & TTM_TT_FLAG_PRIV_SHRUNKEN))
+ return;
+
+ restore = tt->restore;
+
+ if (ttm_pool_restore_valid(restore)) {
+ pgoff_t nr = 1UL << restore->order;
+
+ for (i = restore->restored_pages; i < nr; ++i) {
+ struct page *p = restore->old_pages[i];
+
+ if (ttm_backup_page_ptr_is_handle(p)) {
+ handle = ttm_backup_page_ptr_to_handle(p);
+ if (handle == 0)
+ continue;
+
+ backup->ops->drop(backup, handle);
+ } else if (p) {
+ ttm_pool_split_for_swap(restore->pool, p);
+ __free_pages(p, 0);
+ }
+ }
+ }
+
+ if (restore) {
+ pgoff_t mid = restore->caching_divide - tt->pages;
+
+ start_page = restore->alloced_pages;
+ /* Pages that might be dma-mapped and non-cached */
+ ttm_pool_free_range(restore->pool, tt, tt->caching,
+ 0, mid);
+ /* Pages that might be dma-mapped but cached */
+ ttm_pool_free_range(restore->pool, tt, ttm_cached,
+ mid, restore->alloced_pages);
+ }
+
+ /* Shrunken pages. Cached and not dma-mapped. */
+ ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages);
+
+ if (restore) {
+ kvfree(restore);
+ tt->restore = NULL;
+ }
+
+ tt->page_flags &= ~(TTM_TT_FLAG_PRIV_SHRUNKEN | TTM_TT_FLAG_SWAPPED);
+}
+
+/**
+ * ttm_pool_shrink_tt() - Swap out or purge a struct ttm_tt
+ * @pool: The pool used when allocating the struct ttm_tt.
+ * @ttm: The struct ttm_tt.
+ *
+ * Swap out or purge a struct ttm_tt. If @ttm is marked purgeable, then
+ * all pages will be freed directly to the system rather than to the pool
+ * they were allocated from, making the function behave similarly to
+ * ttm_pool_free(). If @ttm is not marked purgeable, the pages will be
+ * inserted into the swap cache instead, exchanged for a swap entry.
+ * A subsequent call to ttm_pool_alloc() will then read back the content and
+ * a subsequent call to ttm_pool_release_shrunken() will drop it.
+ * If swapout of a page fails for whatever reason, @ttm will still be
+ * partially swapped out, retaining those pages for which swapout fails.
+ *
+ * @Return: Number of pages actually swapped out or freed, or negative
+ * error code on error.
+ */
+long ttm_pool_shrink_tt(struct ttm_pool *pool, struct ttm_tt *ttm, bool purge)
+{
+ struct ttm_backup *backup = ttm->backup;
+ struct page *page;
+ unsigned long handle;
+ gfp_t alloc_gfp;
+ gfp_t gfp;
+ int ret = 0;
+ pgoff_t shrunken = 0;
+ pgoff_t i, num_pages;
+
+ if ((!get_nr_swap_pages() && purge) ||
+ pool->use_dma_alloc ||
+ (ttm->page_flags & TTM_TT_FLAG_PRIV_SHRUNKEN))
+ return -EBUSY;
+
+#ifdef CONFIG_X86
+ /* Anything returned to the system needs to be cached. */
+ if (ttm->caching != ttm_cached)
+ set_pages_array_wb(ttm->pages, ttm->num_pages);
+#endif
+
+ if (ttm->dma_address || purge) {
+ for (i = 0; i < ttm->num_pages; i += num_pages) {
+ unsigned int order;
+
+ page = ttm->pages[i];
+ if (unlikely(!page))
+ continue;
+
+ order = 1UL << ttm_pool_page_order(pool, page);
+ num_pages = 1UL << order;
+ if (ttm->dma_address)
+ ttm_pool_unmap(pool, ttm->dma_address[i],
+ num_pages);
+ if (purge) {
+ shrunken += num_pages;
+ __free_pages(page, order);
+ memset(ttm->pages + i, 0,
+ num_pages * sizeof(*ttm->pages));
+ }
+ }
+ }
+
+ if (purge)
+ return shrunken;
+
+ if (pool->use_dma32)
+ gfp = GFP_DMA32;
+ else
+ gfp = GFP_HIGHUSER;
+
+ alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL;
+
+ for (i = 0; i < ttm->num_pages; ++i) {
+ page = ttm->pages[i];
+ if (unlikely(!page))
+ continue;
+
+ ttm_pool_split_for_swap(pool, page);
+
+ set_page_dirty_lock(page);
+ handle = backup->ops->backup_page(backup, page, false, i, gfp, alloc_gfp);
+ if (handle) {
+ ttm->pages[i] = ttm_backup_handle_to_page_ptr(handle);
+ put_page(page);
+ shrunken++;
+ } else {
+ /* We allow partially shrunken tts */
+ ret = -ENOMEM;
+ break;
+ }
+ cond_resched();
+ }
+
+ if (shrunken)
+ ttm->page_flags |= (TTM_TT_FLAG_PRIV_SHRUNKEN |
+ TTM_TT_FLAG_SWAPPED);
+
+ return shrunken ? shrunken : ret;
+}
+
/**
* ttm_pool_init - Initialize a pool
*
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 578a7c37f00b..3670519703d7 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -39,6 +39,7 @@
#include <drm/drm_cache.h>
#include <drm/drm_device.h>
#include <drm/drm_util.h>
+#include <drm/ttm/ttm_backup.h>
#include <drm/ttm/ttm_bo.h>
#include <drm/ttm/ttm_tt.h>
@@ -157,6 +158,7 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm,
ttm->swap_storage = NULL;
ttm->sg = bo->sg;
ttm->caching = caching;
+ ttm->restore = NULL;
}
int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo,
@@ -181,6 +183,12 @@ void ttm_tt_fini(struct ttm_tt *ttm)
fput(ttm->swap_storage);
ttm->swap_storage = NULL;
+ ttm_pool_release_shrunken(ttm);
+ if (ttm->backup) {
+ ttm->backup->ops->fini(ttm->backup);
+ ttm->backup = NULL;
+ }
+
if (ttm->pages)
kvfree(ttm->pages);
else
@@ -251,6 +259,30 @@ int ttm_tt_swapin(struct ttm_tt *ttm)
return ret;
}
+/**
+ * ttm_tt_shrink() - Helper for the driver bo_shrink() method.
+ * @bdev: The TTM device.
+ * @tt: The struct ttm_tt.
+ *
+ * Helper for a TTM driver to use from the bo_shrink() method to shrink
+ * a struct ttm_tt, after it has done the necessary unbinding. This function
+ * will update the page accounting and call ttm_pool_shrink_tt to free pages
+ * or move them to the swap cache.
+ *
+ * Return: Number of pages freed or swapped out, or negative error code on
+ * error.
+ */
+long ttm_tt_shrink(struct ttm_device *bdev, struct ttm_tt *tt, bool purge)
+{
+ long ret = ttm_pool_shrink_tt(&bdev->pool, tt, purge);
+
+ if (ret > 0)
+ tt->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED;
+
+ return ret;
+}
+EXPORT_SYMBOL(ttm_tt_shrink);
+
/**
* ttm_tt_swapout - swap out tt object
*
diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
index 160d954a261e..7f9e1952914a 100644
--- a/include/drm/ttm/ttm_pool.h
+++ b/include/drm/ttm/ttm_pool.h
@@ -89,6 +89,10 @@ void ttm_pool_fini(struct ttm_pool *pool);
int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
+void ttm_pool_release_shrunken(struct ttm_tt *tt);
+
+long ttm_pool_shrink_tt(struct ttm_pool *pool, struct ttm_tt *ttm, bool purge);
+
int ttm_pool_mgr_init(unsigned long num_pages);
void ttm_pool_mgr_fini(void);
diff --git a/include/drm/ttm/ttm_tt.h b/include/drm/ttm/ttm_tt.h
index 2b9d856ff388..04d24990fb73 100644
--- a/include/drm/ttm/ttm_tt.h
+++ b/include/drm/ttm/ttm_tt.h
@@ -32,11 +32,13 @@
#include <drm/ttm/ttm_caching.h>
#include <drm/ttm/ttm_kmap_iter.h>
+struct ttm_backup;
struct ttm_device;
struct ttm_tt;
struct ttm_resource;
struct ttm_buffer_object;
struct ttm_operation_ctx;
+struct ttm_pool_tt_restore;
/**
* struct ttm_tt - This is a structure holding the pages, caching- and aperture
@@ -85,6 +87,10 @@ struct ttm_tt {
* fault handling abuses the DMA api a bit and dma_map_attrs can't be
* used to assure pgprot always matches.
*
+ * TTM_TT_FLAG_PRIV_SHRUNKEN: TTM internal only. This is set if the
+ * struct ttm_tt has been (possibly partially) swapped out to the
+ * swap cache.
+ *
* TTM_TT_FLAG_PRIV_POPULATED: TTM internal only. DO NOT USE. This is
* set by TTM after ttm_tt_populate() has successfully returned, and is
* then unset when TTM calls ttm_tt_unpopulate().
@@ -96,6 +102,7 @@ struct ttm_tt {
#define TTM_TT_FLAG_DECRYPTED BIT(4)
#define TTM_TT_FLAG_PRIV_POPULATED BIT(5)
+#define TTM_TT_FLAG_PRIV_SHRUNKEN BIT(6)
uint32_t page_flags;
/** @num_pages: Number of pages in the page array. */
uint32_t num_pages;
@@ -105,11 +112,13 @@ struct ttm_tt {
dma_addr_t *dma_address;
/** @swap_storage: Pointer to shmem struct file for swap storage. */
struct file *swap_storage;
+ struct ttm_backup *backup;
/**
* @caching: The current caching state of the pages, see enum
* ttm_caching.
*/
enum ttm_caching caching;
+ struct ttm_pool_tt_restore *restore;
};
/**
@@ -230,6 +239,9 @@ void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages);
struct ttm_kmap_iter *ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt,
struct ttm_tt *tt);
unsigned long ttm_tt_pages_limit(void);
+
+long ttm_tt_shrink(struct ttm_device *bdev, struct ttm_tt *tt, bool purge);
+
#if IS_ENABLED(CONFIG_AGP)
#include <linux/agp_backend.h>
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 7/8] drm/xe, drm/ttm: Provide a generic LRU walker helper
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (5 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 6/8] drm/ttm/pool: Provide a helper to shrink pages Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 8:47 ` [CI v2 8/8] drm/xe: Add a shrinker for xe bos Thomas Hellström
2024-03-28 9:27 ` ✗ CI.Patch_applied: failure for drm/ttm, drm/xe: Shrinker support (rev2) Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
Export the needed functions from TTM and provide a generic LRU
walker in xe, in the spirit of drm_gem_lru_scan() but building
on the restartable TTM LRU functionality.
The LRU walker optionally supports locking objects as part of
a drm_exec locking transaction, and can thus be used for both
exhaustive eviction and shrinking. And, in fact, direct
shrinking in the case where we fail to populate system memory
objects and want to retry by shrinking purgeable or evictable
local objects, which a shrinker is not capable of doing.
The LRU walker helper can easily be moved to TTM when / if used
by other drivers.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/ttm/ttm_resource.c | 3 +
drivers/gpu/drm/xe/Makefile | 1 +
drivers/gpu/drm/xe/xe_ttm_helpers.c | 106 ++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_ttm_helpers.h | 33 +++++++++
4 files changed, 143 insertions(+)
create mode 100644 drivers/gpu/drm/xe/xe_ttm_helpers.c
create mode 100644 drivers/gpu/drm/xe/xe_ttm_helpers.h
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 89f19fb6e2b3..4cad467983f8 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -106,6 +106,7 @@ void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
ttm_resource_cursor_fini_locked(cursor);
spin_unlock(lru_lock);
}
+EXPORT_SYMBOL(ttm_resource_cursor_fini);
/**
* ttm_lru_bulk_move_init - initialize a bulk move structure
@@ -635,6 +636,7 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
return NULL;
}
+EXPORT_SYMBOL(ttm_resource_manager_next);
/**
* ttm_resource_manager_first() - Start iterating over the resources
@@ -661,6 +663,7 @@ ttm_resource_manager_first(struct ttm_resource_manager *man,
return ttm_resource_manager_next(cursor);
}
+EXPORT_SYMBOL(ttm_resource_manager_first);
static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
struct iosys_map *dmap,
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index 3c3e67885559..de457db06dc2 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -132,6 +132,7 @@ xe-y += xe_bb.o \
xe_tile.o \
xe_tile_sysfs.o \
xe_trace.o \
+ xe_ttm_helpers.o \
xe_ttm_sys_mgr.o \
xe_ttm_stolen_mgr.o \
xe_ttm_vram_mgr.o \
diff --git a/drivers/gpu/drm/xe/xe_ttm_helpers.c b/drivers/gpu/drm/xe/xe_ttm_helpers.c
new file mode 100644
index 000000000000..056f836ce9e4
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_ttm_helpers.c
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <drm/drm_exec.h>
+
+#include "xe_ttm_helpers.h"
+
+#include <drm/ttm/ttm_bo.h>
+#include <drm/ttm/ttm_device.h>
+
+static bool xe_ttm_lru_walk_trylock(struct xe_ttm_lru_walk *walk,
+ struct ttm_buffer_object *bo,
+ bool *needs_unlock)
+{
+ struct ttm_operation_ctx *ctx = walk->ctx;
+
+ *needs_unlock = false;
+
+ if (!walk->exec && dma_resv_trylock(bo->base.resv)) {
+ *needs_unlock = true;
+ return true;
+ }
+
+ if (bo->base.resv == ctx->resv && ctx->allow_res_evict) {
+ dma_resv_assert_held(bo->base.resv);
+ return true;
+ }
+
+ return false;
+}
+
+static void xe_ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked)
+{
+ if (locked)
+ dma_resv_unlock(bo->base.resv);
+}
+
+long xe_ttm_lru_walk_for_evict(struct xe_ttm_lru_walk *walk, struct ttm_device *bdev,
+ struct ttm_resource_manager *man, unsigned int mem_type,
+ long target)
+{
+ struct drm_exec *exec = walk->exec;
+ struct ttm_resource_cursor cursor;
+ struct ttm_resource *res;
+ long sofar = 0;
+ long lret;
+ int ret;
+
+ spin_lock(&bdev->lru_lock);
+ ttm_resource_manager_for_each_res(man, &cursor, res) {
+ struct ttm_buffer_object *bo = res->bo;
+ bool bo_needs_unlock = false;
+ bool bo_locked = false;
+
+ if (!bo || bo->resource != res)
+ continue;
+
+ if (xe_ttm_lru_walk_trylock(walk, bo, &bo_needs_unlock))
+ bo_locked = true;
+ else if (!exec)
+ continue;
+
+ if (!ttm_bo_get_unless_zero(bo)) {
+ xe_ttm_lru_walk_unlock(bo, bo_needs_unlock);
+ continue;
+ }
+
+ spin_unlock(&bdev->lru_lock);
+
+ if (!bo_locked) {
+ ret = drm_exec_lock_obj(exec, &bo->base);
+ if (ret)
+ ttm_bo_put(bo);
+ }
+
+ lret = 0;
+
+ /*
+ * Note that in between the release of the lru lock and the
+ * drm_exec_lock_obj, the bo may have switched resource,
+ * and also memory type. In that case we just skip it.
+ */
+ if (bo->resource == res && res->mem_type == mem_type &&
+ walk->ops->allow_bo(walk, bo, mem_type))
+ lret = walk->ops->process_bo(walk, bo);
+
+ xe_ttm_lru_walk_unlock(bo, bo_needs_unlock);
+ ttm_bo_put(bo);
+ if (lret < 0) {
+ sofar = lret;
+ goto out;
+ }
+
+ sofar += lret;
+ if (sofar >= target)
+ goto out;
+
+ spin_lock(&bdev->lru_lock);
+ }
+ spin_unlock(&bdev->lru_lock);
+out:
+ ttm_resource_cursor_fini(&cursor);
+ return sofar;
+}
diff --git a/drivers/gpu/drm/xe/xe_ttm_helpers.h b/drivers/gpu/drm/xe/xe_ttm_helpers.h
new file mode 100644
index 000000000000..41d82eb7de30
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_ttm_helpers.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_TTM_HELPERS_H_
+#define _XE_TTM_HELPERS_H_
+
+#include <linux/types.h>
+
+struct drm_exec;
+struct ttm_device;
+struct ttm_buffer_object;
+struct ttm_operation_ctx;
+struct ttm_resource_manager;
+
+struct xe_ttm_lru_walk;
+struct xe_ttm_lru_walk_ops {
+ bool (*allow_bo)(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo,
+ unsigned int mem_type);
+ long (*process_bo)(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo);
+};
+
+struct xe_ttm_lru_walk {
+ const struct xe_ttm_lru_walk_ops *ops;
+ struct ttm_operation_ctx *ctx;
+ struct drm_exec *exec;
+};
+
+long xe_ttm_lru_walk_for_evict(struct xe_ttm_lru_walk *walk, struct ttm_device *bdev,
+ struct ttm_resource_manager *man, unsigned int mem_type,
+ long target);
+#endif
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [CI v2 8/8] drm/xe: Add a shrinker for xe bos
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (6 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 7/8] drm/xe, drm/ttm: Provide a generic LRU walker helper Thomas Hellström
@ 2024-03-28 8:47 ` Thomas Hellström
2024-03-28 9:27 ` ✗ CI.Patch_applied: failure for drm/ttm, drm/xe: Shrinker support (rev2) Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Thomas Hellström @ 2024-03-28 8:47 UTC (permalink / raw)
To: intel-xe
Rather than relying on the TTM watermark accounting add a shrinker
for xe_bos in TT or system memory.
Leverage the newly added TTM per-page shrinking and shmem backup
support.
Although xe doesn't fully support WONTNEED (purgeable) bos yet,
introduce and add shrinker support for purgeable ttm_tts.
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
---
drivers/gpu/drm/xe/Makefile | 1 +
drivers/gpu/drm/xe/xe_bo.c | 106 +++++++++++---
drivers/gpu/drm/xe/xe_bo.h | 3 +
drivers/gpu/drm/xe/xe_device.c | 8 ++
drivers/gpu/drm/xe/xe_device_types.h | 2 +
drivers/gpu/drm/xe/xe_shrinker.c | 204 +++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_shrinker.h | 18 +++
drivers/gpu/drm/xe/xe_ttm_helpers.c | 60 ++++++++
drivers/gpu/drm/xe/xe_ttm_helpers.h | 3 +
9 files changed, 388 insertions(+), 17 deletions(-)
create mode 100644 drivers/gpu/drm/xe/xe_shrinker.c
create mode 100644 drivers/gpu/drm/xe/xe_shrinker.h
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index de457db06dc2..0c849783126e 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -127,6 +127,7 @@ xe-y += xe_bb.o \
xe_ring_ops.o \
xe_sa.o \
xe_sched_job.o \
+ xe_shrinker.o \
xe_step.o \
xe_sync.o \
xe_tile.o \
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 9298546909b5..b5e043aaef01 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -10,6 +10,7 @@
#include <drm/drm_drv.h>
#include <drm/drm_gem_ttm_helper.h>
#include <drm/drm_managed.h>
+#include <drm/ttm/ttm_backup.h>
#include <drm/ttm/ttm_device.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/ttm/ttm_tt.h>
@@ -24,7 +25,9 @@
#include "xe_migrate.h"
#include "xe_preempt_fence.h"
#include "xe_res_cursor.h"
+#include "xe_shrinker.h"
#include "xe_trace.h"
+#include "xe_ttm_helpers.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_vm.h"
@@ -265,9 +268,10 @@ static void xe_evict_flags(struct ttm_buffer_object *tbo,
struct xe_ttm_tt {
struct ttm_tt ttm;
- struct device *dev;
+ struct xe_device *xe;
struct sg_table sgt;
struct sg_table *sg;
+ bool purgeable;
};
static int xe_tt_map_sg(struct ttm_tt *tt)
@@ -276,7 +280,8 @@ static int xe_tt_map_sg(struct ttm_tt *tt)
unsigned long num_pages = tt->num_pages;
int ret;
- XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL);
+ XE_WARN_ON((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
+ !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE));
if (xe_tt->sg)
return 0;
@@ -284,13 +289,13 @@ static int xe_tt_map_sg(struct ttm_tt *tt)
ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,
num_pages, 0,
(u64)num_pages << PAGE_SHIFT,
- xe_sg_segment_size(xe_tt->dev),
+ xe_sg_segment_size(xe_tt->xe->drm.dev),
GFP_KERNEL);
if (ret)
return ret;
xe_tt->sg = &xe_tt->sgt;
- ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL,
+ ret = dma_map_sgtable(xe_tt->xe->drm.dev, xe_tt->sg, DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (ret) {
sg_free_table(xe_tt->sg);
@@ -309,21 +314,37 @@ struct sg_table *xe_bo_sg(struct xe_bo *bo)
return xe_tt->sg;
}
+static void xe_ttm_tt_account(struct ttm_tt *tt, bool add)
+{
+ struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
+ long num_pages = tt->num_pages;
+
+ if (!add)
+ num_pages = -num_pages;
+
+ if (xe_tt->purgeable)
+ xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, 0, num_pages);
+ else
+ xe_shrinker_mod_pages(xe_tt->xe->mem.shrinker, num_pages, 0);
+}
+
static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
u32 page_flags)
{
struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
struct xe_device *xe = xe_bo_device(bo);
- struct xe_ttm_tt *tt;
+ struct xe_ttm_tt *xe_tt;
+ struct ttm_tt *tt;
unsigned long extra_pages;
enum ttm_caching caching;
int err;
- tt = kzalloc(sizeof(*tt), GFP_KERNEL);
- if (!tt)
+ xe_tt = kzalloc(sizeof(*xe_tt), GFP_KERNEL);
+ if (!xe_tt)
return NULL;
- tt->dev = xe->drm.dev;
+ tt = &xe_tt->ttm;
+ xe_tt->xe = xe;
extra_pages = 0;
if (xe_bo_needs_ccs_pages(bo))
@@ -351,13 +372,23 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
(xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_PAGETABLE))
caching = ttm_write_combined;
- err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);
+ if (ttm_bo->type != ttm_bo_type_sg)
+ page_flags |= TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE;
+
+ err = ttm_tt_init(tt, &bo->ttm, page_flags, caching, extra_pages);
if (err) {
- kfree(tt);
+ kfree(xe_tt);
return NULL;
}
- return &tt->ttm;
+ tt->backup = ttm_backup_shmem_create(tt->num_pages << PAGE_SHIFT);
+ if (IS_ERR(tt->backup)) {
+ ttm_tt_fini(tt);
+ kfree(xe_tt);
+ return NULL;
+ }
+
+ return tt;
}
static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
@@ -369,7 +400,8 @@ static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
* dma-bufs are not populated with pages, and the dma-
* addresses are set up when moved to XE_PL_TT.
*/
- if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
+ if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
+ !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))
return 0;
err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);
@@ -378,27 +410,58 @@ static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
/* A follow up may move this xe_bo_move when BO is moved to XE_PL_TT */
err = xe_tt_map_sg(tt);
- if (err)
+ if (err) {
ttm_pool_free(&ttm_dev->pool, tt);
+ return err;
+ }
- return err;
+ xe_ttm_tt_account(tt, true);
+
+ return 0;
}
static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)
{
struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
+ struct xe_device *xe = xe_tt->xe;
- if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
+ if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
+ !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))
return;
if (xe_tt->sg) {
- dma_unmap_sgtable(xe_tt->dev, xe_tt->sg,
+ dma_unmap_sgtable(xe->drm.dev, xe_tt->sg,
DMA_BIDIRECTIONAL, 0);
sg_free_table(xe_tt->sg);
xe_tt->sg = NULL;
}
- return ttm_pool_free(&ttm_dev->pool, tt);
+ ttm_pool_free(&ttm_dev->pool, tt);
+ xe_ttm_tt_account(tt, false);
+}
+
+long xe_bo_shrinker_process(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo)
+{
+ struct ttm_tt *tt = bo->ttm;
+ struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
+ struct xe_device *xe = xe_tt->xe;
+ long lret;
+
+ lret = xe_ttm_bo_try_shrink(walk, bo, xe_tt->purgeable);
+ if (lret > 0) {
+ xe_assert(xe, !ttm_tt_is_populated(tt));
+
+ xe_ttm_tt_account(tt, false);
+
+ if (xe_tt->sg) {
+ dma_unmap_sgtable(xe->drm.dev, xe_tt->sg,
+ DMA_BIDIRECTIONAL, 0);
+ sg_free_table(xe_tt->sg);
+ xe_tt->sg = NULL;
+ }
+ }
+
+ return lret;
}
static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
@@ -1644,6 +1707,8 @@ int xe_bo_pin_external(struct xe_bo *bo)
}
ttm_bo_pin(&bo->ttm);
+ if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
+ xe_ttm_tt_account(bo->ttm.ttm, false);
/*
* FIXME: If we always use the reserve / unreserve functions for locking
@@ -1702,6 +1767,8 @@ int xe_bo_pin(struct xe_bo *bo)
}
ttm_bo_pin(&bo->ttm);
+ if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
+ xe_ttm_tt_account(bo->ttm.ttm, false);
/*
* FIXME: If we always use the reserve / unreserve functions for locking
@@ -1737,6 +1804,9 @@ void xe_bo_unpin_external(struct xe_bo *bo)
}
ttm_bo_unpin(&bo->ttm);
+ if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
+ xe_ttm_tt_account(bo->ttm.ttm, true);
+
/*
* FIXME: If we always use the reserve / unreserve functions for locking
@@ -1766,6 +1836,8 @@ void xe_bo_unpin(struct xe_bo *bo)
}
ttm_bo_unpin(&bo->ttm);
+ if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
+ xe_ttm_tt_account(bo->ttm.ttm, true);
}
/**
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 52e441f77e96..3866ba9d4523 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -93,6 +93,7 @@
#define XE_BO_PROPS_INVALID (-1)
struct sg_table;
+struct xe_ttm_lru_walk;
struct xe_bo *xe_bo_alloc(void);
void xe_bo_free(struct xe_bo *bo);
@@ -344,6 +345,8 @@ static inline unsigned int xe_sg_segment_size(struct device *dev)
#define i915_gem_object_flush_if_display(obj) ((void)(obj))
+long xe_bo_shrinker_process(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo);
+
#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
/**
* xe_bo_is_mem_type - Whether the bo currently resides in the given
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index b0bfe75eb59f..f05a9d8ed23d 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -38,6 +38,7 @@
#include "xe_pcode.h"
#include "xe_pm.h"
#include "xe_query.h"
+#include "xe_shrinker.h"
#include "xe_sriov.h"
#include "xe_tile.h"
#include "xe_ttm_stolen_mgr.h"
@@ -232,6 +233,9 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
if (xe->unordered_wq)
destroy_workqueue(xe->unordered_wq);
+ if (!IS_ERR_OR_NULL(xe->mem.shrinker))
+ xe_shrinker_destroy(xe->mem.shrinker);
+
ttm_device_fini(&xe->ttm);
}
@@ -261,6 +265,10 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
if (err)
goto err;
+ xe->mem.shrinker = xe_shrinker_create(xe);
+ if (IS_ERR(xe->mem.shrinker))
+ return ERR_CAST(xe->mem.shrinker);
+
xe->info.devid = pdev->device;
xe->info.revid = pdev->revision;
xe->info.force_execlist = xe_modparam.force_execlist;
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 1df3dcc17d75..daafd7a910d7 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -315,6 +315,8 @@ struct xe_device {
struct xe_mem_region vram;
/** @mem.sys_mgr: system TTM manager */
struct ttm_resource_manager sys_mgr;
+ /** @mem.sys_mgr: system memory shrinker. */
+ struct xe_shrinker *shrinker;
} mem;
/** @sriov: device level virtualization data */
diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c
new file mode 100644
index 000000000000..7340229d11a0
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_shrinker.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <linux/shrinker.h>
+#include <linux/swap.h>
+
+#include <drm/ttm/ttm_bo.h>
+#include <drm/ttm/ttm_tt.h>
+
+#include "xe_bo.h"
+#include "xe_shrinker.h"
+#include "xe_ttm_helpers.h"
+
+struct xe_shrinker {
+ struct xe_device *xe;
+ rwlock_t lock;
+ long shrinkable_pages;
+ long purgeable_pages;
+ struct shrinker *shrink;
+};
+
+struct xe_shrink_lru_walk {
+ struct xe_ttm_lru_walk walk;
+ struct xe_shrinker *shrinker;
+ bool purge;
+};
+
+static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink)
+{
+ return shrink->private_data;
+}
+
+static struct xe_shrink_lru_walk *
+to_xe_shrink_lru_walk(struct xe_ttm_lru_walk *walk)
+{
+ return container_of(walk, struct xe_shrink_lru_walk, walk);
+}
+
+void
+xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable)
+{
+ write_lock(&shrinker->lock);
+ shrinker->shrinkable_pages += shrinkable;
+ shrinker->purgeable_pages += purgeable;
+ write_unlock(&shrinker->lock);
+}
+
+static bool xe_shrinker_allow_bo(struct xe_ttm_lru_walk *walk,
+ struct ttm_buffer_object *bo,
+ unsigned int mem_type)
+{
+ struct xe_shrink_lru_walk *shrink_walk = to_xe_shrink_lru_walk(walk);
+ struct ttm_place place = {.mem_type = mem_type};
+ struct ttm_tt *tt = bo->ttm;
+ bool ret;
+
+ if (!tt || !ttm_tt_is_populated(tt) ||
+ !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))
+ return false;
+
+ if (shrink_walk->purge)
+ return false;
+
+ ret = ttm_bo_eviction_valuable(bo, &place);
+
+ return ret;
+}
+
+static long xe_shrinker_walk(struct xe_shrink_lru_walk *shrink_walk, long target)
+{
+ struct xe_device *xe = shrink_walk->shrinker->xe;
+ struct ttm_resource_manager *man;
+ unsigned int mem_type;
+ long sofar = 0;
+ long lret;
+
+ for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) {
+ man = ttm_manager_type(&xe->ttm, mem_type);
+ if (!man || !man->use_tt)
+ continue;
+
+ lret = xe_ttm_lru_walk_for_evict(&shrink_walk->walk,
+ &xe->ttm, man,
+ mem_type, target);
+ if (lret < 0)
+ return lret;
+
+ sofar += lret;
+ if (sofar >= target)
+ break;
+ }
+
+ return sofar;
+}
+
+static unsigned long
+xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+ struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
+ unsigned long num_pages;
+
+ num_pages = get_nr_swap_pages();
+ read_lock(&shrinker->lock);
+ num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages);
+ num_pages += shrinker->purgeable_pages;
+ read_unlock(&shrinker->lock);
+
+ return num_pages ? num_pages : SHRINK_EMPTY;
+}
+
+static const struct xe_ttm_lru_walk_ops xe_shrink_ops = {
+ .process_bo = xe_bo_shrinker_process,
+ .allow_bo = xe_shrinker_allow_bo,
+};
+
+static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+ struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
+ bool is_kswapd = current_is_kswapd();
+ struct ttm_operation_ctx ctx = {
+ .interruptible = false,
+ .no_wait_gpu = !is_kswapd,
+ };
+ unsigned long nr_to_scan, freed = 0;
+ struct xe_shrink_lru_walk shrink_walk = {
+ .walk = {
+ .ops = &xe_shrink_ops,
+ .ctx = &ctx,
+ },
+ .purge = true,
+ .shrinker = shrinker,
+ };
+ bool purgeable;
+ long ret;
+
+ sc->nr_scanned = 0;
+ nr_to_scan = sc->nr_to_scan;
+
+ read_lock(&shrinker->lock);
+ purgeable = !!shrinker->purgeable_pages;
+ read_unlock(&shrinker->lock);
+
+ while (purgeable && freed < nr_to_scan) {
+ ret = xe_shrinker_walk(&shrink_walk, nr_to_scan);
+ if (ret <= 0)
+ break;
+
+ freed += ret;
+ }
+
+ sc->nr_scanned = freed;
+ if (freed < nr_to_scan)
+ nr_to_scan -= freed;
+ else
+ nr_to_scan = 0;
+ if (!nr_to_scan)
+ return freed ? freed : SHRINK_STOP;
+
+ shrink_walk.purge = false;
+ while (freed < nr_to_scan) {
+ ret = xe_shrinker_walk(&shrink_walk, nr_to_scan);
+ if (ret <= 0)
+ break;
+
+ freed += ret;
+ }
+
+ sc->nr_scanned = freed;
+
+ return freed ? freed : SHRINK_STOP;
+}
+
+struct xe_shrinker *xe_shrinker_create(struct xe_device *xe)
+{
+ struct xe_shrinker *shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL);
+
+ if (!shrinker)
+ return ERR_PTR(-ENOMEM);
+
+ shrinker->shrink = shrinker_alloc(0, "xe system shrinker");
+ if (!shrinker->shrink) {
+ kfree(shrinker);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ shrinker->xe = xe;
+ rwlock_init(&shrinker->lock);
+ shrinker->shrink->count_objects = xe_shrinker_count;
+ shrinker->shrink->scan_objects = xe_shrinker_scan;
+ shrinker->shrink->private_data = shrinker;
+ shrinker_register(shrinker->shrink);
+
+ return shrinker;
+}
+
+void xe_shrinker_destroy(struct xe_shrinker *shrinker)
+{
+ pr_info("Shrinkable at exit %ld\n", shrinker->shrinkable_pages);
+ pr_info("Purgeable at exit %ld\n", shrinker->purgeable_pages);
+ shrinker_free(shrinker->shrink);
+ kfree(shrinker);
+}
diff --git a/drivers/gpu/drm/xe/xe_shrinker.h b/drivers/gpu/drm/xe/xe_shrinker.h
new file mode 100644
index 000000000000..28a038f4fcbf
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_shrinker.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_SHRINKER_H_
+#define _XE_SHRINKER_H_
+
+struct xe_shrinker;
+struct xe_device;
+
+void xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable);
+
+struct xe_shrinker *xe_shrinker_create(struct xe_device *xe);
+
+void xe_shrinker_destroy(struct xe_shrinker *shrinker);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_ttm_helpers.c b/drivers/gpu/drm/xe/xe_ttm_helpers.c
index 056f836ce9e4..bbef00369a63 100644
--- a/drivers/gpu/drm/xe/xe_ttm_helpers.c
+++ b/drivers/gpu/drm/xe/xe_ttm_helpers.c
@@ -9,6 +9,66 @@
#include <drm/ttm/ttm_bo.h>
#include <drm/ttm/ttm_device.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_tt.h>
+
+long xe_ttm_bo_try_shrink(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo,
+ bool purge)
+{
+ static const struct ttm_place sys_placement_flags = {
+ .fpfn = 0,
+ .lpfn = 0,
+ .mem_type = TTM_PL_SYSTEM,
+ .flags = 0,
+ };
+ static struct ttm_placement sys_placement = {
+ .num_placement = 1,
+ .placement = &sys_placement_flags,
+ };
+ struct ttm_operation_ctx *ctx = walk->ctx;
+ struct ttm_tt *tt = bo->ttm;
+ long lret;
+
+ dma_resv_assert_held(bo->base.resv);
+
+ if (tt || !ttm_tt_is_populated(tt))
+ return 0;
+
+ if (bo->resource->mem_type != TTM_PL_SYSTEM) {
+ int ret = ttm_bo_validate(bo, &sys_placement, ctx);
+
+ if (ret) {
+ if (ret == -EINTR || ret == -EDEADLK ||
+ ret == -ERESTARTSYS)
+ return ret;
+ return 0;
+ }
+ }
+
+ if (ctx->no_wait_gpu &&
+ !dma_resv_test_signaled(bo->base.resv,
+ DMA_RESV_USAGE_BOOKKEEP))
+ return 0;
+
+ lret = dma_resv_wait_timeout(bo->base.resv,
+ DMA_RESV_USAGE_BOOKKEEP,
+ walk->ctx->interruptible,
+ MAX_SCHEDULE_TIMEOUT);
+ if (lret < 0) {
+ if (lret == -ERESTARTSYS)
+ return lret;
+ return 0;
+ }
+
+ if (bo->deleted)
+ lret = ttm_tt_shrink(bo->bdev, tt, true);
+ else
+ lret = ttm_tt_shrink(bo->bdev, tt, purge);
+ if (lret < 0 && lret != -EINTR)
+ return 0;
+
+ return lret;
+}
static bool xe_ttm_lru_walk_trylock(struct xe_ttm_lru_walk *walk,
struct ttm_buffer_object *bo,
diff --git a/drivers/gpu/drm/xe/xe_ttm_helpers.h b/drivers/gpu/drm/xe/xe_ttm_helpers.h
index 41d82eb7de30..eb5ed7613059 100644
--- a/drivers/gpu/drm/xe/xe_ttm_helpers.h
+++ b/drivers/gpu/drm/xe/xe_ttm_helpers.h
@@ -27,6 +27,9 @@ struct xe_ttm_lru_walk {
struct drm_exec *exec;
};
+long xe_ttm_bo_try_shrink(struct xe_ttm_lru_walk *walk, struct ttm_buffer_object *bo,
+ bool purge);
+
long xe_ttm_lru_walk_for_evict(struct xe_ttm_lru_walk *walk, struct ttm_device *bdev,
struct ttm_resource_manager *man, unsigned int mem_type,
long target);
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* ✗ CI.Patch_applied: failure for drm/ttm, drm/xe: Shrinker support (rev2)
2024-03-28 8:47 [CI v2 0/8] drm/ttm, drm/xe: Shrinker support Thomas Hellström
` (7 preceding siblings ...)
2024-03-28 8:47 ` [CI v2 8/8] drm/xe: Add a shrinker for xe bos Thomas Hellström
@ 2024-03-28 9:27 ` Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Patchwork @ 2024-03-28 9:27 UTC (permalink / raw)
To: Thomas Hellström; +Cc: intel-xe
== Series Details ==
Series: drm/ttm, drm/xe: Shrinker support (rev2)
URL : https://patchwork.freedesktop.org/series/131695/
State : failure
== Summary ==
=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 8d398ffc40ac drm-tip: 2024y-03m-28d-08h-07m-15s UTC integration manifest
=== git am output follows ===
error: patch failed: drivers/gpu/drm/xe/xe_vm.c:1593
error: drivers/gpu/drm/xe/xe_vm.c: patch does not apply
hint: Use 'git am --show-current-patch=diff' to see the failed patch
Applying: drm/ttm: Allow TTM LRU list nodes of different types
Applying: drm/ttm: Use LRU hitches
Applying: drm/ttm, drm/amdgpu, drm/xe: Consider hitch moves within bulk sublist moves
Patch failed at 0003 drm/ttm, drm/amdgpu, drm/xe: Consider hitch moves within bulk sublist moves
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
^ permalink raw reply [flat|nested] 10+ messages in thread