[PATCH] drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages

[PATCH] drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages

[Tejas Upadhyay]

The existing bad-page flow soft-offlines a VRAM page on the first HW
report and promotes it to a permanent (hard) retirement on the second
report at the same address. Once soft-offlined a page stays offline
forever, which over-punishes transient DBE errors: a single hit caused
by a one-off environmental disturbance keeps the page reserved until
the next reboot / probe.

This change adds a per VRAM-manager decay timer so that a SOFT-offlined
page is automatically reinstated after the configured decay window
(default 24 hours) of no further reports at that address. HARD-offlined
pages continue to be permanent.

Also adds the xe_page_offline_mode enum (SOFT/HARD) and per-entry mode
tracking with SOFT->HARD promotion on second hit at the same address.

Configuration via configfs bad_page_decay_secs (default 86400, 0 disables).

IGT coverage:
https://patchwork.freedesktop.org/patch/765064/

Signed-off-by: Tejas Upadhyay <tejas.upadhyay@intel.com>
---
 drivers/gpu/drm/xe/xe_configfs.c           | 54 ++++++++++++++++++
 drivers/gpu/drm/xe/xe_configfs.h           |  1 +
 drivers/gpu/drm/xe/xe_ttm_vram_mgr.c       | 64 +++++++++++++++++++++-
 drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h | 16 ++++++
 4 files changed, 134 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c
index ad4bfd3c0703..c25944650f59 100644
--- a/drivers/gpu/drm/xe/xe_configfs.c
+++ b/drivers/gpu/drm/xe/xe_configfs.c
@@ -305,6 +305,7 @@ struct xe_config_group_device {
 		bool enable_psmi;
 		bool enable_multi_queue;
 		bool disable_vram_page_offline;
+		u32 bad_page_decay_secs;
 		struct {
 			unsigned int max_vfs;
 			bool admin_only_pf;
@@ -327,6 +328,7 @@ static const struct xe_config_device device_defaults = {
 	.enable_psmi = false,
 	.enable_multi_queue = true,
 	.disable_vram_page_offline = false,
+	.bad_page_decay_secs = 24 * 60 * 60,
 	.sriov = {
 		.max_vfs = XE_DEFAULT_MAX_VFS,
 		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,
@@ -702,6 +704,36 @@ static ssize_t disable_vram_page_offline_store(struct config_item *item,
 	return len;
 }
 
+static ssize_t bad_page_decay_secs_show(struct config_item *item, char *page)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item);
+
+	return sprintf(page, "%u\n", dev->config.bad_page_decay_secs);
+}
+
+static ssize_t bad_page_decay_secs_store(struct config_item *item,
+					 const char *page, size_t len)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item);
+	u32 val;
+	int ret;
+
+	ret = kstrtou32(page, 0, &val);
+	if (ret)
+		return ret;
+
+	if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET) / 1000)
+		return -EINVAL;
+
+	guard(mutex)(&dev->lock);
+	if (is_bound(dev))
+		return -EBUSY;
+
+	dev->config.bad_page_decay_secs = val;
+
+	return len;
+}
+
 static bool wa_bb_read_advance(bool dereference, char **p,
 			       const char *append, size_t len,
 			       size_t *max_size)
@@ -943,6 +975,7 @@ CONFIGFS_ATTR(, enable_multi_queue);
 CONFIGFS_ATTR(, migrate_ulls_period_ms);
 CONFIGFS_ATTR(, enable_psmi);
 CONFIGFS_ATTR(, disable_vram_page_offline);
+CONFIGFS_ATTR(, bad_page_decay_secs);
 CONFIGFS_ATTR(, engines_allowed);
 CONFIGFS_ATTR(, gt_types_allowed);
 CONFIGFS_ATTR(, survivability_mode);
@@ -954,6 +987,7 @@ static struct configfs_attribute *xe_config_device_attrs[] = {
 	&attr_migrate_ulls_period_ms,
 	&attr_enable_psmi,
 	&attr_disable_vram_page_offline,
+	&attr_bad_page_decay_secs,
 	&attr_engines_allowed,
 	&attr_gt_types_allowed,
 	&attr_survivability_mode,
@@ -1427,6 +1461,26 @@ bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev)
 	return ret;
 }
 
+/**
+ * xe_configfs_get_bad_page_decay_secs - get configfs bad_page_decay_secs setting
+ * @pdev: pci device
+ *
+ * Return: bad_page_decay_secs setting in configfs
+ */
+u32 xe_configfs_get_bad_page_decay_secs(struct pci_dev *pdev)
+{
+	struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
+	u32 ret;
+
+	if (!dev)
+		return device_defaults.bad_page_decay_secs;
+
+	ret = dev->config.bad_page_decay_secs;
+	config_group_put(&dev->group);
+
+	return ret;
+}
+
 /**
  * xe_configfs_get_ctx_restore_mid_bb - get configfs ctx_restore_mid_bb setting
  * @pdev: pci device
diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h
index 327b648944ff..406b72224702 100644
--- a/drivers/gpu/drm/xe/xe_configfs.h
+++ b/drivers/gpu/drm/xe/xe_configfs.h
@@ -26,6 +26,7 @@ bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev);
 bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev);
 u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev);
 bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev);
+u32 xe_configfs_get_bad_page_decay_secs(struct pci_dev *pdev);
 u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 				       enum xe_engine_class class,
 				       const u32 **cs);
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 36d47de8b11b..ed0ca5f2b144 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -355,6 +355,51 @@ static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = {
 	.debug	= xe_ttm_vram_mgr_debug
 };
 
+static unsigned long xe_ttm_vram_decay_gc_period(struct xe_ttm_vram_mgr *mgr)
+{
+	unsigned long period = mgr->decay_interval_jiffies / 8;
+
+	if (period < msecs_to_jiffies(60 * 1000))
+		period = msecs_to_jiffies(60 * 1000);
+	return period;
+}
+
+static void xe_ttm_vram_decay_gc_worker(struct work_struct *work)
+{
+	struct xe_ttm_vram_mgr *mgr = container_of(to_delayed_work(work),
+						   struct xe_ttm_vram_mgr,
+						   decay_gc_work);
+	struct xe_ttm_vram_offline_resource *pos, *n;
+	struct xe_device *xe = ttm_to_xe_device(mgr->manager.bdev);
+	unsigned long now = jiffies;
+	unsigned long decay;
+
+	mutex_lock(&mgr->lock);
+	decay = mgr->decay_interval_jiffies;
+	if (!decay)
+		goto reschedule;
+
+	list_for_each_entry_safe(pos, n, &mgr->offlined_pages, offlined_link) {
+		if (pos->mode != XE_PAGE_OFFLINE_SOFT)
+			continue;
+		if (!time_after(now, pos->last_seen + decay))
+			continue;
+
+		list_del(&pos->offlined_link);
+		xe_ttm_vram_buddy_free(mgr, &pos->blocks, pos->used_visible_size);
+		--mgr->n_offlined_pages;
+		drm_info(&xe->drm,
+			 "decay: reinstated soft-offlined page 0x%llx\n",
+			 pos->addr);
+		kfree(pos);
+	}
+
+reschedule:
+	schedule_delayed_work(&mgr->decay_gc_work,
+			      xe_ttm_vram_decay_gc_period(mgr));
+	mutex_unlock(&mgr->lock);
+}
+
 static const struct dmem_cgroup_ops xe_ttm_vram_mgr_dmem_ops;
 
 static int xe_ttm_vram_mgr_dmem_reclaim(struct dmem_cgroup_pool_state *pool,
@@ -415,6 +460,8 @@ static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
 	struct xe_ttm_vram_mgr *mgr = arg;
 	struct ttm_resource_manager *man = &mgr->manager;
 
+	cancel_delayed_work_sync(&mgr->decay_gc_work);
+
 	mutex_lock(&mgr->lock);
 	xe_ttm_vram_free_bad_pages(mgr);
 	mutex_unlock(&mgr->lock);
@@ -438,6 +485,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 	struct ttm_resource_manager *man = &mgr->manager;
 	struct dmem_cgroup_region *cg;
 	const char *name;
+	u32 decay_secs;
 	int err;
 
 	man->func = &xe_ttm_vram_mgr_func;
@@ -447,6 +495,9 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 		return err;
 	INIT_LIST_HEAD(&mgr->offlined_pages);
 	INIT_LIST_HEAD(&mgr->queued_pages);
+	INIT_DELAYED_WORK(&mgr->decay_gc_work, xe_ttm_vram_decay_gc_worker);
+	decay_secs = xe_configfs_get_bad_page_decay_secs(to_pci_dev(xe->drm.dev));
+	mgr->decay_interval_jiffies = secs_to_jiffies(decay_secs);
 	mgr->default_page_size = default_page_size;
 	mgr->visible_size = io_size;
 	mgr->visible_avail = io_size;
@@ -725,8 +776,12 @@ static bool xe_ttm_vram_page_already_processed(struct xe_ttm_vram_mgr *mgr,
 	lockdep_assert_held(&mgr->lock);
 
 	list_for_each_entry(pos, &mgr->offlined_pages, offlined_link) {
-		if (pos->addr == addr)
+		if (pos->addr == addr) {
+			if (pos->mode == XE_PAGE_OFFLINE_SOFT)
+				pos->mode = XE_PAGE_OFFLINE_HARD;
+			pos->last_seen = jiffies;
 			return true;
+		}
 	}
 
 	list_for_each_entry(pos, &mgr->queued_pages, queued_link) {
@@ -804,6 +859,8 @@ static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr,
 		INIT_LIST_HEAD(&nentry->blocks);
 		nentry->status = XE_PAGE_RESERVE_PENDING;
 		nentry->addr = addr;
+		nentry->mode = XE_PAGE_OFFLINE_SOFT;
+		nentry->last_seen = jiffies;
 
 		if (block) {
 			pbo = xe_ttm_vram_block_owner_get(xe, block);
@@ -843,6 +900,11 @@ static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr,
 			}
 			++vram_mgr->n_offlined_pages;
 			list_add(&nentry->offlined_link, &vram_mgr->offlined_pages);
+			if (vram_mgr->decay_interval_jiffies &&
+			    !delayed_work_pending(&vram_mgr->decay_gc_work)) {
+				schedule_delayed_work(&vram_mgr->decay_gc_work,
+						      xe_ttm_vram_decay_gc_period(vram_mgr));
+			}
 			return ret;
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
index 4a2cd27a5b61..0a6d8f8c2aa8 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
@@ -7,8 +7,16 @@
 #define _XE_TTM_VRAM_MGR_TYPES_H_
 
 #include <linux/gpu_buddy.h>
+#include <linux/workqueue.h>
 #include <drm/ttm/ttm_device.h>
 
+#define XE_BAD_PAGE_DECAY_SECS_DEFAULT	(24 * 60 * 60)
+
+enum xe_page_offline_mode {
+	XE_PAGE_OFFLINE_SOFT = 0,
+	XE_PAGE_OFFLINE_HARD,
+};
+
 /**
  * struct xe_ttm_vram_mgr - Xe TTM VRAM manager
  *
@@ -37,6 +45,10 @@ struct xe_ttm_vram_mgr {
 	struct mutex lock;
 	/** @mem_type: The TTM memory type */
 	u32 mem_type;
+	/** @decay_gc_work: reinstates expired soft-offlined pages */
+	struct delayed_work decay_gc_work;
+	/** @decay_interval_jiffies: soft-offline decay interval */
+	unsigned long decay_interval_jiffies;
 };
 
 /**
@@ -79,6 +91,10 @@ struct xe_ttm_vram_offline_resource {
 	u64 addr;
 	/** @status: buddy reservation status */
 	enum xe_page_reserve_status status;
+	/** @mode: soft or hard offline mode */
+	enum xe_page_offline_mode mode;
+	/** @last_seen: most recent hardware report time */
+	unsigned long last_seen;
 };
 
 #endif
-- 
2.52.0

Re: [PATCH] drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 6 potential issue(s) to consider:
- [High] drm/xe: permanent bad pages from get_offline_list() marked SOFT and reinstated
- [High] drm/xe: queued pages not promoted to HARD on repeat fault
- [High] drm/xe: decay_gc_work not armed in xe_ttm_vram_retry_queued_pages()
- [Low] drm/xe: missing stub for xe_configfs_get_bad_page_decay_secs()
- [Low] drm/xe: xe_ttm_vram_decay_gc_worker() reschedules itself unconditionally
- [Low] drm/xe: bad_page_decay_secs omitted from dump_custom_dev_config()
--

commit 46cc37a480016d26168ac9267631e7a2c7b48ce8
Author: Tejas Upadhyay <tejas.upadhyay@intel.com>

drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages

This commit introduces a per-VRAM-manager delayed work timer to reinstate
soft-offlined VRAM pages after a configurable decay interval (defaulting to
24 hours via configfs bad_page_decay_secs) if no further faults occur at the
same address. It also adds soft/hard offline mode tracking to promote a page
from soft to permanent hard retirement upon a second fault.

> diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c
> index 2ddd707a6dbd3..110459b8980f4 100644
> --- a/drivers/gpu/drm/xe/xe_configfs.c
> +++ b/drivers/gpu/drm/xe/xe_configfs.c
> @@ -305,6 +305,7 @@ struct xe_config_group_device {
>  		bool enable_psmi;
>  		bool enable_multi_queue;
>  		bool disable_vram_page_offline;
> +		u32 bad_page_decay_secs;
>  		struct {
>  			unsigned int max_vfs;
>  			bool admin_only_pf;
> @@ -327,6 +328,7 @@ static const struct xe_config_device device_defaults = {
>  	.enable_psmi = false,
>  	.enable_multi_queue = true,
>  	.disable_vram_page_offline = false,
> +	.bad_page_decay_secs = 24 * 60 * 60,
>  	.sriov = {
>  		.max_vfs = XE_DEFAULT_MAX_VFS,
>  		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,

[Severity: Low]
When bad_page_decay_secs is modified via configfs, xe_configfs_check_device()
detects the change in struct xe_config_device via memcmp() and invokes
dump_custom_dev_config(), which currently only prints:

drivers/gpu/drm/xe/xe_configfs.c:dump_custom_dev_config() {
	...
	PRI_CUSTOM_ATTR("%d", enable_psmi);
	PRI_CUSTOM_ATTR("%d", disable_vram_page_offline);
	PRI_CUSTOM_ATTR("%d", survivability_mode);
	...
}

Should dump_custom_dev_config() also include PRI_CUSTOM_ATTR("%u",
bad_page_decay_secs) so that non-default decay settings are logged alongside
the other custom attributes?

[ ... ]

> diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h
> index 327b648944ffa..406b722247020 100644
> --- a/drivers/gpu/drm/xe/xe_configfs.h
> +++ b/drivers/gpu/drm/xe/xe_configfs.h
> @@ -26,6 +26,7 @@ bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev);
>  bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev);
>  u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev);
>  bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev);
> +u32 xe_configfs_get_bad_page_decay_secs(struct pci_dev *pdev);
>  u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
>  				       enum xe_engine_class class,
>  				       const u32 **cs);

[Severity: Low]
Will building with CONFIG_DRM_XE enabled and !IS_REACHABLE(CONFIG_CONFIGFS_FS)
fail in __xe_ttm_vram_mgr_init()?

In drivers/gpu/drm/xe/xe_configfs.h, xe_configfs_get_bad_page_decay_secs() is
declared inside the #if IS_REACHABLE(CONFIG_CONFIGFS_FS) block, but no static
inline fallback stub was added to the corresponding #else block:

drivers/gpu/drm/xe/xe_configfs.h:
#else
...
static inline u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev) { return 5; }
static inline bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) { return false; }
static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,

Meanwhile, __xe_ttm_vram_mgr_init() in drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
calls xe_configfs_get_bad_page_decay_secs() unconditionally, and
XE_BAD_PAGE_DECAY_SECS_DEFAULT in drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
is left unused. Should a static inline stub returning
XE_BAD_PAGE_DECAY_SECS_DEFAULT (and using that macro in device_defaults) be
added when configfs is not reachable?

> diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
> index 36d47de8b11ba..ed0ca5f2b1449 100644
> --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
> +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
> @@ -355,6 +355,51 @@ static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = {
>  	.debug	= xe_ttm_vram_mgr_debug
>  };
>  
> +static unsigned long xe_ttm_vram_decay_gc_period(struct xe_ttm_vram_mgr *mgr)
> +{
> +	unsigned long period = mgr->decay_interval_jiffies / 8;
> +
> +	if (period < msecs_to_jiffies(60 * 1000))
> +		period = msecs_to_jiffies(60 * 1000);
> +	return period;
> +}
> +
> +static void xe_ttm_vram_decay_gc_worker(struct work_struct *work)
> +{
> +	struct xe_ttm_vram_mgr *mgr = container_of(to_delayed_work(work),
> +						   struct xe_ttm_vram_mgr,
> +						   decay_gc_work);
> +	struct xe_ttm_vram_offline_resource *pos, *n;
> +	struct xe_device *xe = ttm_to_xe_device(mgr->manager.bdev);
> +	unsigned long now = jiffies;
> +	unsigned long decay;
> +
> +	mutex_lock(&mgr->lock);
> +	decay = mgr->decay_interval_jiffies;
> +	if (!decay)
> +		goto reschedule;
> +
> +	list_for_each_entry_safe(pos, n, &mgr->offlined_pages, offlined_link) {
> +		if (pos->mode != XE_PAGE_OFFLINE_SOFT)
> +			continue;
> +		if (!time_after(now, pos->last_seen + decay))
> +			continue;
> +
> +		list_del(&pos->offlined_link);
> +		xe_ttm_vram_buddy_free(mgr, &pos->blocks, pos->used_visible_size);
> +		--mgr->n_offlined_pages;
> +		drm_info(&xe->drm,
> +			 "decay: reinstated soft-offlined page 0x%llx\n",
> +			 pos->addr);
> +		kfree(pos);
> +	}
> +
> +reschedule:
> +	schedule_delayed_work(&mgr->decay_gc_work,
> +			      xe_ttm_vram_decay_gc_period(mgr));
> +	mutex_unlock(&mgr->lock);
> +}

[Severity: Low]
Does xe_ttm_vram_decay_gc_worker() need to reschedule mgr->decay_gc_work
unconditionally once it has been started?

After all XE_PAGE_OFFLINE_SOFT entries in mgr->offlined_pages have been
reinstated (or promoted to XE_PAGE_OFFLINE_HARD), or if!decay is taken,
jumping to reschedule keeps re-arming mgr->decay_gc_work every period until
driver teardown. Should the worker only reschedule itself when decay is
non-zero and at least one XE_PAGE_OFFLINE_SOFT entry still remains?

[ ... ]

> @@ -725,8 +776,12 @@ static bool xe_ttm_vram_page_already_processed(struct xe_ttm_vram_mgr *mgr,
>  	lockdep_assert_held(&mgr->lock);
>  
>  	list_for_each_entry(pos, &mgr->offlined_pages, offlined_link) {
> -		if (pos->addr == addr)
> +		if (pos->addr == addr) {
> +			if (pos->mode == XE_PAGE_OFFLINE_SOFT)
> +				pos->mode = XE_PAGE_OFFLINE_HARD;
> +			pos->last_seen = jiffies;
>  			return true;
> +		}
>  	}
>  
>  	list_for_each_entry(pos, &mgr->queued_pages, queued_link) {

[Severity: High]
As reported by Sashiko on RFC [1], what happens if a second hardware fault is
reported for the same address while the page's reservation is still deferred
on mgr->queued_pages?

In xe_ttm_vram_page_already_processed(), the second loop checks
mgr->queued_pages:

	list_for_each_entry(pos, &mgr->queued_pages, queued_link) {
		if (pos->addr == addr)
			return true;
	}

When a page was allocated to a BO at the time of the first fault (or its
buddy block was still busy), xe_ttm_vram_reserve_page_at_addr() places nentry
on mgr->queued_pages with mode = XE_PAGE_OFFLINE_SOFT while
xe_ttm_vram_purge_page() / TTM ghost object cleanup is still in flight.

If a second fault arrives before xe_ttm_vram_mgr_del() calls
xe_ttm_vram_retry_queued_pages(), xe_ttm_vram_page_already_processed() finds
the entry on mgr->queued_pages and returns true without promoting pos->mode
to XE_PAGE_OFFLINE_HARD or updating pos->last_seen. Once
xe_ttm_vram_retry_queued_pages() moves that entry to mgr->offlined_pages,
won't it remain in XE_PAGE_OFFLINE_SOFT mode and be reinstated after the
decay interval despite having faulted twice and being permanently retired in
firmware by handle_page_offline()?

> @@ -804,6 +859,8 @@ static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr,
>  		INIT_LIST_HEAD(&nentry->blocks);
>  		nentry->status = XE_PAGE_RESERVE_PENDING;
>  		nentry->addr = addr;
> +		nentry->mode = XE_PAGE_OFFLINE_SOFT;
> +		nentry->last_seen = jiffies;
>  
>  		if (block) {
>  			pbo = xe_ttm_vram_block_owner_get(xe, block);

[Severity: High]
Won't this also mark permanently retired pages loaded from firmware at driver
probe as XE_PAGE_OFFLINE_SOFT and reinstate them after the decay window?

During probe, xe_ras_init() -> get_offline_list() -> process_offline_pages()
queries firmware for pages that were permanently retired on prior boots and
calls handle_page_offline(xe, common->page_addresses[i], false) once per
address:

drivers/gpu/drm/xe/xe_ras.c:process_offline_pages() {
	...
	for (i = 0; i < common->pages_returned && i < XE_RAS_NUM_PAGES; i++) {
		handle_page_offline(xe, common->page_addresses[i], notify_fw);
		/* The pages are already offlined by firmware */
		if (!notify_fw) {
			index = common->page_addresses[i] >> XE_PTE_SHIFT;
			ret = xa_err(xa_store(&state->offlined_pages, index,
					      XE_RAS_PAGE_OFFLINED, GFP_KERNEL));
	...
}

Because handle_page_offline() -> xe_ttm_vram_handle_addr_fault() ->
xe_ttm_vram_reserve_page_at_addr() is only called once per address at boot
when vram_mgr->offlined_pages is empty, nentry->mode is set to
XE_PAGE_OFFLINE_SOFT and decay_gc_work is scheduled. Once bad_page_decay_secs
elapses, xe_ttm_vram_decay_gc_worker() will free those permanently retired
pages back into the VRAM buddy allocator.

Should pages retired with notify_fw == false (or where firmware retirement has
already occurred) be initialized as XE_PAGE_OFFLINE_HARD?

> @@ -843,6 +900,11 @@ static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr,
>  			}
>  			++vram_mgr->n_offlined_pages;
>  			list_add(&nentry->offlined_link, &vram_mgr->offlined_pages);
> +			if (vram_mgr->decay_interval_jiffies &&
> +			    !delayed_work_pending(&vram_mgr->decay_gc_work)) {
> +				schedule_delayed_work(&vram_mgr->decay_gc_work,
> +						      xe_ttm_vram_decay_gc_period(vram_mgr));
> +			}
>  			return ret;
>  		}
>  	}

[Severity: High]
As reported by Sashiko on RFC [2], if a faulty page is currently allocated to
a BO (block != NULL) or xe_ttm_vram_buddy_alloc() fails because the block is
still busy, xe_ttm_vram_reserve_page_at_addr() queues nentry onto
vram_mgr->queued_pages without scheduling vram_mgr->decay_gc_work.

Later, when the BO is freed in xe_ttm_vram_mgr_del(),
xe_ttm_vram_retry_queued_pages() reserves the buddy block and moves the entry
to mgr->offlined_pages:

drivers/gpu/drm/xe/xe_ttm_vram_mgr.c:xe_ttm_vram_retry_queued_pages() {
	...
		--mgr->n_queued_pages;
		list_del(&pos->queued_link);
		++mgr->n_offlined_pages;
		list_add(&pos->offlined_link, &mgr->offlined_pages);
	}
}

Because xe_ttm_vram_retry_queued_pages() never schedules mgr->decay_gc_work,
won't soft-offlined pages that faulted while in use stay offline forever
unless an unallocated page happens to fault separately on the same manager?

[1] https://lore.kernel.org/r/20260911094616.1333590-2-tejas.upadhyay@intel.com
[2] https://lore.kernel.org/r/20260911094616.1333590-2-tejas.upadhyay@intel.com

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261102122147.2361513-2-tejas.upadhyay@intel.com?part=1

✗ CI.KUnit: failure for drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages (rev2)

[Patchwork]

== Series Details ==

Series: drm/xe/vram: add decay-based reinstatement of soft-offlined VRAM pages (rev2)
URL   : https://patchwork.freedesktop.org/series/173858/
State : failure

== Summary ==

+ trap cleanup EXIT
+ kunitconfigs=('/kernel/drivers/gpu/tests/.kunitconfig' '/kernel/drivers/gpu/drm/xe/.kunitconfig' '/kernel/drivers/gpu/drm/tests/.kunitconfig' '/kernel/drivers/gpu/drm/ttm/tests/.kunitconfig' '/kernel/drivers/dma-buf/.kunitconfig')
+ for kcfg in "${kunitconfigs[@]}"
+ [[ ! -f /kernel/drivers/gpu/tests/.kunitconfig ]]
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/tests/.kunitconfig
[13:24:41] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[13:24:46] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make all compile_commands.json scripts_gdb ARCH=um O=.kunit --jobs=48
[13:25:07] Starting KUnit Kernel (1/1)...
[13:25:07] ============================================================
Running tests with:
$ .kunit/linux kunit.enable=1 mem=1G console=tty kunit_shutdown=halt
[13:25:07] ================= gpu_buddy (14 subtests) ==================
[13:25:07] [PASSED] gpu_test_buddy_alloc_limit
[13:25:07] [PASSED] gpu_test_buddy_alloc_optimistic
[13:25:07] [PASSED] gpu_test_buddy_alloc_pessimistic
[13:25:07] [PASSED] gpu_test_buddy_alloc_pathological
[13:25:07] [PASSED] gpu_test_buddy_alloc_contiguous
[13:25:07] [PASSED] gpu_test_buddy_alloc_clear
[13:25:07] [PASSED] gpu_test_buddy_alloc_range
[13:25:07] [PASSED] gpu_test_buddy_alloc_range_bias
[13:25:08] [PASSED] gpu_test_buddy_fragmentation_performance
[13:25:09] [PASSED] gpu_test_buddy_dirty_tracker_performance
[13:25:09] [PASSED] gpu_test_buddy_alloc_exceeds_max_order
[13:25:09] [PASSED] gpu_test_buddy_offset_aligned_allocation
[13:25:09] [PASSED] gpu_test_buddy_subtree_offset_alignment_stress
[13:25:09] [PASSED] gpu_test_buddy_addr_to_block
[13:25:09] ==================== [PASSED] gpu_buddy ====================
[13:25:09] ============================================================
[13:25:09] Testing complete. Ran 14 tests: passed: 14
[13:25:09] Elapsed time: 27.481s total, 5.164s configuring, 20.450s building, 1.823s running

+ for kcfg in "${kunitconfigs[@]}"
+ [[ ! -f /kernel/drivers/gpu/drm/xe/.kunitconfig ]]
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig
[13:25:09] Configuring KUnit Kernel ...
Regenerating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[13:25:11] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make all compile_commands.json scripts_gdb ARCH=um O=.kunit --jobs=48
ERROR:root:../drivers/gpu/drm/xe/xe_ttm_vram_mgr.c: In function ‘__xe_ttm_vram_mgr_init’:
../drivers/gpu/drm/xe/xe_ttm_vram_mgr.c:499:22: error: implicit declaration of function ‘xe_configfs_get_bad_page_decay_secs’ [-Werror=implicit-function-declaration]
  499 |         decay_secs = xe_configfs_get_bad_page_decay_secs(to_pci_dev(xe->drm.dev));
      |                      ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
make[7]: *** [../scripts/Makefile.build:290: drivers/gpu/drm/xe/xe_ttm_vram_mgr.o] Error 1
make[7]: *** Waiting for unfinished jobs....
make[6]: *** [../scripts/Makefile.build:572: drivers/gpu/drm/xe] Error 2
make[5]: *** [../scripts/Makefile.build:572: drivers/gpu/drm] Error 2
make[4]: *** [../scripts/Makefile.build:572: drivers/gpu] Error 2
make[3]: *** [../scripts/Makefile.build:572: drivers] Error 2
make[3]: *** Waiting for unfinished jobs....
make[2]: *** [/kernel/Makefile:2244: .] Error 2
make[1]: *** [/kernel/Makefile:248: __sub-make] Error 2
make: *** [Makefile:248: __sub-make] Error 2

+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel