[igt-dev] [PATCH i-g-t v2 0/2] vram d3cold threshold test

[igt-dev] [PATCH i-g-t v2 0/2] vram d3cold threshold test

[Anshuman Gupta]

Adding vram_d3cold_threshold subtest.
Also adding support to disbale all crtc to avoid d3 test
failures.

Anshuman Gupta (2):
  tests/xe_pm : Add support to disable all crtc
  test/xe_pm: Add vram_d3cold_threshold subtest

 tests/xe/xe_pm.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)

-- 
2.25.1

[igt-dev] [PATCH i-g-t v2 1/2] tests/xe_pm : Add support to disable all crtc

[Anshuman Gupta]

xe_pm d3 test requires to disable all crtc in order to enter
to d3 state therefore, adding the support to disable all crtc.

Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
---
 tests/xe/xe_pm.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
index 23b8246ed..c71fce892 100644
--- a/tests/xe/xe_pm.c
+++ b/tests/xe/xe_pm.c
@@ -34,6 +34,7 @@ typedef struct {
 	int fd_xe;
 	struct pci_device *pci_xe;
 	struct pci_device *pci_root;
+	igt_display_t display;
 } device_t;
 
 /* runtime_usage is only available if kernel build CONFIG_PM_ADVANCED_DEBUG */
@@ -150,6 +151,23 @@ static bool out_of_d3(device_t device, enum igt_acpi_d_state state)
 	return true;
 }
 
+static void disable_all_crtc(device_t *device)
+{
+	if (!drmModeGetResources(device->fd_xe))
+		return;
+
+	igt_display_require(&device->display, device->fd_xe);
+	igt_modeset_disable_all_outputs(&device->display);
+}
+
+static void display_fini(device_t *device)
+{
+	if (!drmModeGetResources(device->fd_xe))
+		return;
+
+	igt_display_fini(&device->display);
+}
+
 /**
  * SUBTEST: %s-basic
  * Description: set GPU state to %arg[1] and test suspend/autoresume
@@ -378,6 +396,7 @@ igt_main
 
 		get_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_assert(igt_setup_runtime_pm(device.fd_xe));
+		disable_all_crtc(&device);
 	}
 
 	for (const struct s_state *s = s_states; s->name; s++) {
@@ -438,6 +457,7 @@ igt_main
 	}
 
 	igt_fixture {
+		display_fini(&device);
 		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_restore_runtime_pm();
 		xe_device_put(device.fd_xe);
-- 
2.25.1

[igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Anshuman Gupta]

Adding a vram_d3cold_threshold subtest, which creates a Xe bo and
set the vram_d3cold_threshold according to vram used and bo size.
Test setups the d3cold and expect card to be limited to d3hot.

v2:
- Add subtest doc.

Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
---
 tests/xe/xe_pm.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
index c71fce892..eaafead23 100644
--- a/tests/xe/xe_pm.c
+++ b/tests/xe/xe_pm.c
@@ -19,6 +19,7 @@
 #include "igt.h"
 #include "lib/igt_device.h"
 #include "lib/igt_pm.h"
+#include "lib/igt_sysfs.h"
 #include "lib/igt_syncobj.h"
 #include "lib/intel_reg.h"
 
@@ -30,6 +31,8 @@
 #define NO_SUSPEND -1
 #define NO_RPM -1
 
+#define SIZE (4096 * 1024)
+
 typedef struct {
 	int fd_xe;
 	struct pci_device *pci_xe;
@@ -77,6 +80,22 @@ static void set_d3cold_allowed(struct pci_device *pci,
 	close(fd);
 }
 
+static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
+{
+	char path[64];
+	int ret;
+
+	sprintf(path, "device/vram_d3cold_threshold");
+
+	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
+		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
+	else
+		igt_warn("vram_d3cold_threshold is not present\n");
+
+	igt_info("ret value %d", ret);
+	igt_assert(ret > 0);
+}
+
 static bool setup_d3(device_t device, enum igt_acpi_d_state state)
 {
 	switch (state) {
@@ -359,6 +378,62 @@ NULL));
 		igt_assert(in_d3(device, d_state));
 }
 
+/**
+ * SUBTEST: vram_d3cold_threshold
+ * Description:
+ *	Validate whether card is limited to d3hot while vram used
+ *	is greater than vram_d3cold_threshol_d3cold.
+ * Run type: FULL
+ */
+static void test_vram_d3cold_threshold(device_t device)
+{
+	struct drm_xe_query_mem_usage *mem_usage;
+	struct drm_xe_device_query query = {
+		.extensions = 0,
+		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
+		.size = 0,
+		.data = 0,
+	};
+	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold, mmo;
+	uint32_t bo;
+	void *map;
+	int i, sysfs_fd;
+
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+	igt_assert_neq(query.size, 0);
+
+	mem_usage = malloc(query.size);
+	igt_assert(mem_usage);
+
+	query.data = to_user_pointer(mem_usage);
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+
+	for (i = 0; i < mem_usage->num_regions; i++) {
+		if (mem_usage->regions[i].mem_class == XE_MEM_REGION_CLASS_VRAM) {
+			vram_used_mb +=  (mem_usage->regions[i].used / (1024 * 1024));
+			vram_total_mb += (mem_usage->regions[i].total_size / (1024 * 1024));
+		}
+	}
+
+	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, vram_memory(device.fd_xe, 0));
+	mmo = xe_bo_mmap_offset(device.fd_xe, bo);
+	map = mmap(NULL, SIZE, PROT_WRITE, MAP_SHARED, device.fd_xe, mmo);
+	igt_assert(map != MAP_FAILED);
+
+	memset(map, 0, SIZE);
+	munmap(map, SIZE);
+	threshold = vram_used_mb + (SIZE / (1024 * 1024));
+	igt_require(threshold < vram_total_mb);
+	sysfs_fd = igt_sysfs_open(device.fd_xe);
+	set_vram_d3cold_threshold(sysfs_fd, threshold);
+	close(sysfs_fd);
+
+	/* Setup D3Cold but card should be in D0 */
+	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
+	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
+	igt_assert(out_of_d3(device, IGT_ACPI_D3Cold));
+}
+
 igt_main
 {
 	struct drm_xe_engine_class_instance *hwe;
@@ -456,6 +531,11 @@ igt_main
 		}
 	}
 
+	igt_describe("Validate whether card is limited to d3hot, if vram used > vram threshold");
+	igt_subtest("vram-d3cold-threshold") {
+		test_vram_d3cold_threshold(device);
+	}
+
 	igt_fixture {
 		display_fini(&device);
 		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
-- 
2.25.1

Re: [igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Riana Tauro]

Hi Anshuman

On 6/28/2023 4:56 PM, Anshuman Gupta wrote:
> Adding a vram_d3cold_threshold subtest, which creates a Xe bo and
> set the vram_d3cold_threshold according to vram used and bo size.
> Test setups the d3cold and expect card to be limited to d3hot.
> 
> v2:
> - Add subtest doc.
> 
> Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
> ---
>   tests/xe/xe_pm.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 80 insertions(+)
> 
> diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
> index c71fce892..eaafead23 100644
> --- a/tests/xe/xe_pm.c
> +++ b/tests/xe/xe_pm.c
> @@ -19,6 +19,7 @@
>   #include "igt.h"
>   #include "lib/igt_device.h"
>   #include "lib/igt_pm.h"
> +#include "lib/igt_sysfs.h"
>   #include "lib/igt_syncobj.h"
>   #include "lib/intel_reg.h"
>   
> @@ -30,6 +31,8 @@
>   #define NO_SUSPEND -1
>   #define NO_RPM -1
>   
> +#define SIZE (4096 * 1024)
> +
>   typedef struct {
>   	int fd_xe;
>   	struct pci_device *pci_xe;
> @@ -77,6 +80,22 @@ static void set_d3cold_allowed(struct pci_device *pci,
>   	close(fd);
>   }
>   
> +static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
> +{
> +	char path[64];
> +	int ret;
> +
> +	sprintf(path, "device/vram_d3cold_threshold");
> +
> +	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
> +		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
> +	else
> +		igt_warn("vram_d3cold_threshold is not present\n");
> +
> +	igt_info("ret value %d", ret);
> +	igt_assert(ret > 0);
> +}
> +
>   static bool setup_d3(device_t device, enum igt_acpi_d_state state)
>   {
>   	switch (state) {
> @@ -359,6 +378,62 @@ NULL));
>   		igt_assert(in_d3(device, d_state));
>   }
>   
> +/**
> + * SUBTEST: vram_d3cold_threshold
Subtest: vram-d3cold-threshold
> + * Description:
> + *	Validate whether card is limited to d3hot while vram used
> + *	is greater than vram_d3cold_threshol_d3cold.
s/vram_d3cold_threshol_d3cold/vram_d3cold_threshold
> + * Run type: FULL
> + */
> +static void test_vram_d3cold_threshold(device_t device)
> +{
> +	struct drm_xe_query_mem_usage *mem_usage;
> +	struct drm_xe_device_query query = {
> +		.extensions = 0,
> +		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
> +		.size = 0,
> +		.data = 0,
> +	};
> +	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold, mmo;
> +	uint32_t bo;
> +	void *map;
> +	int i, sysfs_fd;
> +
> +	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
> +	igt_assert_neq(query.size, 0);
> +
> +	mem_usage = malloc(query.size);
> +	igt_assert(mem_usage);
> +
> +	query.data = to_user_pointer(mem_usage);
> +	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
> +
> +	for (i = 0; i < mem_usage->num_regions; i++) {
> +		if (mem_usage->regions[i].mem_class == XE_MEM_REGION_CLASS_VRAM) {
> +			vram_used_mb +=  (mem_usage->regions[i].used / (1024 * 1024));
> +			vram_total_mb += (mem_usage->regions[i].total_size / (1024 * 1024));
> +		}
> +	}
> +
> +	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, vram_memory(device.fd_xe, 0));
It will fail on assert of ioctl if vram is not present. Should this test 
be skipped if !xe_has_vram
> +	mmo = xe_bo_mmap_offset(device.fd_xe, bo);
> +	map = mmap(NULL, SIZE, PROT_WRITE, MAP_SHARED, device.fd_xe, mmo);
> +	igt_assert(map != MAP_FAILED);
Replace with xe_bo_map call
> +
> +	memset(map, 0, SIZE);
> +	munmap(map, SIZE);
> +	threshold = vram_used_mb + (SIZE / (1024 * 1024));threshold should be set to vram_used_mb
> +	igt_require(threshold < vram_total_mb); > +	sysfs_fd = igt_sysfs_open(device.fd_xe);
> +	set_vram_d3cold_threshold(sysfs_fd, threshold);
> +	close(sysfs_fd);
> +
> +	/* Setup D3Cold but card should be in D0 */
s/D0/D3hot
> +	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
> +	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
> +	igt_assert(out_of_d3(device, IGT_ACPI_D3Cold));
Release
gem_close(fd, bo);

Thanks
Riana
> +}
> +
>   igt_main
>   {
>   	struct drm_xe_engine_class_instance *hwe;
> @@ -456,6 +531,11 @@ igt_main
>   		}
>   	}
>   
> +	igt_describe("Validate whether card is limited to d3hot, if vram used > vram threshold");
> +	igt_subtest("vram-d3cold-threshold") {
> +		test_vram_d3cold_threshold(device);
> +	}
> +
>   	igt_fixture {
>   		display_fini(&device);
>   		set_d3cold_allowed(device.pci_xe, d3cold_allowed);

[igt-dev] ✗ Fi.CI.BUILD: failure for vram d3cold threshold test (rev2)

[Patchwork]

== Series Details ==

Series: vram d3cold threshold test (rev2)
URL   : https://patchwork.freedesktop.org/series/119914/
State : failure

== Summary ==

IGT patchset build failed on latest successful build
07fe9ec40f779a788946ad371b0b683cab3c2536 Update MAINTAINERS

ninja: Entering directory `/opt/igt/build'
[1/443] Generating version.h with a custom command.
[2/5] Generating xe_tests.rst with a custom command.
FAILED: docs/testplan/xe_tests.rst 
/usr/src/igt-gpu-tools/scripts/igt_doc.py --config /usr/src/igt-gpu-tools/tests/xe/xe_test_config.json --rest docs/testplan/xe_tests.rst --check-testlist --igt-build-path /opt/igt/build
Warning: Missing documentation for igt@xe_pm@vram-d3cold-threshold
ninja: build stopped: subcommand failed.

[igt-dev] [PATCH i-g-t v2 0/2] Add vram_d3cold_threshold test

[Anshuman Gupta]

v2 has major changes in "test/xe_pm: Add vram_d3cold_threshold subtest"

Tests was expecting card to enter to d3cold after freeing up the Xe Bo.
But it wasn't entering because gem_close() doesn't wake device from d3hot.
Therefore test requires to use forcewake debugfs handle to wake device from d3hot and
let XeKMD to compute d3cold.allow based upon used vram and
vram_d3cold_threshold.
Due to above major change in test, i removed the existing RB.

Anshuamn Gupta (1):
  lib/igt_pm: Remove pci_system_{init, cleanup}

Anshuman Gupta (1):
  test/xe_pm: Add vram_d3cold_threshold subtest

 lib/igt_pm.c     |   5 --
 tests/xe/xe_pm.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+), 5 deletions(-)

-- 
2.25.1

[igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Anshuman Gupta]

Adding a vram_d3cold_threshold subtest, which creates a Xe bo and
set the vram_d3cold_threshold according to vram used and bo size.
Test setups the d3cold and expect card to be limited to d3hot.

v2:
- Add subtest doc.
v3:
- skip the test on igfx. [Riana]
- Test doc enhancement. [Riana]
- Create the bo before vram query. [Riana]
- Use xe_bo_map insead of xe_bo_mmap_offset and mmap. [Riana]
- Close the bo handle. [Riana]
v3:
- Restore the vram_d3cold_threshold value. [Badal]
- Don't fail the test if there is no vram_d3cold_threshold.
- Test d3cold after closing the xe BO.
v4:
- When d3cold is not allowed, don't use out_of_d3() as device will
  be runtime suspended.
- Set vram_d3cold_threshold to vram_used + Xe BO size such that
  it can enter to d3cold after freeing up the Xe BO.
- Use forcewake handle to wake the device.

Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
---
 tests/xe/xe_pm.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)

diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
index 23b8246ed..480382f9e 100644
--- a/tests/xe/xe_pm.c
+++ b/tests/xe/xe_pm.c
@@ -19,6 +19,7 @@
 #include "igt.h"
 #include "lib/igt_device.h"
 #include "lib/igt_pm.h"
+#include "lib/igt_sysfs.h"
 #include "lib/igt_syncobj.h"
 #include "lib/intel_reg.h"
 
@@ -30,12 +31,16 @@
 #define NO_SUSPEND -1
 #define NO_RPM -1
 
+#define SIZE (4096 * 1024)
+
 typedef struct {
 	int fd_xe;
 	struct pci_device *pci_xe;
 	struct pci_device *pci_root;
 } device_t;
 
+uint64_t orig_threshold;
+
 /* runtime_usage is only available if kernel build CONFIG_PM_ADVANCED_DEBUG */
 static bool runtime_usage_available(struct pci_device *pci)
 {
@@ -76,6 +81,49 @@ static void set_d3cold_allowed(struct pci_device *pci,
 	close(fd);
 }
 
+static uint64_t get_vram_d3cold_threshold(int sysfs)
+{
+	uint64_t threshold;
+	char path[64];
+	int ret;
+
+	sprintf(path, "device/vram_d3cold_threshold");
+	igt_require_f(!faccessat(sysfs, path, R_OK, 0), "vram_d3cold_threshold is not present\n");
+
+	ret = igt_sysfs_scanf(sysfs, path, "%lu", &threshold);
+	igt_assert(ret > 0);
+
+	return threshold;
+}
+
+static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
+{
+	char path[64];
+	int ret;
+
+	sprintf(path, "device/vram_d3cold_threshold");
+
+	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
+		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
+	else
+		igt_warn("vram_d3cold_threshold is not present\n");
+
+	igt_assert(ret > 0);
+}
+
+static void vram_d3cold_threshold_restore(int sig)
+{
+	int fd, sysfs_fd;
+
+	fd = drm_open_driver(DRIVER_XE);
+	sysfs_fd = igt_sysfs_open(fd);
+
+	set_vram_d3cold_threshold(sysfs_fd, orig_threshold);
+
+	close(sysfs_fd);
+	close(fd);
+}
+
 static bool setup_d3(device_t device, enum igt_acpi_d_state state)
 {
 	switch (state) {
@@ -341,11 +389,82 @@ NULL));
 		igt_assert(in_d3(device, d_state));
 }
 
+/**
+ * SUBTEST: vram-d3cold-threshold
+ * Description:
+ *	Validate whether card is limited to d3hot while vram used
+ *	is greater than vram_d3cold_threshold.
+ * Run type: FULL
+ */
+static void test_vram_d3cold_threshold(device_t device, int sysfs_fd)
+{
+	struct drm_xe_query_mem_usage *mem_usage;
+	struct drm_xe_device_query query = {
+		.extensions = 0,
+		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
+		.size = 0,
+		.data = 0,
+	};
+	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold;
+	uint32_t bo, flags;
+	int handle, i;
+	bool active;
+	void *map;
+
+	igt_require(xe_has_vram(device.fd_xe));
+
+	flags = vram_memory(device.fd_xe, 0);
+	igt_require_f(flags, "Device doesn't support vram memory region\n");
+
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+	igt_assert_neq(query.size, 0);
+
+	mem_usage = malloc(query.size);
+	igt_assert(mem_usage);
+
+	query.data = to_user_pointer(mem_usage);
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+
+	for (i = 0; i < mem_usage->num_regions; i++) {
+		if (mem_usage->regions[i].mem_class == XE_MEM_REGION_CLASS_VRAM) {
+			vram_used_mb +=  (mem_usage->regions[i].used / (1024 * 1024));
+			vram_total_mb += (mem_usage->regions[i].total_size / (1024 * 1024));
+		}
+	}
+
+	threshold = vram_used_mb + (SIZE / 1024 /1024);
+	igt_require(threshold < vram_total_mb);
+
+	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, flags);
+	map = xe_bo_map(device.fd_xe, bo, SIZE);
+	memset(map, 0, SIZE);
+	munmap(map, SIZE);
+	set_vram_d3cold_threshold(sysfs_fd, threshold);
+
+	/* Setup D3Cold but card should be in D3hot */
+	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
+	sleep(1);
+	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
+	igt_assert(igt_pm_get_acpi_real_d_state(device.pci_root) == IGT_ACPI_D0);
+	gem_close(device.fd_xe, bo);
+
+	/* open and close fw handle to wake the device */
+	handle = igt_debugfs_open(device.fd_xe, "forcewake_all", O_RDONLY);
+	igt_assert(handle >= 0);
+	active = igt_get_runtime_pm_status() == IGT_RUNTIME_PM_STATUS_ACTIVE;
+	close(handle);
+	igt_assert(active);
+
+	/* Test D3Cold again after freeing up the Xe BO */
+	igt_assert(in_d3(device, IGT_ACPI_D3Cold));
+}
+
 igt_main
 {
 	struct drm_xe_engine_class_instance *hwe;
 	device_t device;
 	char d3cold_allowed[2];
+	int sysfs_fd;
 	const struct s_state {
 		const char *name;
 		enum igt_suspend_state state;
@@ -378,6 +497,7 @@ igt_main
 
 		get_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_assert(igt_setup_runtime_pm(device.fd_xe));
+		sysfs_fd = igt_sysfs_open(device.fd_xe);
 	}
 
 	for (const struct s_state *s = s_states; s->name; s++) {
@@ -437,7 +557,15 @@ igt_main
 		}
 	}
 
+	igt_describe("Validate whether card is limited to d3hot, if vram used > vram threshold");
+	igt_subtest("vram-d3cold-threshold") {
+		orig_threshold = get_vram_d3cold_threshold(sysfs_fd);
+		igt_install_exit_handler(vram_d3cold_threshold_restore);
+		test_vram_d3cold_threshold(device, sysfs_fd);
+	}
+
 	igt_fixture {
+		close(sysfs_fd);
 		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_restore_runtime_pm();
 		xe_device_put(device.fd_xe);
-- 
2.25.1

Re: [igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Rodrigo Vivi]

On Fri, Aug 11, 2023 at 04:40:26PM +0530, Anshuman Gupta wrote:
> Adding a vram_d3cold_threshold subtest, which creates a Xe bo and
> set the vram_d3cold_threshold according to vram used and bo size.
> Test setups the d3cold and expect card to be limited to d3hot.
> 
> v2:
> - Add subtest doc.
> v3:
> - skip the test on igfx. [Riana]
> - Test doc enhancement. [Riana]
> - Create the bo before vram query. [Riana]
> - Use xe_bo_map insead of xe_bo_mmap_offset and mmap. [Riana]
> - Close the bo handle. [Riana]
> v3:
> - Restore the vram_d3cold_threshold value. [Badal]
> - Don't fail the test if there is no vram_d3cold_threshold.
> - Test d3cold after closing the xe BO.
> v4:
> - When d3cold is not allowed, don't use out_of_d3() as device will
>   be runtime suspended.
> - Set vram_d3cold_threshold to vram_used + Xe BO size such that
>   it can enter to d3cold after freeing up the Xe BO.
> - Use forcewake handle to wake the device.
> 
> Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
> ---
>  tests/xe/xe_pm.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 128 insertions(+)
> 
> diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
> index 23b8246ed..480382f9e 100644
> --- a/tests/xe/xe_pm.c
> +++ b/tests/xe/xe_pm.c
> @@ -19,6 +19,7 @@
>  #include "igt.h"
>  #include "lib/igt_device.h"
>  #include "lib/igt_pm.h"
> +#include "lib/igt_sysfs.h"
>  #include "lib/igt_syncobj.h"
>  #include "lib/intel_reg.h"
>  
> @@ -30,12 +31,16 @@
>  #define NO_SUSPEND -1
>  #define NO_RPM -1
>  
> +#define SIZE (4096 * 1024)
> +
>  typedef struct {
>  	int fd_xe;
>  	struct pci_device *pci_xe;
>  	struct pci_device *pci_root;
>  } device_t;
>  
> +uint64_t orig_threshold;
> +
>  /* runtime_usage is only available if kernel build CONFIG_PM_ADVANCED_DEBUG */
>  static bool runtime_usage_available(struct pci_device *pci)
>  {
> @@ -76,6 +81,49 @@ static void set_d3cold_allowed(struct pci_device *pci,
>  	close(fd);
>  }
>  
> +static uint64_t get_vram_d3cold_threshold(int sysfs)
> +{
> +	uint64_t threshold;
> +	char path[64];
> +	int ret;
> +
> +	sprintf(path, "device/vram_d3cold_threshold");
> +	igt_require_f(!faccessat(sysfs, path, R_OK, 0), "vram_d3cold_threshold is not present\n");
> +
> +	ret = igt_sysfs_scanf(sysfs, path, "%lu", &threshold);
> +	igt_assert(ret > 0);
> +
> +	return threshold;
> +}
> +
> +static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
> +{
> +	char path[64];
> +	int ret;
> +
> +	sprintf(path, "device/vram_d3cold_threshold");
> +
> +	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
> +		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
> +	else
> +		igt_warn("vram_d3cold_threshold is not present\n");
> +
> +	igt_assert(ret > 0);
> +}
> +
> +static void vram_d3cold_threshold_restore(int sig)
> +{
> +	int fd, sysfs_fd;
> +
> +	fd = drm_open_driver(DRIVER_XE);
> +	sysfs_fd = igt_sysfs_open(fd);
> +
> +	set_vram_d3cold_threshold(sysfs_fd, orig_threshold);
> +
> +	close(sysfs_fd);
> +	close(fd);
> +}
> +
>  static bool setup_d3(device_t device, enum igt_acpi_d_state state)
>  {
>  	switch (state) {
> @@ -341,11 +389,82 @@ NULL));
>  		igt_assert(in_d3(device, d_state));
>  }
>  
> +/**
> + * SUBTEST: vram-d3cold-threshold
> + * Description:
> + *	Validate whether card is limited to d3hot while vram used
> + *	is greater than vram_d3cold_threshold.
> + * Run type: FULL
> + */
> +static void test_vram_d3cold_threshold(device_t device, int sysfs_fd)
> +{
> +	struct drm_xe_query_mem_usage *mem_usage;
> +	struct drm_xe_device_query query = {
> +		.extensions = 0,
> +		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
> +		.size = 0,
> +		.data = 0,
> +	};
> +	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold;
> +	uint32_t bo, flags;
> +	int handle, i;
> +	bool active;
> +	void *map;
> +
> +	igt_require(xe_has_vram(device.fd_xe));
> +
> +	flags = vram_memory(device.fd_xe, 0);
> +	igt_require_f(flags, "Device doesn't support vram memory region\n");
> +
> +	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
> +	igt_assert_neq(query.size, 0);
> +
> +	mem_usage = malloc(query.size);
> +	igt_assert(mem_usage);
> +
> +	query.data = to_user_pointer(mem_usage);
> +	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
> +
> +	for (i = 0; i < mem_usage->num_regions; i++) {
> +		if (mem_usage->regions[i].mem_class == XE_MEM_REGION_CLASS_VRAM) {
> +			vram_used_mb +=  (mem_usage->regions[i].used / (1024 * 1024));
> +			vram_total_mb += (mem_usage->regions[i].total_size / (1024 * 1024));
> +		}
> +	}
> +
> +	threshold = vram_used_mb + (SIZE / 1024 /1024);
> +	igt_require(threshold < vram_total_mb);
> +
> +	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, flags);
> +	map = xe_bo_map(device.fd_xe, bo, SIZE);
> +	memset(map, 0, SIZE);
> +	munmap(map, SIZE);
> +	set_vram_d3cold_threshold(sysfs_fd, threshold);
> +
> +	/* Setup D3Cold but card should be in D3hot */
> +	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
> +	sleep(1);
> +	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
> +	igt_assert(igt_pm_get_acpi_real_d_state(device.pci_root) == IGT_ACPI_D0);
> +	gem_close(device.fd_xe, bo);
> +
> +	/* open and close fw handle to wake the device */

hmmm I would expect that we should wake the device with mem_access when cleaning
up the stuff...
Maybe I'm wrong, but I'm in the process of moving the mem_access to xe_pm,
documenting it and taking care of other suspicious cases like xe_exec without VM.

https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/551 assigned for myself.

But anyway, while I don't get to that, let's move with this patch as is.
Maybe adding an "XXX" along with the comment line above? up to you.

Reviewed-by: Rodrigo Vivi <rodrigo.vivi@intel.com>

> +	handle = igt_debugfs_open(device.fd_xe, "forcewake_all", O_RDONLY);
> +	igt_assert(handle >= 0);
> +	active = igt_get_runtime_pm_status() == IGT_RUNTIME_PM_STATUS_ACTIVE;
> +	close(handle);
> +	igt_assert(active);
> +
> +	/* Test D3Cold again after freeing up the Xe BO */
> +	igt_assert(in_d3(device, IGT_ACPI_D3Cold));
> +}
> +
>  igt_main
>  {
>  	struct drm_xe_engine_class_instance *hwe;
>  	device_t device;
>  	char d3cold_allowed[2];
> +	int sysfs_fd;
>  	const struct s_state {
>  		const char *name;
>  		enum igt_suspend_state state;
> @@ -378,6 +497,7 @@ igt_main
>  
>  		get_d3cold_allowed(device.pci_xe, d3cold_allowed);
>  		igt_assert(igt_setup_runtime_pm(device.fd_xe));
> +		sysfs_fd = igt_sysfs_open(device.fd_xe);
>  	}
>  
>  	for (const struct s_state *s = s_states; s->name; s++) {
> @@ -437,7 +557,15 @@ igt_main
>  		}
>  	}
>  
> +	igt_describe("Validate whether card is limited to d3hot, if vram used > vram threshold");
> +	igt_subtest("vram-d3cold-threshold") {
> +		orig_threshold = get_vram_d3cold_threshold(sysfs_fd);
> +		igt_install_exit_handler(vram_d3cold_threshold_restore);
> +		test_vram_d3cold_threshold(device, sysfs_fd);
> +	}
> +
>  	igt_fixture {
> +		close(sysfs_fd);
>  		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
>  		igt_restore_runtime_pm();
>  		xe_device_put(device.fd_xe);
> -- 
> 2.25.1
> 

Re: [igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Gupta, Anshuman]

> -----Original Message-----
> From: Vivi, Rodrigo <rodrigo.vivi@intel.com>
> Sent: Saturday, August 12, 2023 2:39 AM
> To: Gupta, Anshuman <anshuman.gupta@intel.com>
> Cc: igt-dev@lists.freedesktop.org; Nilawar, Badal <badal.nilawar@intel.com>
> Subject: Re: [igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add
> vram_d3cold_threshold subtest
> 
> On Fri, Aug 11, 2023 at 04:40:26PM +0530, Anshuman Gupta wrote:
> > Adding a vram_d3cold_threshold subtest, which creates a Xe bo and set
> > the vram_d3cold_threshold according to vram used and bo size.
> > Test setups the d3cold and expect card to be limited to d3hot.
> >
> > v2:
> > - Add subtest doc.
> > v3:
> > - skip the test on igfx. [Riana]
> > - Test doc enhancement. [Riana]
> > - Create the bo before vram query. [Riana]
> > - Use xe_bo_map insead of xe_bo_mmap_offset and mmap. [Riana]
> > - Close the bo handle. [Riana]
> > v3:
> > - Restore the vram_d3cold_threshold value. [Badal]
> > - Don't fail the test if there is no vram_d3cold_threshold.
> > - Test d3cold after closing the xe BO.
> > v4:
> > - When d3cold is not allowed, don't use out_of_d3() as device will
> >   be runtime suspended.
> > - Set vram_d3cold_threshold to vram_used + Xe BO size such that
> >   it can enter to d3cold after freeing up the Xe BO.
> > - Use forcewake handle to wake the device.
> >
> > Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
> > ---
> >  tests/xe/xe_pm.c | 128
> > +++++++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 128 insertions(+)
> >
> > diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c index
> > 23b8246ed..480382f9e 100644
> > --- a/tests/xe/xe_pm.c
> > +++ b/tests/xe/xe_pm.c
> > @@ -19,6 +19,7 @@
> >  #include "igt.h"
> >  #include "lib/igt_device.h"
> >  #include "lib/igt_pm.h"
> > +#include "lib/igt_sysfs.h"
> >  #include "lib/igt_syncobj.h"
> >  #include "lib/intel_reg.h"
> >
> > @@ -30,12 +31,16 @@
> >  #define NO_SUSPEND -1
> >  #define NO_RPM -1
> >
> > +#define SIZE (4096 * 1024)
> > +
> >  typedef struct {
> >  	int fd_xe;
> >  	struct pci_device *pci_xe;
> >  	struct pci_device *pci_root;
> >  } device_t;
> >
> > +uint64_t orig_threshold;
> > +
> >  /* runtime_usage is only available if kernel build
> > CONFIG_PM_ADVANCED_DEBUG */  static bool
> > runtime_usage_available(struct pci_device *pci)  { @@ -76,6 +81,49 @@
> > static void set_d3cold_allowed(struct pci_device *pci,
> >  	close(fd);
> >  }
> >
> > +static uint64_t get_vram_d3cold_threshold(int sysfs) {
> > +	uint64_t threshold;
> > +	char path[64];
> > +	int ret;
> > +
> > +	sprintf(path, "device/vram_d3cold_threshold");
> > +	igt_require_f(!faccessat(sysfs, path, R_OK, 0),
> > +"vram_d3cold_threshold is not present\n");
> > +
> > +	ret = igt_sysfs_scanf(sysfs, path, "%lu", &threshold);
> > +	igt_assert(ret > 0);
> > +
> > +	return threshold;
> > +}
> > +
> > +static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
> > +{
> > +	char path[64];
> > +	int ret;
> > +
> > +	sprintf(path, "device/vram_d3cold_threshold");
> > +
> > +	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
> > +		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
> > +	else
> > +		igt_warn("vram_d3cold_threshold is not present\n");
> > +
> > +	igt_assert(ret > 0);
> > +}
> > +
> > +static void vram_d3cold_threshold_restore(int sig) {
> > +	int fd, sysfs_fd;
> > +
> > +	fd = drm_open_driver(DRIVER_XE);
> > +	sysfs_fd = igt_sysfs_open(fd);
> > +
> > +	set_vram_d3cold_threshold(sysfs_fd, orig_threshold);
> > +
> > +	close(sysfs_fd);
> > +	close(fd);
> > +}
> > +
> >  static bool setup_d3(device_t device, enum igt_acpi_d_state state)  {
> >  	switch (state) {
> > @@ -341,11 +389,82 @@ NULL));
> >  		igt_assert(in_d3(device, d_state));  }
> >
> > +/**
> > + * SUBTEST: vram-d3cold-threshold
> > + * Description:
> > + *	Validate whether card is limited to d3hot while vram used
> > + *	is greater than vram_d3cold_threshold.
> > + * Run type: FULL
> > + */
> > +static void test_vram_d3cold_threshold(device_t device, int sysfs_fd)
> > +{
> > +	struct drm_xe_query_mem_usage *mem_usage;
> > +	struct drm_xe_device_query query = {
> > +		.extensions = 0,
> > +		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
> > +		.size = 0,
> > +		.data = 0,
> > +	};
> > +	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold;
> > +	uint32_t bo, flags;
> > +	int handle, i;
> > +	bool active;
> > +	void *map;
> > +
> > +	igt_require(xe_has_vram(device.fd_xe));
> > +
> > +	flags = vram_memory(device.fd_xe, 0);
> > +	igt_require_f(flags, "Device doesn't support vram memory
> region\n");
> > +
> > +	igt_assert_eq(igt_ioctl(device.fd_xe,
> DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
> > +	igt_assert_neq(query.size, 0);
> > +
> > +	mem_usage = malloc(query.size);
> > +	igt_assert(mem_usage);
> > +
> > +	query.data = to_user_pointer(mem_usage);
> > +	igt_assert_eq(igt_ioctl(device.fd_xe,
> DRM_IOCTL_XE_DEVICE_QUERY,
> > +&query), 0);
> > +
> > +	for (i = 0; i < mem_usage->num_regions; i++) {
> > +		if (mem_usage->regions[i].mem_class ==
> XE_MEM_REGION_CLASS_VRAM) {
> > +			vram_used_mb +=  (mem_usage->regions[i].used /
> (1024 * 1024));
> > +			vram_total_mb += (mem_usage-
> >regions[i].total_size / (1024 * 1024));
> > +		}
> > +	}
> > +
> > +	threshold = vram_used_mb + (SIZE / 1024 /1024);
> > +	igt_require(threshold < vram_total_mb);
> > +
> > +	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, flags);
> > +	map = xe_bo_map(device.fd_xe, bo, SIZE);
> > +	memset(map, 0, SIZE);
> > +	munmap(map, SIZE);
> > +	set_vram_d3cold_threshold(sysfs_fd, threshold);
> > +
> > +	/* Setup D3Cold but card should be in D3hot */
> > +	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
> > +	sleep(1);
> > +	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
> > +	igt_assert(igt_pm_get_acpi_real_d_state(device.pci_root) ==
> IGT_ACPI_D0);
> > +	gem_close(device.fd_xe, bo);
> > +
> > +	/* open and close fw handle to wake the device */
> 
> hmmm I would expect that we should wake the device with mem_access
> when cleaning up the stuff...
> Maybe I'm wrong, but I'm in the process of moving the mem_access to
> xe_pm, documenting it and taking care of other suspicious cases like xe_exec
> without VM.
> 
> https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/551 assigned for
> myself.
> 
> But anyway, while I don't get to that, let's move with this patch as is.
> Maybe adding an "XXX" along with the comment line above? up to you.
> 
> Reviewed-by: Rodrigo Vivi <rodrigo.vivi@intel.com>
Thanks for review, I will  add a XXX comment.
Br,
Anshuman Gupta.
> 
> > +	handle = igt_debugfs_open(device.fd_xe, "forcewake_all",
> O_RDONLY);
> > +	igt_assert(handle >= 0);
> > +	active = igt_get_runtime_pm_status() ==
> IGT_RUNTIME_PM_STATUS_ACTIVE;
> > +	close(handle);
> > +	igt_assert(active);
> > +
> > +	/* Test D3Cold again after freeing up the Xe BO */
> > +	igt_assert(in_d3(device, IGT_ACPI_D3Cold)); }
> > +
> >  igt_main
> >  {
> >  	struct drm_xe_engine_class_instance *hwe;
> >  	device_t device;
> >  	char d3cold_allowed[2];
> > +	int sysfs_fd;
> >  	const struct s_state {
> >  		const char *name;
> >  		enum igt_suspend_state state;
> > @@ -378,6 +497,7 @@ igt_main
> >
> >  		get_d3cold_allowed(device.pci_xe, d3cold_allowed);
> >  		igt_assert(igt_setup_runtime_pm(device.fd_xe));
> > +		sysfs_fd = igt_sysfs_open(device.fd_xe);
> >  	}
> >
> >  	for (const struct s_state *s = s_states; s->name; s++) { @@ -437,7
> > +557,15 @@ igt_main
> >  		}
> >  	}
> >
> > +	igt_describe("Validate whether card is limited to d3hot, if vram used
> > vram threshold");
> > +	igt_subtest("vram-d3cold-threshold") {
> > +		orig_threshold = get_vram_d3cold_threshold(sysfs_fd);
> > +		igt_install_exit_handler(vram_d3cold_threshold_restore);
> > +		test_vram_d3cold_threshold(device, sysfs_fd);
> > +	}
> > +
> >  	igt_fixture {
> > +		close(sysfs_fd);
> >  		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
> >  		igt_restore_runtime_pm();
> >  		xe_device_put(device.fd_xe);
> > --
> > 2.25.1
> >

[igt-dev] [PATCH i-g-t v2 2/2] test/xe_pm: Add vram_d3cold_threshold subtest

[Anshuman Gupta]

Adding a vram_d3cold_threshold subtest, which creates a Xe bo and
set the vram_d3cold_threshold according to vram used and bo size.
Test setups the d3cold and expect card to be limited to d3hot.

v2:
- Add subtest doc.
v3:
- skip the test on igfx. [Riana]
- Test doc enhancement. [Riana]
- Create the bo before vram query. [Riana]
- Use xe_bo_map insead of xe_bo_mmap_offset and mmap. [Riana]
- Close the bo handle. [Riana]
v3:
- Restore the vram_d3cold_threshold value. [Badal]
- Don't fail the test if there is no vram_d3cold_threshold.
- Test d3cold after closing the xe BO.
v4:
- When d3cold is not allowed, don't use out_of_d3() as device will
  be runtime suspended.
- Set vram_d3cold_threshold to vram_used + Xe BO size such that
  it can enter to d3cold after freeing up the Xe BO.
- Use forcewake handle to wake the device.
v5:
- Added a XXX comment about gem_close() doesn't wake device from
  runtime suspend.

Signed-off-by: Anshuman Gupta <anshuman.gupta@intel.com>
Reviewed-by: Rodrigo Vivi <rodrigo.vivi@intel.com>
---
 tests/xe/xe_pm.c | 132 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 132 insertions(+)

diff --git a/tests/xe/xe_pm.c b/tests/xe/xe_pm.c
index 23b8246ed..c48cc7f33 100644
--- a/tests/xe/xe_pm.c
+++ b/tests/xe/xe_pm.c
@@ -19,6 +19,7 @@
 #include "igt.h"
 #include "lib/igt_device.h"
 #include "lib/igt_pm.h"
+#include "lib/igt_sysfs.h"
 #include "lib/igt_syncobj.h"
 #include "lib/intel_reg.h"
 
@@ -30,12 +31,16 @@
 #define NO_SUSPEND -1
 #define NO_RPM -1
 
+#define SIZE (4096 * 1024)
+
 typedef struct {
 	int fd_xe;
 	struct pci_device *pci_xe;
 	struct pci_device *pci_root;
 } device_t;
 
+uint64_t orig_threshold;
+
 /* runtime_usage is only available if kernel build CONFIG_PM_ADVANCED_DEBUG */
 static bool runtime_usage_available(struct pci_device *pci)
 {
@@ -76,6 +81,49 @@ static void set_d3cold_allowed(struct pci_device *pci,
 	close(fd);
 }
 
+static uint64_t get_vram_d3cold_threshold(int sysfs)
+{
+	uint64_t threshold;
+	char path[64];
+	int ret;
+
+	sprintf(path, "device/vram_d3cold_threshold");
+	igt_require_f(!faccessat(sysfs, path, R_OK, 0), "vram_d3cold_threshold is not present\n");
+
+	ret = igt_sysfs_scanf(sysfs, path, "%lu", &threshold);
+	igt_assert(ret > 0);
+
+	return threshold;
+}
+
+static void set_vram_d3cold_threshold(int sysfs, uint64_t threshold)
+{
+	char path[64];
+	int ret;
+
+	sprintf(path, "device/vram_d3cold_threshold");
+
+	if (!faccessat(sysfs, path, R_OK | W_OK, 0))
+		ret = igt_sysfs_printf(sysfs, path, "%lu", threshold);
+	else
+		igt_warn("vram_d3cold_threshold is not present\n");
+
+	igt_assert(ret > 0);
+}
+
+static void vram_d3cold_threshold_restore(int sig)
+{
+	int fd, sysfs_fd;
+
+	fd = drm_open_driver(DRIVER_XE);
+	sysfs_fd = igt_sysfs_open(fd);
+
+	set_vram_d3cold_threshold(sysfs_fd, orig_threshold);
+
+	close(sysfs_fd);
+	close(fd);
+}
+
 static bool setup_d3(device_t device, enum igt_acpi_d_state state)
 {
 	switch (state) {
@@ -341,11 +389,86 @@ NULL));
 		igt_assert(in_d3(device, d_state));
 }
 
+/**
+ * SUBTEST: vram-d3cold-threshold
+ * Description:
+ *	Validate whether card is limited to d3hot while vram used
+ *	is greater than vram_d3cold_threshold.
+ * Run type: FULL
+ */
+static void test_vram_d3cold_threshold(device_t device, int sysfs_fd)
+{
+	struct drm_xe_query_mem_usage *mem_usage;
+	struct drm_xe_device_query query = {
+		.extensions = 0,
+		.query = DRM_XE_DEVICE_QUERY_MEM_USAGE,
+		.size = 0,
+		.data = 0,
+	};
+	uint64_t vram_used_mb = 0, vram_total_mb = 0, threshold;
+	uint32_t bo, flags;
+	int handle, i;
+	bool active;
+	void *map;
+
+	igt_require(xe_has_vram(device.fd_xe));
+
+	flags = vram_memory(device.fd_xe, 0);
+	igt_require_f(flags, "Device doesn't support vram memory region\n");
+
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+	igt_assert_neq(query.size, 0);
+
+	mem_usage = malloc(query.size);
+	igt_assert(mem_usage);
+
+	query.data = to_user_pointer(mem_usage);
+	igt_assert_eq(igt_ioctl(device.fd_xe, DRM_IOCTL_XE_DEVICE_QUERY, &query), 0);
+
+	for (i = 0; i < mem_usage->num_regions; i++) {
+		if (mem_usage->regions[i].mem_class == XE_MEM_REGION_CLASS_VRAM) {
+			vram_used_mb +=  (mem_usage->regions[i].used / (1024 * 1024));
+			vram_total_mb += (mem_usage->regions[i].total_size / (1024 * 1024));
+		}
+	}
+
+	threshold = vram_used_mb + (SIZE / 1024 /1024);
+	igt_require(threshold < vram_total_mb);
+
+	bo = xe_bo_create_flags(device.fd_xe, 0, SIZE, flags);
+	map = xe_bo_map(device.fd_xe, bo, SIZE);
+	memset(map, 0, SIZE);
+	munmap(map, SIZE);
+	set_vram_d3cold_threshold(sysfs_fd, threshold);
+
+	/* Setup D3Cold but card should be in D3hot */
+	igt_assert(setup_d3(device, IGT_ACPI_D3Cold));
+	sleep(1);
+	igt_assert(in_d3(device, IGT_ACPI_D3Hot));
+	igt_assert(igt_pm_get_acpi_real_d_state(device.pci_root) == IGT_ACPI_D0);
+	gem_close(device.fd_xe, bo);
+
+	/*
+	 * XXX: Xe gem_close() doesn't get any mem_access ref count to wake
+	 * the device from runtime suspend.
+	 * Therefore open and close fw handle to wake the device.
+	 */
+	handle = igt_debugfs_open(device.fd_xe, "forcewake_all", O_RDONLY);
+	igt_assert(handle >= 0);
+	active = igt_get_runtime_pm_status() == IGT_RUNTIME_PM_STATUS_ACTIVE;
+	close(handle);
+	igt_assert(active);
+
+	/* Test D3Cold again after freeing up the Xe BO */
+	igt_assert(in_d3(device, IGT_ACPI_D3Cold));
+}
+
 igt_main
 {
 	struct drm_xe_engine_class_instance *hwe;
 	device_t device;
 	char d3cold_allowed[2];
+	int sysfs_fd;
 	const struct s_state {
 		const char *name;
 		enum igt_suspend_state state;
@@ -378,6 +501,7 @@ igt_main
 
 		get_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_assert(igt_setup_runtime_pm(device.fd_xe));
+		sysfs_fd = igt_sysfs_open(device.fd_xe);
 	}
 
 	for (const struct s_state *s = s_states; s->name; s++) {
@@ -437,7 +561,15 @@ igt_main
 		}
 	}
 
+	igt_describe("Validate whether card is limited to d3hot, if vram used > vram threshold");
+	igt_subtest("vram-d3cold-threshold") {
+		orig_threshold = get_vram_d3cold_threshold(sysfs_fd);
+		igt_install_exit_handler(vram_d3cold_threshold_restore);
+		test_vram_d3cold_threshold(device, sysfs_fd);
+	}
+
 	igt_fixture {
+		close(sysfs_fd);
 		set_d3cold_allowed(device.pci_xe, d3cold_allowed);
 		igt_restore_runtime_pm();
 		xe_device_put(device.fd_xe);
-- 
2.25.1