[PATCH 0/1] Fix device private page livelock on CPU fault

[PATCH 0/1] Fix device private page livelock on CPU fault

[Matthew Brost]

The patch should explain the issue thoroughly.

It would be helpful if AMD could test this patch with their SVM
implementation and see if any issues arise.

Matt

Matthew Brost (1):
  mm/migrate: Trylock device page in do_swap_page

 mm/memory.c         | 13 +++++++---
 mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
 2 files changed, 50 insertions(+), 23 deletions(-)

-- 
2.34.1

[PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

Avoid multiple CPU page faults to the same device page racing by locking
the page in do_swap_page before taking an additional reference to the
page. This prevents scenarios where multiple CPU page faults each take
an extra reference to a device page, which could abort migration in
folio_migrate_mapping. With the device page locked in do_swap_page, the
migrate_vma_* functions need to be updated to avoid locking the
fault_page argument.

Prior to this change, a livelock scenario could occur in Xe's (Intel GPU
DRM driver) SVM implementation if enough threads faulted the same device
page.

Cc: Philip Yang <Philip.Yang@amd.com>
Cc: Felix Kuehling <felix.kuehling@amd.com>
Cc: Christian König <christian.koenig@amd.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
---
 mm/memory.c         | 13 +++++++---
 mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
 2 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index 3c01d68065be..bbd97d16a96a 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 			 * Get a page reference while we know the page can't be
 			 * freed.
 			 */
-			get_page(vmf->page);
-			pte_unmap_unlock(vmf->pte, vmf->ptl);
-			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
-			put_page(vmf->page);
+			if (trylock_page(vmf->page)) {
+				get_page(vmf->page);
+				pte_unmap_unlock(vmf->pte, vmf->ptl);
+				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
+				put_page(vmf->page);
+				unlock_page(vmf->page);
+			} else {
+				pte_unmap_unlock(vmf->pte, vmf->ptl);
+			}
 		} else if (is_hwpoison_entry(entry)) {
 			ret = VM_FAULT_HWPOISON;
 		} else if (is_pte_marker_entry(entry)) {
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 6d66dc1c6ffa..049893a5a179 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 				   struct mm_walk *walk)
 {
 	struct migrate_vma *migrate = walk->private;
+	struct folio *fault_folio = migrate->fault_page ?
+		page_folio(migrate->fault_page) : NULL;
 	struct vm_area_struct *vma = walk->vma;
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long addr = start, unmapped = 0;
@@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 
 			folio_get(folio);
 			spin_unlock(ptl);
-			if (unlikely(!folio_trylock(folio)))
+			if (unlikely(fault_folio != folio &&
+				     !folio_trylock(folio)))
 				return migrate_vma_collect_skip(start, end,
 								walk);
 			ret = split_folio(folio);
-			folio_unlock(folio);
+			if (fault_folio != folio)
+				folio_unlock(folio);
 			folio_put(folio);
 			if (ret)
 				return migrate_vma_collect_skip(start, end,
@@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 		 * optimisation to avoid walking the rmap later with
 		 * try_to_migrate().
 		 */
-		if (folio_trylock(folio)) {
+		if (fault_folio == folio || folio_trylock(folio)) {
 			bool anon_exclusive;
 			pte_t swp_pte;
 
@@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 
 				if (folio_try_share_anon_rmap_pte(folio, page)) {
 					set_pte_at(mm, addr, ptep, pte);
-					folio_unlock(folio);
+					if (fault_folio != folio)
+						folio_unlock(folio);
 					folio_put(folio);
 					mpfn = 0;
 					goto next;
@@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
 					  unsigned long npages,
 					  struct page *fault_page)
 {
+	struct folio *fault_folio = fault_page ?
+		page_folio(fault_page) : NULL;
 	unsigned long i, restore = 0;
 	bool allow_drain = true;
 	unsigned long unmapped = 0;
@@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
 		remove_migration_ptes(folio, folio, false);
 
 		src_pfns[i] = 0;
-		folio_unlock(folio);
+		if (fault_folio != folio)
+			folio_unlock(folio);
 		folio_put(folio);
 		restore--;
 	}
@@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
 		return -EINVAL;
 	if (args->fault_page && !is_device_private_page(args->fault_page))
 		return -EINVAL;
+	if (args->fault_page && !PageLocked(args->fault_page))
+		return -EINVAL;
 
 	memset(args->src, 0, sizeof(*args->src) * nr_pages);
 	args->cpages = 0;
@@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
 }
 EXPORT_SYMBOL(migrate_vma_pages);
 
-/*
- * migrate_device_finalize() - complete page migration
- * @src_pfns: src_pfns returned from migrate_device_range()
- * @dst_pfns: array of pfns allocated by the driver to migrate memory to
- * @npages: number of pages in the range
- *
- * Completes migration of the page by removing special migration entries.
- * Drivers must ensure copying of page data is complete and visible to the CPU
- * before calling this.
- */
-void migrate_device_finalize(unsigned long *src_pfns,
-			unsigned long *dst_pfns, unsigned long npages)
+static void __migrate_device_finalize(unsigned long *src_pfns,
+				      unsigned long *dst_pfns,
+				      unsigned long npages,
+				      struct page *fault_page)
 {
+	struct folio *fault_folio = fault_page ?
+		page_folio(fault_page) : NULL;
 	unsigned long i;
 
 	for (i = 0; i < npages; i++) {
@@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
 		src = page_folio(page);
 		dst = page_folio(newpage);
 		remove_migration_ptes(src, dst, false);
-		folio_unlock(src);
+		if (fault_folio != src)
+			folio_unlock(src);
 
 		if (is_zone_device_page(page))
 			put_page(page);
@@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
 		}
 	}
 }
+
+/*
+ * migrate_device_finalize() - complete page migration
+ * @src_pfns: src_pfns returned from migrate_device_range()
+ * @dst_pfns: array of pfns allocated by the driver to migrate memory to
+ * @npages: number of pages in the range
+ *
+ * Completes migration of the page by removing special migration entries.
+ * Drivers must ensure copying of page data is complete and visible to the CPU
+ * before calling this.
+ */
+void migrate_device_finalize(unsigned long *src_pfns,
+			unsigned long *dst_pfns, unsigned long npages)
+{
+	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
+}
 EXPORT_SYMBOL(migrate_device_finalize);
 
 /**
@@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
  */
 void migrate_vma_finalize(struct migrate_vma *migrate)
 {
-	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
+	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
+				  migrate->fault_page);
 }
 EXPORT_SYMBOL(migrate_vma_finalize);
 
-- 
2.34.1

✓ CI.Patch_applied: success for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : success

== Summary ==

=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 43852924969a drm-tip: 2024y-09m-11d-02h-46m-59s UTC integration manifest
=== git am output follows ===
Applying: mm/migrate: Trylock device page in do_swap_page

✗ CI.checkpatch: warning for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : warning

== Summary ==

+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt
Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master
c62d7e164862503a3662a095da1c6c9014248cb2
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit a4b4054ef029e32d07154acac270ada3909e9cce
Author: Matthew Brost <matthew.brost@intel.com>
Date:   Tue Sep 10 20:03:37 2024 -0700

    mm/migrate: Trylock device page in do_swap_page
    
    Avoid multiple CPU page faults to the same device page racing by locking
    the page in do_swap_page before taking an additional reference to the
    page. This prevents scenarios where multiple CPU page faults each take
    an extra reference to a device page, which could abort migration in
    folio_migrate_mapping. With the device page locked in do_swap_page, the
    migrate_vma_* functions need to be updated to avoid locking the
    fault_page argument.
    
    Prior to this change, a livelock scenario could occur in Xe's (Intel GPU
    DRM driver) SVM implementation if enough threads faulted the same device
    page.
    
    Cc: Philip Yang <Philip.Yang@amd.com>
    Cc: Felix Kuehling <felix.kuehling@amd.com>
    Cc: Christian König <christian.koenig@amd.com>
    Cc: Andrew Morton <akpm@linux-foundation.org>
    Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
    Signed-off-by: Matthew Brost <matthew.brost@intel.com>
+ /mt/dim checkpatch 43852924969a3fbe93796d71bece8e5539aa4845 drm-intel
a4b4054ef029 mm/migrate: Trylock device page in do_swap_page
-:25: WARNING:BAD_SIGN_OFF: Non-standard signature: 'Suggessted-by:' - perhaps 'Suggested-by:'?
#25: 
Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>

-:180: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#180: FILE: mm/migrate_device.c:874:
+void migrate_device_finalize(unsigned long *src_pfns,
+			unsigned long *dst_pfns, unsigned long npages)

total: 0 errors, 1 warnings, 1 checks, 149 lines checked

✓ CI.KUnit: success for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : success

== Summary ==

+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig
[03:10:05] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[03:10:09] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
  156 | u64 ioread64_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
  163 | u64 ioread64_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
  170 | u64 ioread64be_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
  178 | u64 ioread64be_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
  264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
  272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
  280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
  288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~

[03:10:35] Starting KUnit Kernel (1/1)...
[03:10:35] ============================================================
Running tests with:
$ .kunit/linux kunit.enable=1 mem=1G console=tty kunit_shutdown=halt
[03:10:35] =================== guc_dbm (7 subtests) ===================
[03:10:35] [PASSED] test_empty
[03:10:35] [PASSED] test_default
[03:10:35] ======================== test_size  ========================
[03:10:35] [PASSED] 4
[03:10:35] [PASSED] 8
[03:10:35] [PASSED] 32
[03:10:35] [PASSED] 256
[03:10:35] ==================== [PASSED] test_size ====================
[03:10:35] ======================= test_reuse  ========================
[03:10:35] [PASSED] 4
[03:10:35] [PASSED] 8
[03:10:35] [PASSED] 32
[03:10:35] [PASSED] 256
[03:10:35] =================== [PASSED] test_reuse ====================
[03:10:35] =================== test_range_overlap  ====================
[03:10:35] [PASSED] 4
[03:10:35] [PASSED] 8
[03:10:35] [PASSED] 32
[03:10:35] [PASSED] 256
[03:10:35] =============== [PASSED] test_range_overlap ================
[03:10:35] =================== test_range_compact  ====================
[03:10:35] [PASSED] 4
[03:10:35] [PASSED] 8
[03:10:35] [PASSED] 32
[03:10:35] [PASSED] 256
[03:10:35] =============== [PASSED] test_range_compact ================
[03:10:35] ==================== test_range_spare  =====================
[03:10:35] [PASSED] 4
[03:10:35] [PASSED] 8
[03:10:35] [PASSED] 32
[03:10:35] [PASSED] 256
[03:10:35] ================ [PASSED] test_range_spare =================
[03:10:35] ===================== [PASSED] guc_dbm =====================
[03:10:35] =================== guc_idm (6 subtests) ===================
[03:10:35] [PASSED] bad_init
[03:10:35] [PASSED] no_init
[03:10:35] [PASSED] init_fini
[03:10:35] [PASSED] check_used
[03:10:35] [PASSED] check_quota
[03:10:35] [PASSED] check_all
[03:10:35] ===================== [PASSED] guc_idm =====================
[03:10:35] ================== no_relay (3 subtests) ===================
[03:10:35] [PASSED] xe_drops_guc2pf_if_not_ready
[03:10:35] [PASSED] xe_drops_guc2vf_if_not_ready
[03:10:35] [PASSED] xe_rejects_send_if_not_ready
[03:10:35] ==================== [PASSED] no_relay =====================
[03:10:35] ================== pf_relay (14 subtests) ==================
[03:10:35] [PASSED] pf_rejects_guc2pf_too_short
[03:10:35] [PASSED] pf_rejects_guc2pf_too_long
[03:10:35] [PASSED] pf_rejects_guc2pf_no_payload
[03:10:35] [PASSED] pf_fails_no_payload
[03:10:35] [PASSED] pf_fails_bad_origin
[03:10:35] [PASSED] pf_fails_bad_type
[03:10:35] [PASSED] pf_txn_reports_error
[03:10:35] [PASSED] pf_txn_sends_pf2guc
[03:10:35] [PASSED] pf_sends_pf2guc
[03:10:35] [SKIPPED] pf_loopback_nop
[03:10:35] [SKIPPED] pf_loopback_echo
[03:10:35] [SKIPPED] pf_loopback_fail
[03:10:35] [SKIPPED] pf_loopback_busy
[03:10:35] [SKIPPED] pf_loopback_retry
[03:10:35] ==================== [PASSED] pf_relay =====================
[03:10:35] ================== vf_relay (3 subtests) ===================
[03:10:35] [PASSED] vf_rejects_guc2vf_too_short
[03:10:35] [PASSED] vf_rejects_guc2vf_too_long
[03:10:35] [PASSED] vf_rejects_guc2vf_no_payload
[03:10:35] ==================== [PASSED] vf_relay =====================
[03:10:35] ================= pf_service (11 subtests) =================
[03:10:35] [PASSED] pf_negotiate_any
[03:10:35] [PASSED] pf_negotiate_base_match
[03:10:35] [PASSED] pf_negotiate_base_newer
[03:10:35] [PASSED] pf_negotiate_base_next
[03:10:35] [SKIPPED] pf_negotiate_base_older
[03:10:35] [PASSED] pf_negotiate_base_prev
[03:10:35] [PASSED] pf_negotiate_latest_match
[03:10:35] [PASSED] pf_negotiate_latest_newer
[03:10:35] [PASSED] pf_negotiate_latest_next
[03:10:35] [SKIPPED] pf_negotiate_latest_older
[03:10:35] [SKIPPED] pf_negotiate_latest_prev
[03:10:35] =================== [PASSED] pf_service ====================
[03:10:35] ===================== lmtt (1 subtest) =====================
[03:10:35] ======================== test_ops  =========================
[03:10:35] [PASSED] 2-level
[03:10:35] [PASSED] multi-level
[03:10:35] ==================== [PASSED] test_ops =====================
[03:10:35] ====================== [PASSED] lmtt =======================
[03:10:35] =================== xe_mocs (2 subtests) ===================
[03:10:35] ================ xe_live_mocs_kernel_kunit  ================
[03:10:35] =========== [SKIPPED] xe_live_mocs_kernel_kunit ============
[03:10:35] ================ xe_live_mocs_reset_kunit  =================
[03:10:35] ============ [SKIPPED] xe_live_mocs_reset_kunit ============
[03:10:35] ==================== [SKIPPED] xe_mocs =====================
[03:10:35] ================= xe_migrate (2 subtests) ==================
[03:10:35] ================= xe_migrate_sanity_kunit  =================
[03:10:35] ============ [SKIPPED] xe_migrate_sanity_kunit =============
[03:10:35] ================== xe_validate_ccs_kunit  ==================
[03:10:35] ============= [SKIPPED] xe_validate_ccs_kunit ==============
[03:10:35] =================== [SKIPPED] xe_migrate ===================
[03:10:35] ================== xe_dma_buf (1 subtest) ==================
[03:10:35] ==================== xe_dma_buf_kunit  =====================
[03:10:35] ================ [SKIPPED] xe_dma_buf_kunit ================
[03:10:35] =================== [SKIPPED] xe_dma_buf ===================
[03:10:35] ==================== xe_bo (2 subtests) ====================
[03:10:35] ================== xe_ccs_migrate_kunit  ===================
[03:10:35] ============== [SKIPPED] xe_ccs_migrate_kunit ==============
[03:10:35] ==================== xe_bo_evict_kunit  ====================
[03:10:35] =============== [SKIPPED] xe_bo_evict_kunit ================
[03:10:35] ===================== [SKIPPED] xe_bo ======================
[03:10:35] ==================== args (11 subtests) ====================
[03:10:35] [PASSED] count_args_test
[03:10:35] [PASSED] call_args_example
[03:10:35] [PASSED] call_args_test
[03:10:35] [PASSED] drop_first_arg_example
[03:10:35] [PASSED] drop_first_arg_test
[03:10:35] [PASSED] first_arg_example
[03:10:35] [PASSED] first_arg_test
[03:10:35] [PASSED] last_arg_example
[03:10:35] [PASSED] last_arg_test
[03:10:35] [PASSED] pick_arg_example
[03:10:35] [PASSED] sep_comma_example
[03:10:35] ====================== [PASSED] args =======================
[03:10:35] =================== xe_pci (2 subtests) ====================
stty: 'standard input': Inappropriate ioctl for device
[03:10:35] [PASSED] xe_gmdid_graphics_ip
[03:10:35] [PASSED] xe_gmdid_media_ip
[03:10:35] ===================== [PASSED] xe_pci ======================
[03:10:35] =================== xe_rtp (2 subtests) ====================
[03:10:35] =============== xe_rtp_process_to_sr_tests  ================
[03:10:35] [PASSED] coalesce-same-reg
[03:10:35] [PASSED] no-match-no-add
[03:10:35] [PASSED] match-or
[03:10:35] [PASSED] match-or-xfail
[03:10:35] [PASSED] no-match-no-add-multiple-rules
[03:10:35] [PASSED] two-regs-two-entries
[03:10:35] [PASSED] clr-one-set-other
[03:10:35] [PASSED] set-field
[03:10:35] [PASSED] conflict-duplicate
[03:10:35] [PASSED] conflict-not-disjoint
[03:10:35] [PASSED] conflict-reg-type
[03:10:35] =========== [PASSED] xe_rtp_process_to_sr_tests ============
[03:10:35] ================== xe_rtp_process_tests  ===================
[03:10:35] [PASSED] active1
[03:10:35] [PASSED] active2
[03:10:35] [PASSED] active-inactive
[03:10:35] [PASSED] inactive-active
[03:10:35] [PASSED] inactive-1st_or_active-inactive
[03:10:35] [PASSED] inactive-2nd_or_active-inactive
[03:10:35] [PASSED] inactive-last_or_active-inactive
[03:10:35] [PASSED] inactive-no_or_active-inactive
[03:10:35] ============== [PASSED] xe_rtp_process_tests ===============
[03:10:35] ===================== [PASSED] xe_rtp ======================
[03:10:35] ==================== xe_wa (1 subtest) =====================
[03:10:35] ======================== xe_wa_gt  =========================
[03:10:35] [PASSED] TIGERLAKE (B0)
[03:10:35] [PASSED] DG1 (A0)
[03:10:35] [PASSED] DG1 (B0)
[03:10:35] [PASSED] ALDERLAKE_S (A0)
[03:10:35] [PASSED] ALDERLAKE_S (B0)
[03:10:35] [PASSED] ALDERLAKE_S (C0)
[03:10:35] [PASSED] ALDERLAKE_S (D0)
[03:10:35] [PASSED] ALDERLAKE_P (A0)
[03:10:35] [PASSED] ALDERLAKE_P (B0)
[03:10:35] [PASSED] ALDERLAKE_P (C0)
[03:10:35] [PASSED] ALDERLAKE_S_RPLS (D0)
[03:10:35] [PASSED] ALDERLAKE_P_RPLU (E0)
[03:10:35] [PASSED] DG2_G10 (C0)
[03:10:35] [PASSED] DG2_G11 (B1)
[03:10:35] [PASSED] DG2_G12 (A1)
[03:10:35] [PASSED] METEORLAKE (g:A0, m:A0)
[03:10:35] [PASSED] METEORLAKE (g:A0, m:A0)
[03:10:35] [PASSED] METEORLAKE (g:A0, m:A0)
[03:10:35] [PASSED] LUNARLAKE (g:A0, m:A0)
[03:10:35] [PASSED] LUNARLAKE (g:B0, m:A0)
[03:10:35] [PASSED] BATTLEMAGE (g:A0, m:A1)
[03:10:35] ==================== [PASSED] xe_wa_gt =====================
[03:10:35] ====================== [PASSED] xe_wa ======================
[03:10:35] ============================================================
[03:10:35] Testing complete. Ran 121 tests: passed: 106, skipped: 15
[03:10:35] Elapsed time: 30.141s total, 4.153s configuring, 25.712s building, 0.237s running

+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/tests/.kunitconfig
[03:10:35] Configuring KUnit Kernel ...
Regenerating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[03:10:37] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
  156 | u64 ioread64_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
  163 | u64 ioread64_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
  170 | u64 ioread64be_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
  178 | u64 ioread64be_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
  264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
  272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
  280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
  288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~

[03:10:58] Starting KUnit Kernel (1/1)...
[03:10:58] ============================================================
Running tests with:
$ .kunit/linux kunit.enable=1 mem=1G console=tty kunit_shutdown=halt
[03:10:59] ============ drm_test_pick_cmdline (2 subtests) ============
[03:10:59] [PASSED] drm_test_pick_cmdline_res_1920_1080_60
[03:10:59] =============== drm_test_pick_cmdline_named  ===============
[03:10:59] [PASSED] NTSC
[03:10:59] [PASSED] NTSC-J
[03:10:59] [PASSED] PAL
[03:10:59] [PASSED] PAL-M
[03:10:59] =========== [PASSED] drm_test_pick_cmdline_named ===========
[03:10:59] ============== [PASSED] drm_test_pick_cmdline ==============
[03:10:59] ================== drm_buddy (7 subtests) ==================
[03:10:59] [PASSED] drm_test_buddy_alloc_limit
[03:10:59] [PASSED] drm_test_buddy_alloc_optimistic
[03:10:59] [PASSED] drm_test_buddy_alloc_pessimistic
[03:10:59] [PASSED] drm_test_buddy_alloc_pathological
[03:10:59] [PASSED] drm_test_buddy_alloc_contiguous
[03:10:59] [PASSED] drm_test_buddy_alloc_clear
[03:10:59] [PASSED] drm_test_buddy_alloc_range_bias
[03:10:59] ==================== [PASSED] drm_buddy ====================
[03:10:59] ============= drm_cmdline_parser (40 subtests) =============
[03:10:59] [PASSED] drm_test_cmdline_force_d_only
[03:10:59] [PASSED] drm_test_cmdline_force_D_only_dvi
[03:10:59] [PASSED] drm_test_cmdline_force_D_only_hdmi
[03:10:59] [PASSED] drm_test_cmdline_force_D_only_not_digital
[03:10:59] [PASSED] drm_test_cmdline_force_e_only
[03:10:59] [PASSED] drm_test_cmdline_res
[03:10:59] [PASSED] drm_test_cmdline_res_vesa
[03:10:59] [PASSED] drm_test_cmdline_res_vesa_rblank
[03:10:59] [PASSED] drm_test_cmdline_res_rblank
[03:10:59] [PASSED] drm_test_cmdline_res_bpp
[03:10:59] [PASSED] drm_test_cmdline_res_refresh
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_interlaced
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_margins
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_force_off
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_force_on
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_force_on_analog
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_force_on_digital
[03:10:59] [PASSED] drm_test_cmdline_res_bpp_refresh_interlaced_margins_force_on
[03:10:59] [PASSED] drm_test_cmdline_res_margins_force_on
[03:10:59] [PASSED] drm_test_cmdline_res_vesa_margins
[03:10:59] [PASSED] drm_test_cmdline_name
[03:10:59] [PASSED] drm_test_cmdline_name_bpp
[03:10:59] [PASSED] drm_test_cmdline_name_option
[03:10:59] [PASSED] drm_test_cmdline_name_bpp_option
[03:10:59] [PASSED] drm_test_cmdline_rotate_0
[03:10:59] [PASSED] drm_test_cmdline_rotate_90
[03:10:59] [PASSED] drm_test_cmdline_rotate_180
[03:10:59] [PASSED] drm_test_cmdline_rotate_270
[03:10:59] [PASSED] drm_test_cmdline_hmirror
[03:10:59] [PASSED] drm_test_cmdline_vmirror
[03:10:59] [PASSED] drm_test_cmdline_margin_options
[03:10:59] [PASSED] drm_test_cmdline_multiple_options
[03:10:59] [PASSED] drm_test_cmdline_bpp_extra_and_option
[03:10:59] [PASSED] drm_test_cmdline_extra_and_option
[03:10:59] [PASSED] drm_test_cmdline_freestanding_options
[03:10:59] [PASSED] drm_test_cmdline_freestanding_force_e_and_options
[03:10:59] [PASSED] drm_test_cmdline_panel_orientation
[03:10:59] ================ drm_test_cmdline_invalid  =================
[03:10:59] [PASSED] margin_only
[03:10:59] [PASSED] interlace_only
[03:10:59] [PASSED] res_missing_x
[03:10:59] [PASSED] res_missing_y
[03:10:59] [PASSED] res_bad_y
[03:10:59] [PASSED] res_missing_y_bpp
[03:10:59] [PASSED] res_bad_bpp
[03:10:59] [PASSED] res_bad_refresh
[03:10:59] [PASSED] res_bpp_refresh_force_on_off
[03:10:59] [PASSED] res_invalid_mode
[03:10:59] [PASSED] res_bpp_wrong_place_mode
[03:10:59] [PASSED] name_bpp_refresh
[03:10:59] [PASSED] name_refresh
[03:10:59] [PASSED] name_refresh_wrong_mode
[03:10:59] [PASSED] name_refresh_invalid_mode
[03:10:59] [PASSED] rotate_multiple
[03:10:59] [PASSED] rotate_invalid_val
[03:10:59] [PASSED] rotate_truncated
[03:10:59] [PASSED] invalid_option
[03:10:59] [PASSED] invalid_tv_option
[03:10:59] [PASSED] truncated_tv_option
[03:10:59] ============ [PASSED] drm_test_cmdline_invalid =============
[03:10:59] =============== drm_test_cmdline_tv_options  ===============
[03:10:59] [PASSED] NTSC
[03:10:59] [PASSED] NTSC_443
[03:10:59] [PASSED] NTSC_J
[03:10:59] [PASSED] PAL
[03:10:59] [PASSED] PAL_M
[03:10:59] [PASSED] PAL_N
[03:10:59] [PASSED] SECAM
[03:10:59] [PASSED] MONO_525
[03:10:59] [PASSED] MONO_625
[03:10:59] =========== [PASSED] drm_test_cmdline_tv_options ===========
[03:10:59] =============== [PASSED] drm_cmdline_parser ================
[03:10:59] ========== drmm_connector_hdmi_init (19 subtests) ==========
[03:10:59] [PASSED] drm_test_connector_hdmi_init_valid
[03:10:59] [PASSED] drm_test_connector_hdmi_init_bpc_8
[03:10:59] [PASSED] drm_test_connector_hdmi_init_bpc_10
[03:10:59] [PASSED] drm_test_connector_hdmi_init_bpc_12
[03:10:59] [PASSED] drm_test_connector_hdmi_init_bpc_invalid
[03:10:59] [PASSED] drm_test_connector_hdmi_init_bpc_null
[03:10:59] [PASSED] drm_test_connector_hdmi_init_formats_empty
[03:10:59] [PASSED] drm_test_connector_hdmi_init_formats_no_rgb
[03:10:59] [PASSED] drm_test_connector_hdmi_init_null_ddc
[03:10:59] [PASSED] drm_test_connector_hdmi_init_null_product
[03:10:59] [PASSED] drm_test_connector_hdmi_init_null_vendor
[03:10:59] [PASSED] drm_test_connector_hdmi_init_product_length_exact
[03:10:59] [PASSED] drm_test_connector_hdmi_init_product_length_too_long
[03:10:59] [PASSED] drm_test_connector_hdmi_init_product_valid
[03:10:59] [PASSED] drm_test_connector_hdmi_init_vendor_length_exact
[03:10:59] [PASSED] drm_test_connector_hdmi_init_vendor_length_too_long
[03:10:59] [PASSED] drm_test_connector_hdmi_init_vendor_valid
[03:10:59] ========= drm_test_connector_hdmi_init_type_valid  =========
[03:10:59] [PASSED] HDMI-A
[03:10:59] [PASSED] HDMI-B
[03:10:59] ===== [PASSED] drm_test_connector_hdmi_init_type_valid =====
[03:10:59] ======== drm_test_connector_hdmi_init_type_invalid  ========
[03:10:59] [PASSED] Unknown
[03:10:59] [PASSED] VGA
[03:10:59] [PASSED] DVI-I
[03:10:59] [PASSED] DVI-D
[03:10:59] [PASSED] DVI-A
[03:10:59] [PASSED] Composite
[03:10:59] [PASSED] SVIDEO
[03:10:59] [PASSED] LVDS
[03:10:59] [PASSED] Component
[03:10:59] [PASSED] DIN
[03:10:59] [PASSED] DP
[03:10:59] [PASSED] TV
[03:10:59] [PASSED] eDP
[03:10:59] [PASSED] Virtual
[03:10:59] [PASSED] DSI
[03:10:59] [PASSED] DPI
[03:10:59] [PASSED] Writeback
[03:10:59] [PASSED] SPI
[03:10:59] [PASSED] USB
[03:10:59] ==== [PASSED] drm_test_connector_hdmi_init_type_invalid ====
[03:10:59] ============ [PASSED] drmm_connector_hdmi_init =============
[03:10:59] ============= drmm_connector_init (3 subtests) =============
[03:10:59] [PASSED] drm_test_drmm_connector_init
[03:10:59] [PASSED] drm_test_drmm_connector_init_null_ddc
[03:10:59] ========= drm_test_drmm_connector_init_type_valid  =========
[03:10:59] [PASSED] Unknown
[03:10:59] [PASSED] VGA
[03:10:59] [PASSED] DVI-I
[03:10:59] [PASSED] DVI-D
[03:10:59] [PASSED] DVI-A
[03:10:59] [PASSED] Composite
[03:10:59] [PASSED] SVIDEO
[03:10:59] [PASSED] LVDS
[03:10:59] [PASSED] Component
[03:10:59] [PASSED] DIN
[03:10:59] [PASSED] DP
[03:10:59] [PASSED] HDMI-A
[03:10:59] [PASSED] HDMI-B
[03:10:59] [PASSED] TV
[03:10:59] [PASSED] eDP
[03:10:59] [PASSED] Virtual
[03:10:59] [PASSED] DSI
[03:10:59] [PASSED] DPI
[03:10:59] [PASSED] Writeback
[03:10:59] [PASSED] SPI
[03:10:59] [PASSED] USB
[03:10:59] ===== [PASSED] drm_test_drmm_connector_init_type_valid =====
[03:10:59] =============== [PASSED] drmm_connector_init ===============
[03:10:59] = drm_connector_attach_broadcast_rgb_property (2 subtests) =
[03:10:59] [PASSED] drm_test_drm_connector_attach_broadcast_rgb_property
[03:10:59] [PASSED] drm_test_drm_connector_attach_broadcast_rgb_property_hdmi_connector
[03:10:59] === [PASSED] drm_connector_attach_broadcast_rgb_property ===
[03:10:59] ========== drm_get_tv_mode_from_name (2 subtests) ==========
[03:10:59] ========== drm_test_get_tv_mode_from_name_valid  ===========
[03:10:59] [PASSED] NTSC
[03:10:59] [PASSED] NTSC-443
[03:10:59] [PASSED] NTSC-J
[03:10:59] [PASSED] PAL
[03:10:59] [PASSED] PAL-M
[03:10:59] [PASSED] PAL-N
[03:10:59] [PASSED] SECAM
[03:10:59] [PASSED] Mono
[03:10:59] ====== [PASSED] drm_test_get_tv_mode_from_name_valid =======
[03:10:59] [PASSED] drm_test_get_tv_mode_from_name_truncated
[03:10:59] ============ [PASSED] drm_get_tv_mode_from_name ============
[03:10:59] = drm_test_connector_hdmi_compute_mode_clock (12 subtests) =
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb_10bpc
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb_10bpc_vic_1
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb_12bpc
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb_12bpc_vic_1
[03:10:59] [PASSED] drm_test_drm_hdmi_compute_mode_clock_rgb_double
[03:10:59] = drm_test_connector_hdmi_compute_mode_clock_yuv420_valid  =
[03:10:59] [PASSED] VIC 96
[03:10:59] [PASSED] VIC 97
[03:10:59] [PASSED] VIC 101
[03:10:59] [PASSED] VIC 102
[03:10:59] [PASSED] VIC 106
[03:10:59] [PASSED] VIC 107
[03:10:59] === [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv420_valid ===
[03:10:59] [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv420_10_bpc
[03:10:59] [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv420_12_bpc
[03:10:59] [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv422_8_bpc
[03:10:59] [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv422_10_bpc
[03:10:59] [PASSED] drm_test_connector_hdmi_compute_mode_clock_yuv422_12_bpc
[03:10:59] === [PASSED] drm_test_connector_hdmi_compute_mode_clock ====
[03:10:59] == drm_hdmi_connector_get_broadcast_rgb_name (2 subtests) ==
[03:10:59] === drm_test_drm_hdmi_connector_get_broadcast_rgb_name  ====
[03:10:59] [PASSED] Automatic
[03:10:59] [PASSED] Full
[03:10:59] [PASSED] Limited 16:235
[03:10:59] === [PASSED] drm_test_drm_hdmi_connector_get_broadcast_rgb_name ===
[03:10:59] [PASSED] drm_test_drm_hdmi_connector_get_broadcast_rgb_name_invalid
[03:10:59] ==== [PASSED] drm_hdmi_connector_get_broadcast_rgb_name ====
[03:10:59] == drm_hdmi_connector_get_output_format_name (2 subtests) ==
[03:10:59] === drm_test_drm_hdmi_connector_get_output_format_name  ====
[03:10:59] [PASSED] RGB
[03:10:59] [PASSED] YUV 4:2:0
[03:10:59] [PASSED] YUV 4:2:2
[03:10:59] [PASSED] YUV 4:4:4
[03:10:59] === [PASSED] drm_test_drm_hdmi_connector_get_output_format_name ===
[03:10:59] [PASSED] drm_test_drm_hdmi_connector_get_output_format_name_invalid
[03:10:59] ==== [PASSED] drm_hdmi_connector_get_output_format_name ====
[03:10:59] ============= drm_damage_helper (21 subtests) ==============
[03:10:59] [PASSED] drm_test_damage_iter_no_damage
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_fractional_src
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_src_moved
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_fractional_src_moved
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_not_visible
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_no_crtc
[03:10:59] [PASSED] drm_test_damage_iter_no_damage_no_fb
[03:10:59] [PASSED] drm_test_damage_iter_simple_damage
[03:10:59] [PASSED] drm_test_damage_iter_single_damage
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_intersect_src
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_outside_src
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_fractional_src
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_intersect_fractional_src
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_outside_fractional_src
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_src_moved
[03:10:59] [PASSED] drm_test_damage_iter_single_damage_fractional_src_moved
[03:10:59] [PASSED] drm_test_damage_iter_damage
[03:10:59] [PASSED] drm_test_damage_iter_damage_one_intersect
[03:10:59] [PASSED] drm_test_damage_iter_damage_one_outside
[03:10:59] [PASSED] drm_test_damage_iter_damage_src_moved
[03:10:59] [PASSED] drm_test_damage_iter_damage_not_visible
[03:10:59] ================ [PASSED] drm_damage_helper ================
[03:10:59] ============== drm_dp_mst_helper (3 subtests) ==============
[03:10:59] ============== drm_test_dp_mst_calc_pbn_mode  ==============
[03:10:59] [PASSED] Clock 154000 BPP 30 DSC disabled
[03:10:59] [PASSED] Clock 234000 BPP 30 DSC disabled
[03:10:59] [PASSED] Clock 297000 BPP 24 DSC disabled
[03:10:59] [PASSED] Clock 332880 BPP 24 DSC enabled
[03:10:59] [PASSED] Clock 324540 BPP 24 DSC enabled
[03:10:59] ========== [PASSED] drm_test_dp_mst_calc_pbn_mode ==========
[03:10:59] ============== drm_test_dp_mst_calc_pbn_div  ===============
[03:10:59] [PASSED] Link rate 2000000 lane count 4
[03:10:59] [PASSED] Link rate 2000000 lane count 2
[03:10:59] [PASSED] Link rate 2000000 lane count 1
[03:10:59] [PASSED] Link rate 1350000 lane count 4
[03:10:59] [PASSED] Link rate 1350000 lane count 2
[03:10:59] [PASSED] Link rate 1350000 lane count 1
[03:10:59] [PASSED] Link rate 1000000 lane count 4
[03:10:59] [PASSED] Link rate 1000000 lane count 2
[03:10:59] [PASSED] Link rate 1000000 lane count 1
[03:10:59] [PASSED] Link rate 810000 lane count 4
[03:10:59] [PASSED] Link rate 810000 lane count 2
[03:10:59] [PASSED] Link rate 810000 lane count 1
[03:10:59] [PASSED] Link rate 540000 lane count 4
[03:10:59] [PASSED] Link rate 540000 lane count 2
[03:10:59] [PASSED] Link rate 540000 lane count 1
[03:10:59] [PASSED] Link rate 270000 lane count 4
[03:10:59] [PASSED] Link rate 270000 lane count 2
[03:10:59] [PASSED] Link rate 270000 lane count 1
[03:10:59] [PASSED] Link rate 162000 lane count 4
[03:10:59] [PASSED] Link rate 162000 lane count 2
[03:10:59] [PASSED] Link rate 162000 lane count 1
[03:10:59] ========== [PASSED] drm_test_dp_mst_calc_pbn_div ===========
[03:10:59] ========= drm_test_dp_mst_sideband_msg_req_decode  =========
[03:10:59] [PASSED] DP_ENUM_PATH_RESOURCES with port number
[03:10:59] [PASSED] DP_POWER_UP_PHY with port number
[03:10:59] [PASSED] DP_POWER_DOWN_PHY with port number
[03:10:59] [PASSED] DP_ALLOCATE_PAYLOAD with SDP stream sinks
[03:10:59] [PASSED] DP_ALLOCATE_PAYLOAD with port number
[03:10:59] [PASSED] DP_ALLOCATE_PAYLOAD with VCPI
[03:10:59] [PASSED] DP_ALLOCATE_PAYLOAD with PBN
[03:10:59] [PASSED] DP_QUERY_PAYLOAD with port number
[03:10:59] [PASSED] DP_QUERY_PAYLOAD with VCPI
[03:10:59] [PASSED] DP_REMOTE_DPCD_READ with port number
[03:10:59] [PASSED] DP_REMOTE_DPCD_READ with DPCD address
[03:10:59] [PASSED] DP_REMOTE_DPCD_READ with max number of bytes
[03:10:59] [PASSED] DP_REMOTE_DPCD_WRITE with port number
[03:10:59] [PASSED] DP_REMOTE_DPCD_WRITE with DPCD address
[03:10:59] [PASSED] DP_REMOTE_DPCD_WRITE with data array
[03:10:59] [PASSED] DP_REMOTE_I2C_READ with port number
[03:10:59] [PASSED] DP_REMOTE_I2C_READ with I2C device ID
[03:10:59] [PASSED] DP_REMOTE_I2C_READ with transactions array
[03:10:59] [PASSED] DP_REMOTE_I2C_WRITE with port number
[03:10:59] [PASSED] DP_REMOTE_I2C_WRITE with I2C device ID
[03:10:59] [PASSED] DP_REMOTE_I2C_WRITE with data array
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with stream ID
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with client ID
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with stream event
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with valid stream event
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with stream behavior
[03:10:59] [PASSED] DP_QUERY_STREAM_ENC_STATUS with a valid stream behavior
[03:10:59] ===== [PASSED] drm_test_dp_mst_sideband_msg_req_decode =====
[03:10:59] ================ [PASSED] drm_dp_mst_helper ================
[03:10:59] ================== drm_exec (7 subtests) ===================
[03:10:59] [PASSED] sanitycheck
[03:10:59] [PASSED] test_lock
[03:10:59] [PASSED] test_lock_unlock
[03:10:59] [PASSED] test_duplicates
[03:10:59] [PASSED] test_prepare
[03:10:59] [PASSED] test_prepare_array
[03:10:59] [PASSED] test_multiple_loops
[03:10:59] ==================== [PASSED] drm_exec =====================
[03:10:59] =========== drm_format_helper_test (17 subtests) ===========
[03:10:59] ============== drm_test_fb_xrgb8888_to_gray8  ==============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ========== [PASSED] drm_test_fb_xrgb8888_to_gray8 ==========
[03:10:59] ============= drm_test_fb_xrgb8888_to_rgb332  ==============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ========= [PASSED] drm_test_fb_xrgb8888_to_rgb332 ==========
[03:10:59] ============= drm_test_fb_xrgb8888_to_rgb565  ==============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ========= [PASSED] drm_test_fb_xrgb8888_to_rgb565 ==========
[03:10:59] ============ drm_test_fb_xrgb8888_to_xrgb1555  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_xrgb1555 =========
[03:10:59] ============ drm_test_fb_xrgb8888_to_argb1555  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_argb1555 =========
[03:10:59] ============ drm_test_fb_xrgb8888_to_rgba5551  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_rgba5551 =========
[03:10:59] ============= drm_test_fb_xrgb8888_to_rgb888  ==============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ========= [PASSED] drm_test_fb_xrgb8888_to_rgb888 ==========
[03:10:59] ============ drm_test_fb_xrgb8888_to_argb8888  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_argb8888 =========
[03:10:59] =========== drm_test_fb_xrgb8888_to_xrgb2101010  ===========
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======= [PASSED] drm_test_fb_xrgb8888_to_xrgb2101010 =======
[03:10:59] =========== drm_test_fb_xrgb8888_to_argb2101010  ===========
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======= [PASSED] drm_test_fb_xrgb8888_to_argb2101010 =======
[03:10:59] ============== drm_test_fb_xrgb8888_to_mono  ===============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ========== [PASSED] drm_test_fb_xrgb8888_to_mono ===========
[03:10:59] ==================== drm_test_fb_swab  =====================
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ================ [PASSED] drm_test_fb_swab =================
[03:10:59] ============ drm_test_fb_xrgb8888_to_xbgr8888  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_xbgr8888 =========
[03:10:59] ============ drm_test_fb_xrgb8888_to_abgr8888  =============
[03:10:59] [PASSED] single_pixel_source_buffer
[03:10:59] [PASSED] single_pixel_clip_rectangle
[03:10:59] [PASSED] well_known_colors
[03:10:59] [PASSED] destination_pitch
[03:10:59] ======== [PASSED] drm_test_fb_xrgb8888_to_abgr8888 =========
[03:10:59] ================= drm_test_fb_clip_offset  =================
[03:10:59] [PASSED] pass through
[03:10:59] [PASSED] horizontal offset
[03:10:59] [PASSED] vertical offset
[03:10:59] [PASSED] horizontal and vertical offset
[03:10:59] [PASSED] horizontal offset (custom pitch)
[03:10:59] [PASSED] vertical offset (custom pitch)
[03:10:59] [PASSED] horizontal and vertical offset (custom pitch)
[03:10:59] ============= [PASSED] drm_test_fb_clip_offset =============
[03:10:59] ============== drm_test_fb_build_fourcc_list  ==============
[03:10:59] [PASSED] no native formats
[03:10:59] [PASSED] XRGB8888 as native format
[03:10:59] [PASSED] remove duplicates
[03:10:59] [PASSED] convert alpha formats
[03:10:59] [PASSED] random formats
[03:10:59] ========== [PASSED] drm_test_fb_build_fourcc_list ==========
[03:10:59] =================== drm_test_fb_memcpy  ====================
[03:10:59] [PASSED] single_pixel_source_buffer: XR24 little-endian (0x34325258)
[03:10:59] [PASSED] single_pixel_source_buffer: XRA8 little-endian (0x38415258)
[03:10:59] [PASSED] single_pixel_source_buffer: YU24 little-endian (0x34325559)
[03:10:59] [PASSED] single_pixel_clip_rectangle: XB24 little-endian (0x34324258)
[03:10:59] [PASSED] single_pixel_clip_rectangle: XRA8 little-endian (0x38415258)
[03:10:59] [PASSED] single_pixel_clip_rectangle: YU24 little-endian (0x34325559)
[03:10:59] [PASSED] well_known_colors: XB24 little-endian (0x34324258)
[03:10:59] [PASSED] well_known_colors: XRA8 little-endian (0x38415258)
[03:10:59] [PASSED] well_known_colors: YU24 little-endian (0x34325559)
[03:10:59] [PASSED] destination_pitch: XB24 little-endian (0x34324258)
[03:10:59] [PASSED] destination_pitch: XRA8 little-endian (0x38415258)
[03:10:59] [PASSED] destination_pitch: YU24 little-endian (0x34325559)
[03:10:59] =============== [PASSED] drm_test_fb_memcpy ================
[03:10:59] ============= [PASSED] drm_format_helper_test ==============
[03:10:59] ================= drm_format (18 subtests) =================
[03:10:59] [PASSED] drm_test_format_block_width_invalid
[03:10:59] [PASSED] drm_test_format_block_width_one_plane
[03:10:59] [PASSED] drm_test_format_block_width_two_plane
[03:10:59] [PASSED] drm_test_format_block_width_three_plane
[03:10:59] [PASSED] drm_test_format_block_width_tiled
[03:10:59] [PASSED] drm_test_format_block_height_invalid
[03:10:59] [PASSED] drm_test_format_block_height_one_plane
[03:10:59] [PASSED] drm_test_format_block_height_two_plane
[03:10:59] [PASSED] drm_test_format_block_height_three_plane
[03:10:59] [PASSED] drm_test_format_block_height_tiled
[03:10:59] [PASSED] drm_test_format_min_pitch_invalid
[03:10:59] [PASSED] drm_test_format_min_pitch_one_plane_8bpp
[03:10:59] [PASSED] drm_test_format_min_pitch_one_plane_16bpp
[03:10:59] [PASSED] drm_test_format_min_pitch_one_plane_24bpp
[03:10:59] [PASSED] drm_test_format_min_pitch_one_plane_32bpp
[03:10:59] [PASSED] drm_test_format_min_pitch_two_plane
[03:10:59] [PASSED] drm_test_format_min_pitch_three_plane_8bpp
[03:10:59] [PASSED] drm_test_format_min_pitch_tiled
[03:10:59] =================== [PASSED] drm_format ====================
[03:10:59] =============== drm_framebuffer (1 subtest) ================
[03:10:59] =============== drm_test_framebuffer_create  ===============
[03:10:59] [PASSED] ABGR8888 normal sizes
[03:10:59] [PASSED] ABGR8888 max sizes
[03:10:59] [PASSED] ABGR8888 pitch greater than min required
[03:10:59] [PASSED] ABGR8888 pitch less than min required
[03:10:59] [PASSED] ABGR8888 Invalid width
[03:10:59] [PASSED] ABGR8888 Invalid buffer handle
[03:10:59] [PASSED] No pixel format
[03:10:59] [PASSED] ABGR8888 Width 0
[03:10:59] [PASSED] ABGR8888 Height 0
[03:10:59] [PASSED] ABGR8888 Out of bound height * pitch combination
[03:10:59] [PASSED] ABGR8888 Large buffer offset
[03:10:59] [PASSED] ABGR8888 Set DRM_MODE_FB_MODIFIERS without modifiers
[03:10:59] [PASSED] ABGR8888 Valid buffer modifier
[03:10:59] [PASSED] ABGR8888 Invalid buffer modifier(DRM_FORMAT_MOD_SAMSUNG_64_32_TILE)
[03:10:59] [PASSED] ABGR8888 Extra pitches without DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] ABGR8888 Extra pitches with DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] NV12 Normal sizes
[03:10:59] [PASSED] NV12 Max sizes
[03:10:59] [PASSED] NV12 Invalid pitch
[03:10:59] [PASSED] NV12 Invalid modifier/missing DRM_MODE_FB_MODIFIERS flag
[03:10:59] [PASSED] NV12 different  modifier per-plane
[03:10:59] [PASSED] NV12 with DRM_FORMAT_MOD_SAMSUNG_64_32_TILE
[03:10:59] [PASSED] NV12 Valid modifiers without DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] NV12 Modifier for inexistent plane
[03:10:59] [PASSED] NV12 Handle for inexistent plane
[03:10:59] [PASSED] NV12 Handle for inexistent plane without DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] YVU420 DRM_MODE_FB_MODIFIERS set without modifier
[03:10:59] [PASSED] YVU420 Normal sizes
[03:10:59] [PASSED] YVU420 Max sizes
[03:10:59] [PASSED] YVU420 Invalid pitch
[03:10:59] [PASSED] YVU420 Different pitches
[03:10:59] [PASSED] YVU420 Different buffer offsets/pitches
[03:10:59] [PASSED] YVU420 Modifier set just for plane 0, without DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] YVU420 Modifier set just for planes 0, 1, without DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] YVU420 Modifier set just for plane 0, 1, with DRM_MODE_FB_MODIFIERS
[03:10:59] [PASSED] YVU420 Valid modifier
[03:10:59] [PASSED] YVU420 Different modifiers per plane
[03:10:59] [PASSED] YVU420 Modifier for inexistent plane
[03:10:59] [PASSED] X0L2 Normal sizes
[03:10:59] [PASSED] X0L2 Max sizes
[03:10:59] [PASSED] X0L2 Invalid pitch
[03:10:59] [PASSED] X0L2 Pitch greater than minimum required
[03:10:59] [PASSED] X0L2 Handle for inexistent plane
[03:10:59] [PASSED] X0L2 Offset for inexistent plane, without DRM_MODE_FB_MODIFIERS set
[03:10:59] [PASSED] X0L2 Modifier without DRM_MODE_FB_MODIFIERS set
[03:10:59] [PASSED] X0L2 Valid modifier
[03:10:59] [PASSED] X0L2 Modifier for inexistent plane
[03:10:59] =========== [PASSED] drm_test_framebuffer_create ===========
[03:10:59] ================= [PASSED] drm_framebuffer =================
[03:10:59] ================ drm_gem_shmem (8 subtests) ================
[03:10:59] [PASSED] drm_gem_shmem_test_obj_create
[03:10:59] [PASSED] drm_gem_shmem_test_obj_create_private
[03:10:59] [PASSED] drm_gem_shmem_test_pin_pages
[03:10:59] [PASSED] drm_gem_shmem_test_vmap
[03:10:59] [PASSED] drm_gem_shmem_test_get_pages_sgt
[03:10:59] [PASSED] drm_gem_shmem_test_get_sg_table
[03:10:59] [PASSED] drm_gem_shmem_test_madvise
[03:10:59] [PASSED] drm_gem_shmem_test_purge
[03:10:59] ================== [PASSED] drm_gem_shmem ==================
[03:10:59] === drm_atomic_helper_connector_hdmi_check (22 subtests) ===
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_auto_cea_mode
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_auto_cea_mode_vic_1
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_full_cea_mode
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_full_cea_mode_vic_1
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_limited_cea_mode
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_limited_cea_mode_vic_1
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_crtc_mode_changed
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_crtc_mode_not_changed
[03:10:59] [PASSED] drm_test_check_hdmi_funcs_reject_rate
[03:10:59] [PASSED] drm_test_check_max_tmds_rate_bpc_fallback
[03:10:59] [PASSED] drm_test_check_max_tmds_rate_format_fallback
[03:10:59] [PASSED] drm_test_check_output_bpc_crtc_mode_changed
[03:10:59] [PASSED] drm_test_check_output_bpc_crtc_mode_not_changed
[03:10:59] [PASSED] drm_test_check_output_bpc_dvi
[03:10:59] [PASSED] drm_test_check_output_bpc_format_vic_1
[03:10:59] [PASSED] drm_test_check_output_bpc_format_display_8bpc_only
[03:10:59] [PASSED] drm_test_check_output_bpc_format_display_rgb_only
[03:10:59] [PASSED] drm_test_check_output_bpc_format_driver_8bpc_only
[03:10:59] [PASSED] drm_test_check_output_bpc_format_driver_rgb_only
[03:10:59] [PASSED] drm_test_check_tmds_char_rate_rgb_8bpc
[03:10:59] [PASSED] drm_test_check_tmds_char_rate_rgb_10bpc
[03:10:59] [PASSED] drm_test_check_tmds_char_rate_rgb_12bpc
[03:10:59] ===== [PASSED] drm_atomic_helper_connector_hdmi_check ======
[03:10:59] === drm_atomic_helper_connector_hdmi_reset (6 subtests) ====
[03:10:59] [PASSED] drm_test_check_broadcast_rgb_value
[03:10:59] [PASSED] drm_test_check_bpc_8_value
[03:10:59] [PASSED] drm_test_check_bpc_10_value
[03:10:59] [PASSED] drm_test_check_bpc_12_value
[03:10:59] [PASSED] drm_test_check_format_value
[03:10:59] [PASSED] drm_test_check_tmds_char_value
[03:10:59] ===== [PASSED] drm_atomic_helper_connector_hdmi_reset ======
[03:10:59] ================= drm_managed (2 subtests) =================
[03:10:59] [PASSED] drm_test_managed_release_action
[03:10:59] [PASSED] drm_test_managed_run_action
[03:10:59] =================== [PASSED] drm_managed ===================
[03:10:59] =================== drm_mm (6 subtests) ====================
[03:10:59] [PASSED] drm_test_mm_init
[03:10:59] [PASSED] drm_test_mm_debug
[03:10:59] [PASSED] drm_test_mm_align32
[03:10:59] [PASSED] drm_test_mm_align64
[03:10:59] [PASSED] drm_test_mm_lowest
[03:10:59] [PASSED] drm_test_mm_highest
[03:10:59] ===================== [PASSED] drm_mm ======================
[03:10:59] ============= drm_modes_analog_tv (5 subtests) =============
[03:10:59] [PASSED] drm_test_modes_analog_tv_mono_576i
[03:10:59] [PASSED] drm_test_modes_analog_tv_ntsc_480i
[03:10:59] [PASSED] drm_test_modes_analog_tv_ntsc_480i_inlined
[03:10:59] [PASSED] drm_test_modes_analog_tv_pal_576i
[03:10:59] [PASSED] drm_test_modes_analog_tv_pal_576i_inlined
[03:10:59] =============== [PASSED] drm_modes_analog_tv ===============
[03:10:59] ============== drm_plane_helper (2 subtests) ===============
[03:10:59] =============== drm_test_check_plane_state  ================
[03:10:59] [PASSED] clipping_simple
[03:10:59] [PASSED] clipping_rotate_reflect
[03:10:59] [PASSED] positioning_simple
[03:10:59] [PASSED] upscaling
[03:10:59] [PASSED] downscaling
[03:10:59] [PASSED] rounding1
[03:10:59] [PASSED] rounding2
[03:10:59] [PASSED] rounding3
[03:10:59] [PASSED] rounding4
[03:10:59] =========== [PASSED] drm_test_check_plane_state ============
[03:10:59] =========== drm_test_check_invalid_plane_state  ============
[03:10:59] [PASSED] positioning_invalid
[03:10:59] [PASSED] upscaling_invalid
stty: 'standard input': Inappropriate ioctl for device
[03:10:59] [PASSED] downscaling_invalid
[03:10:59] ======= [PASSED] drm_test_check_invalid_plane_state ========
[03:10:59] ================ [PASSED] drm_plane_helper =================
[03:10:59] ====== drm_connector_helper_tv_get_modes (1 subtest) =======
[03:10:59] ====== drm_test_connector_helper_tv_get_modes_check  =======
[03:10:59] [PASSED] None
[03:10:59] [PASSED] PAL
[03:10:59] [PASSED] NTSC
[03:10:59] [PASSED] Both, NTSC Default
[03:10:59] [PASSED] Both, PAL Default
[03:10:59] [PASSED] Both, NTSC Default, with PAL on command-line
[03:10:59] [PASSED] Both, PAL Default, with NTSC on command-line
[03:10:59] == [PASSED] drm_test_connector_helper_tv_get_modes_check ===
[03:10:59] ======== [PASSED] drm_connector_helper_tv_get_modes ========
[03:10:59] ================== drm_rect (9 subtests) ===================
[03:10:59] [PASSED] drm_test_rect_clip_scaled_div_by_zero
[03:10:59] [PASSED] drm_test_rect_clip_scaled_not_clipped
[03:10:59] [PASSED] drm_test_rect_clip_scaled_clipped
[03:10:59] [PASSED] drm_test_rect_clip_scaled_signed_vs_unsigned
[03:10:59] ================= drm_test_rect_intersect  =================
[03:10:59] [PASSED] top-left x bottom-right: 2x2+1+1 x 2x2+0+0
[03:10:59] [PASSED] top-right x bottom-left: 2x2+0+0 x 2x2+1-1
[03:10:59] [PASSED] bottom-left x top-right: 2x2+1-1 x 2x2+0+0
[03:10:59] [PASSED] bottom-right x top-left: 2x2+0+0 x 2x2+1+1
[03:10:59] [PASSED] right x left: 2x1+0+0 x 3x1+1+0
[03:10:59] [PASSED] left x right: 3x1+1+0 x 2x1+0+0
[03:10:59] [PASSED] up x bottom: 1x2+0+0 x 1x3+0-1
[03:10:59] [PASSED] bottom x up: 1x3+0-1 x 1x2+0+0
[03:10:59] [PASSED] touching corner: 1x1+0+0 x 2x2+1+1
[03:10:59] [PASSED] touching side: 1x1+0+0 x 1x1+1+0
[03:10:59] [PASSED] equal rects: 2x2+0+0 x 2x2+0+0
[03:10:59] [PASSED] inside another: 2x2+0+0 x 1x1+1+1
[03:10:59] [PASSED] far away: 1x1+0+0 x 1x1+3+6
[03:10:59] [PASSED] points intersecting: 0x0+5+10 x 0x0+5+10
[03:10:59] [PASSED] points not intersecting: 0x0+0+0 x 0x0+5+10
[03:10:59] ============= [PASSED] drm_test_rect_intersect =============
[03:10:59] ================ drm_test_rect_calc_hscale  ================
[03:10:59] [PASSED] normal use
[03:10:59] [PASSED] out of max range
[03:10:59] [PASSED] out of min range
[03:10:59] [PASSED] zero dst
[03:10:59] [PASSED] negative src
[03:10:59] [PASSED] negative dst
[03:10:59] ============ [PASSED] drm_test_rect_calc_hscale ============
[03:10:59] ================ drm_test_rect_calc_vscale  ================
[03:10:59] [PASSED] normal use
[03:10:59] [PASSED] out of max range
[03:10:59] [PASSED] out of min range
[03:10:59] [PASSED] zero dst
[03:10:59] [PASSED] negative src
[03:10:59] [PASSED] negative dst
[03:10:59] ============ [PASSED] drm_test_rect_calc_vscale ============
[03:10:59] ================== drm_test_rect_rotate  ===================
[03:10:59] [PASSED] reflect-x
[03:10:59] [PASSED] reflect-y
[03:10:59] [PASSED] rotate-0
[03:10:59] [PASSED] rotate-90
[03:10:59] [PASSED] rotate-180
[03:10:59] [PASSED] rotate-270
[03:10:59] ============== [PASSED] drm_test_rect_rotate ===============
[03:10:59] ================ drm_test_rect_rotate_inv  =================
[03:10:59] [PASSED] reflect-x
[03:10:59] [PASSED] reflect-y
[03:10:59] [PASSED] rotate-0
[03:10:59] [PASSED] rotate-90
[03:10:59] [PASSED] rotate-180
[03:10:59] [PASSED] rotate-270
[03:10:59] ============ [PASSED] drm_test_rect_rotate_inv =============
[03:10:59] ==================== [PASSED] drm_rect =====================
[03:10:59] ============================================================
[03:10:59] Testing complete. Ran 515 tests: passed: 515
[03:10:59] Elapsed time: 23.348s total, 1.753s configuring, 21.426s building, 0.152s running

+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/ttm/tests/.kunitconfig
[03:10:59] Configuring KUnit Kernel ...
Regenerating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
[03:11:00] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
[03:11:09] Starting KUnit Kernel (1/1)...
[03:11:09] ============================================================
Running tests with:
$ .kunit/linux kunit.enable=1 mem=1G console=tty kunit_shutdown=halt
[03:11:09] ================= ttm_device (5 subtests) ==================
[03:11:09] [PASSED] ttm_device_init_basic
[03:11:09] [PASSED] ttm_device_init_multiple
[03:11:09] [PASSED] ttm_device_fini_basic
[03:11:09] [PASSED] ttm_device_init_no_vma_man
[03:11:09] ================== ttm_device_init_pools  ==================
[03:11:09] [PASSED] No DMA allocations, no DMA32 required
[03:11:09] [PASSED] DMA allocations, DMA32 required
[03:11:09] [PASSED] No DMA allocations, DMA32 required
[03:11:09] [PASSED] DMA allocations, no DMA32 required
[03:11:09] ============== [PASSED] ttm_device_init_pools ==============
[03:11:09] =================== [PASSED] ttm_device ====================
[03:11:09] ================== ttm_pool (8 subtests) ===================
[03:11:09] ================== ttm_pool_alloc_basic  ===================
[03:11:09] [PASSED] One page
[03:11:09] [PASSED] More than one page
[03:11:09] [PASSED] Above the allocation limit
[03:11:09] [PASSED] One page, with coherent DMA mappings enabled
[03:11:09] [PASSED] Above the allocation limit, with coherent DMA mappings enabled
[03:11:09] ============== [PASSED] ttm_pool_alloc_basic ===============
[03:11:09] ============== ttm_pool_alloc_basic_dma_addr  ==============
[03:11:09] [PASSED] One page
[03:11:09] [PASSED] More than one page
[03:11:09] [PASSED] Above the allocation limit
[03:11:09] [PASSED] One page, with coherent DMA mappings enabled
[03:11:09] [PASSED] Above the allocation limit, with coherent DMA mappings enabled
[03:11:09] ========== [PASSED] ttm_pool_alloc_basic_dma_addr ==========
[03:11:09] [PASSED] ttm_pool_alloc_order_caching_match
[03:11:09] [PASSED] ttm_pool_alloc_caching_mismatch
[03:11:09] [PASSED] ttm_pool_alloc_order_mismatch
[03:11:09] [PASSED] ttm_pool_free_dma_alloc
[03:11:09] [PASSED] ttm_pool_free_no_dma_alloc
[03:11:09] [PASSED] ttm_pool_fini_basic
[03:11:09] ==================== [PASSED] ttm_pool =====================
[03:11:09] ================ ttm_resource (8 subtests) =================
[03:11:09] ================= ttm_resource_init_basic  =================
[03:11:09] [PASSED] Init resource in TTM_PL_SYSTEM
[03:11:09] [PASSED] Init resource in TTM_PL_VRAM
[03:11:09] [PASSED] Init resource in a private placement
[03:11:09] [PASSED] Init resource in TTM_PL_SYSTEM, set placement flags
[03:11:09] ============= [PASSED] ttm_resource_init_basic =============
[03:11:09] [PASSED] ttm_resource_init_pinned
[03:11:09] [PASSED] ttm_resource_fini_basic
[03:11:09] [PASSED] ttm_resource_manager_init_basic
[03:11:09] [PASSED] ttm_resource_manager_usage_basic
[03:11:09] [PASSED] ttm_resource_manager_set_used_basic
[03:11:09] [PASSED] ttm_sys_man_alloc_basic
[03:11:09] [PASSED] ttm_sys_man_free_basic
[03:11:09] ================== [PASSED] ttm_resource ===================
[03:11:09] =================== ttm_tt (15 subtests) ===================
[03:11:09] ==================== ttm_tt_init_basic  ====================
[03:11:09] [PASSED] Page-aligned size
[03:11:09] [PASSED] Extra pages requested
[03:11:09] ================ [PASSED] ttm_tt_init_basic ================
[03:11:09] [PASSED] ttm_tt_init_misaligned
[03:11:09] [PASSED] ttm_tt_fini_basic
[03:11:09] [PASSED] ttm_tt_fini_sg
[03:11:09] [PASSED] ttm_tt_fini_shmem
[03:11:09] [PASSED] ttm_tt_create_basic
[03:11:09] [PASSED] ttm_tt_create_invalid_bo_type
[03:11:09] [PASSED] ttm_tt_create_ttm_exists
[03:11:09] [PASSED] ttm_tt_create_failed
[03:11:09] [PASSED] ttm_tt_destroy_basic
[03:11:09] [PASSED] ttm_tt_populate_null_ttm
[03:11:09] [PASSED] ttm_tt_populate_populated_ttm
[03:11:09] [PASSED] ttm_tt_unpopulate_basic
[03:11:09] [PASSED] ttm_tt_unpopulate_empty_ttm
[03:11:09] [PASSED] ttm_tt_swapin_basic
[03:11:09] ===================== [PASSED] ttm_tt ======================
[03:11:09] =================== ttm_bo (14 subtests) ===================
[03:11:09] =========== ttm_bo_reserve_optimistic_no_ticket  ===========
[03:11:09] [PASSED] Cannot be interrupted and sleeps
[03:11:09] [PASSED] Cannot be interrupted, locks straight away
[03:11:09] [PASSED] Can be interrupted, sleeps
[03:11:09] ======= [PASSED] ttm_bo_reserve_optimistic_no_ticket =======
[03:11:09] [PASSED] ttm_bo_reserve_locked_no_sleep
[03:11:09] [PASSED] ttm_bo_reserve_no_wait_ticket
[03:11:10] [PASSED] ttm_bo_reserve_double_resv
[03:11:10] [PASSED] ttm_bo_reserve_interrupted
[03:11:10] [PASSED] ttm_bo_reserve_deadlock
[03:11:10] [PASSED] ttm_bo_unreserve_basic
[03:11:10] [PASSED] ttm_bo_unreserve_pinned
[03:11:10] [PASSED] ttm_bo_unreserve_bulk
[03:11:10] [PASSED] ttm_bo_put_basic
[03:11:10] [PASSED] ttm_bo_put_shared_resv
[03:11:10] [PASSED] ttm_bo_pin_basic
[03:11:10] [PASSED] ttm_bo_pin_unpin_resource
[03:11:10] [PASSED] ttm_bo_multiple_pin_one_unpin
[03:11:10] ===================== [PASSED] ttm_bo ======================
[03:11:10] ============== ttm_bo_validate (22 subtests) ===============
[03:11:10] ============== ttm_bo_init_reserved_sys_man  ===============
[03:11:10] [PASSED] Buffer object for userspace
[03:11:10] [PASSED] Kernel buffer object
[03:11:10] [PASSED] Shared buffer object
[03:11:10] ========== [PASSED] ttm_bo_init_reserved_sys_man ===========
[03:11:10] ============== ttm_bo_init_reserved_mock_man  ==============
[03:11:10] [PASSED] Buffer object for userspace
[03:11:10] [PASSED] Kernel buffer object
[03:11:10] [PASSED] Shared buffer object
[03:11:10] ========== [PASSED] ttm_bo_init_reserved_mock_man ==========
[03:11:10] [PASSED] ttm_bo_init_reserved_resv
[03:11:10] ================== ttm_bo_validate_basic  ==================
[03:11:10] [PASSED] Buffer object for userspace
[03:11:10] [PASSED] Kernel buffer object
[03:11:10] [PASSED] Shared buffer object
[03:11:10] ============== [PASSED] ttm_bo_validate_basic ==============
[03:11:10] [PASSED] ttm_bo_validate_invalid_placement
[03:11:10] ============= ttm_bo_validate_same_placement  ==============
[03:11:10] [PASSED] System manager
[03:11:10] [PASSED] VRAM manager
[03:11:10] ========= [PASSED] ttm_bo_validate_same_placement ==========
[03:11:10] [PASSED] ttm_bo_validate_failed_alloc
[03:11:10] [PASSED] ttm_bo_validate_pinned
[03:11:10] [PASSED] ttm_bo_validate_busy_placement
[03:11:10] ================ ttm_bo_validate_multihop  =================
[03:11:10] [PASSED] Buffer object for userspace
[03:11:10] [PASSED] Kernel buffer object
[03:11:10] [PASSED] Shared buffer object
[03:11:10] ============ [PASSED] ttm_bo_validate_multihop =============
[03:11:10] ========== ttm_bo_validate_no_placement_signaled  ==========
[03:11:10] [PASSED] Buffer object in system domain, no page vector
[03:11:10] [PASSED] Buffer object in system domain with an existing page vector
[03:11:10] ====== [PASSED] ttm_bo_validate_no_placement_signaled ======
[03:11:10] ======== ttm_bo_validate_no_placement_not_signaled  ========
[03:11:10] [PASSED] Buffer object for userspace
[03:11:10] [PASSED] Kernel buffer object
[03:11:10] [PASSED] Shared buffer object
[03:11:10] ==== [PASSED] ttm_bo_validate_no_placement_not_signaled ====
[03:11:10] [PASSED] ttm_bo_validate_move_fence_signaled
[03:11:10] ========= ttm_bo_validate_move_fence_not_signaled  =========
[03:11:10] [PASSED] Waits for GPU
[03:11:10] [PASSED] Tries to lock straight away
[03:11:10] ===== [PASSED] ttm_bo_validate_move_fence_not_signaled =====
[03:11:10] [PASSED] ttm_bo_validate_swapout
[03:11:10] [PASSED] ttm_bo_validate_happy_evict
[03:11:10] [PASSED] ttm_bo_validate_all_pinned_evict
[03:11:10] [PASSED] ttm_bo_validate_allowed_only_evict
[03:11:10] [PASSED] ttm_bo_validate_deleted_evict
[03:11:10] [PASSED] ttm_bo_validate_busy_domain_evict
[03:11:10] [PASSED] ttm_bo_validate_evict_gutting
[03:11:10] [PASSED] ttm_bo_validate_recrusive_evict
stty: 'standard input': Inappropriate ioctl for device
[03:11:10] ================= [PASSED] ttm_bo_validate =================
[03:11:10] ============================================================
[03:11:10] Testing complete. Ran 102 tests: passed: 102
[03:11:10] Elapsed time: 11.284s total, 1.715s configuring, 8.999s building, 0.509s running

+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel

✓ CI.Build: success for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : success

== Summary ==

lib/modules/6.11.0-rc7-xe/kernel/sound/core/seq/
lib/modules/6.11.0-rc7-xe/kernel/sound/core/seq/snd-seq.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd-seq-device.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd-hwdep.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd-pcm.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd-compress.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/core/snd-timer.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soundcore.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/atom/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/atom/snd-soc-sst-atom-hifi2-platform.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/atom/sst/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/atom/sst/snd-intel-sst-acpi.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/atom/sst/snd-intel-sst-core.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/common/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/intel/common/snd-soc-acpi-intel-match.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/amd/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/amd/snd-acp-config.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-pci-intel-tgl.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-intel-hda-mlink.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-pci-intel-cnl.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-pci-intel-lnl.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-intel-hda-common.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-intel-hda-generic.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-intel-hda.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/intel/snd-sof-pci-intel-mtl.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/amd/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/amd/snd-sof-amd-renoir.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/amd/snd-sof-amd-acp.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/snd-sof-utils.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/snd-sof-pci.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/snd-sof.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/snd-sof-probes.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/xtensa/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/sof/xtensa/snd-sof-xtensa-dsp.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/snd-soc-core.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/snd-soc-acpi.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/codecs/
lib/modules/6.11.0-rc7-xe/kernel/sound/soc/codecs/snd-soc-hdac-hda.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/snd-intel-sdw-acpi.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/ext/
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/ext/snd-hda-ext-core.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/snd-intel-dspcfg.ko
lib/modules/6.11.0-rc7-xe/kernel/sound/hda/snd-hda-core.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kernel/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kernel/msr.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kernel/cpuid.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/sha512-ssse3.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/crct10dif-pclmul.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/ghash-clmulni-intel.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/sha1-ssse3.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/crc32-pclmul.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/sha256-ssse3.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/aesni-intel.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/crypto/polyval-clmulni.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/events/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/events/intel/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/events/intel/intel-cstate.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/events/rapl.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kvm/
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kvm/kvm.ko
lib/modules/6.11.0-rc7-xe/kernel/arch/x86/kvm/kvm-intel.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/
lib/modules/6.11.0-rc7-xe/kernel/crypto/crypto_simd.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/cmac.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/ccm.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/cryptd.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/polyval-generic.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/async_xor.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/async_tx.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/async_memcpy.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/async_pq.ko
lib/modules/6.11.0-rc7-xe/kernel/crypto/async_tx/async_raid6_recov.ko
lib/modules/6.11.0-rc7-xe/build
lib/modules/6.11.0-rc7-xe/modules.alias.bin
lib/modules/6.11.0-rc7-xe/modules.builtin
lib/modules/6.11.0-rc7-xe/modules.softdep
lib/modules/6.11.0-rc7-xe/modules.alias
lib/modules/6.11.0-rc7-xe/modules.order
lib/modules/6.11.0-rc7-xe/modules.symbols
lib/modules/6.11.0-rc7-xe/modules.dep.bin
+ mv kernel-nodebug.tar.gz ..
+ cd ..
+ rm -rf archive
++ date +%s
^[[0Ksection_end:1726024974:package_x86_64_nodebug
^[[0K
+ echo -e '\e[0Ksection_end:1726024974:package_x86_64_nodebug\r\e[0K'
+ sync
+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel

✓ CI.Hooks: success for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : success

== Summary ==

run-parts: executing /workspace/ci/hooks/00-showenv
+ export
+ grep -Ei '(^|\W)CI_'
declare -x CI_KERNEL_BUILD_DIR="/workspace/kernel/build64-default"
declare -x CI_KERNEL_SRC_DIR="/workspace/kernel"
declare -x CI_TOOLS_SRC_DIR="/workspace/ci"
declare -x CI_WORKSPACE_DIR="/workspace"
run-parts: executing /workspace/ci/hooks/10-build-W1
+ SRC_DIR=/workspace/kernel
+ RESTORE_DISPLAY_CONFIG=0
+ '[' -n /workspace/kernel/build64-default ']'
+ BUILD_DIR=/workspace/kernel/build64-default
+ cd /workspace/kernel
++ nproc
+ make -j48 O=/workspace/kernel/build64-default modules_prepare
make[1]: Entering directory '/workspace/kernel/build64-default'
  GEN     Makefile
  UPD     include/generated/compile.h
  UPD     include/config/kernel.release
mkdir -p /workspace/kernel/build64-default/tools/objtool && make O=/workspace/kernel/build64-default subdir=tools/objtool --no-print-directory -C objtool 
  UPD     include/generated/utsrelease.h
  CALL    ../scripts/checksyscalls.sh
  HOSTCC  /workspace/kernel/build64-default/tools/objtool/fixdep.o
  HOSTLD  /workspace/kernel/build64-default/tools/objtool/fixdep-in.o
  LINK    /workspace/kernel/build64-default/tools/objtool/fixdep
  INSTALL libsubcmd_headers
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/exec-cmd.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/help.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/pager.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/parse-options.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/run-command.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/subcmd-config.o
  CC      /workspace/kernel/build64-default/tools/objtool/libsubcmd/sigchain.o
  LD      /workspace/kernel/build64-default/tools/objtool/libsubcmd/libsubcmd-in.o
  AR      /workspace/kernel/build64-default/tools/objtool/libsubcmd/libsubcmd.a
  CC      /workspace/kernel/build64-default/tools/objtool/weak.o
  CC      /workspace/kernel/build64-default/tools/objtool/check.o
  CC      /workspace/kernel/build64-default/tools/objtool/special.o
  CC      /workspace/kernel/build64-default/tools/objtool/builtin-check.o
  CC      /workspace/kernel/build64-default/tools/objtool/elf.o
  CC      /workspace/kernel/build64-default/tools/objtool/objtool.o
  CC      /workspace/kernel/build64-default/tools/objtool/orc_gen.o
  CC      /workspace/kernel/build64-default/tools/objtool/libstring.o
  CC      /workspace/kernel/build64-default/tools/objtool/orc_dump.o
  CC      /workspace/kernel/build64-default/tools/objtool/libctype.o
  CC      /workspace/kernel/build64-default/tools/objtool/str_error_r.o
  CC      /workspace/kernel/build64-default/tools/objtool/librbtree.o
  CC      /workspace/kernel/build64-default/tools/objtool/arch/x86/special.o
  CC      /workspace/kernel/build64-default/tools/objtool/arch/x86/decode.o
  CC      /workspace/kernel/build64-default/tools/objtool/arch/x86/orc.o
  LD      /workspace/kernel/build64-default/tools/objtool/arch/x86/objtool-in.o
  LD      /workspace/kernel/build64-default/tools/objtool/objtool-in.o
  LINK    /workspace/kernel/build64-default/tools/objtool/objtool
make[1]: Leaving directory '/workspace/kernel/build64-default'
++ nproc
+ make -j48 O=/workspace/kernel/build64-default W=1 drivers/gpu/drm/xe
make[1]: Entering directory '/workspace/kernel/build64-default'
make[2]: Nothing to be done for 'drivers/gpu/drm/xe'.
make[1]: Leaving directory '/workspace/kernel/build64-default'
run-parts: executing /workspace/ci/hooks/11-build-32b
+++ realpath /workspace/ci/hooks/11-build-32b
++ dirname /workspace/ci/hooks/11-build-32b
+ THIS_SCRIPT_DIR=/workspace/ci/hooks
+ SRC_DIR=/workspace/kernel
+ TOOLS_SRC_DIR=/workspace/ci
+ '[' -n /workspace/kernel/build64-default ']'
+ BUILD_DIR=/workspace/kernel/build64-default
+ BUILD_DIR=/workspace/kernel/build64-default/build32
+ cd /workspace/kernel
+ mkdir -p /workspace/kernel/build64-default/build32
++ nproc
+ make -j48 ARCH=i386 O=/workspace/kernel/build64-default/build32 defconfig
make[1]: Entering directory '/workspace/kernel/build64-default/build32'
  GEN     Makefile
  HOSTCC  scripts/basic/fixdep
  HOSTCC  scripts/kconfig/conf.o
  HOSTCC  scripts/kconfig/confdata.o
  HOSTCC  scripts/kconfig/expr.o
  LEX     scripts/kconfig/lexer.lex.c
  YACC    scripts/kconfig/parser.tab.[ch]
  HOSTCC  scripts/kconfig/menu.o
  HOSTCC  scripts/kconfig/preprocess.o
  HOSTCC  scripts/kconfig/symbol.o
  HOSTCC  scripts/kconfig/util.o
  HOSTCC  scripts/kconfig/lexer.lex.o
  HOSTCC  scripts/kconfig/parser.tab.o
  HOSTLD  scripts/kconfig/conf
*** Default configuration is based on 'i386_defconfig'
#
# configuration written to .config
#
make[1]: Leaving directory '/workspace/kernel/build64-default/build32'
+ cd /workspace/kernel/build64-default/build32
+ /workspace/kernel/scripts/kconfig/merge_config.sh .config /workspace/ci/kernel/10-xe.fragment
Using .config as base
Merging /workspace/ci/kernel/10-xe.fragment
Value of CONFIG_DRM_XE is redefined by fragment /workspace/ci/kernel/10-xe.fragment:
Previous value: # CONFIG_DRM_XE is not set
New value: CONFIG_DRM_XE=m

Value of CONFIG_SND_DEBUG is redefined by fragment /workspace/ci/kernel/10-xe.fragment:
Previous value: # CONFIG_SND_DEBUG is not set
New value: CONFIG_SND_DEBUG=y

Value of CONFIG_SND_HDA_INTEL is redefined by fragment /workspace/ci/kernel/10-xe.fragment:
Previous value: CONFIG_SND_HDA_INTEL=y
New value: CONFIG_SND_HDA_INTEL=m

Value of CONFIG_SND_HDA_CODEC_HDMI is redefined by fragment /workspace/ci/kernel/10-xe.fragment:
Previous value: # CONFIG_SND_HDA_CODEC_HDMI is not set
New value: CONFIG_SND_HDA_CODEC_HDMI=m

  GEN     Makefile

WARNING: unmet direct dependencies detected for FB_IOMEM_HELPERS
  Depends on [n]: HAS_IOMEM [=y] && FB_CORE [=n]
  Selected by [m]:
  - DRM_XE_DISPLAY [=y] && HAS_IOMEM [=y] && DRM [=y] && DRM_XE [=m] && DRM_XE [=m]=m [=m]
#
# configuration written to .config
#
Value requested for CONFIG_HAVE_UID16 not in final .config
Requested value:  CONFIG_HAVE_UID16=y
Actual value:     

Value requested for CONFIG_UID16 not in final .config
Requested value:  CONFIG_UID16=y
Actual value:     

Value requested for CONFIG_X86_32 not in final .config
Requested value:  CONFIG_X86_32=y
Actual value:     

Value requested for CONFIG_OUTPUT_FORMAT not in final .config
Requested value:  CONFIG_OUTPUT_FORMAT="elf32-i386"
Actual value:     CONFIG_OUTPUT_FORMAT="elf64-x86-64"

Value requested for CONFIG_ARCH_MMAP_RND_BITS_MIN not in final .config
Requested value:  CONFIG_ARCH_MMAP_RND_BITS_MIN=8
Actual value:     CONFIG_ARCH_MMAP_RND_BITS_MIN=28

Value requested for CONFIG_ARCH_MMAP_RND_BITS_MAX not in final .config
Requested value:  CONFIG_ARCH_MMAP_RND_BITS_MAX=16
Actual value:     CONFIG_ARCH_MMAP_RND_BITS_MAX=32

Value requested for CONFIG_PGTABLE_LEVELS not in final .config
Requested value:  CONFIG_PGTABLE_LEVELS=2
Actual value:     CONFIG_PGTABLE_LEVELS=5

Value requested for CONFIG_X86_BIGSMP not in final .config
Requested value:  # CONFIG_X86_BIGSMP is not set
Actual value:     

Value requested for CONFIG_X86_INTEL_QUARK not in final .config
Requested value:  # CONFIG_X86_INTEL_QUARK is not set
Actual value:     

Value requested for CONFIG_X86_RDC321X not in final .config
Requested value:  # CONFIG_X86_RDC321X is not set
Actual value:     

Value requested for CONFIG_X86_32_NON_STANDARD not in final .config
Requested value:  # CONFIG_X86_32_NON_STANDARD is not set
Actual value:     

Value requested for CONFIG_X86_32_IRIS not in final .config
Requested value:  # CONFIG_X86_32_IRIS is not set
Actual value:     

Value requested for CONFIG_M486SX not in final .config
Requested value:  # CONFIG_M486SX is not set
Actual value:     

Value requested for CONFIG_M486 not in final .config
Requested value:  # CONFIG_M486 is not set
Actual value:     

Value requested for CONFIG_M586 not in final .config
Requested value:  # CONFIG_M586 is not set
Actual value:     

Value requested for CONFIG_M586TSC not in final .config
Requested value:  # CONFIG_M586TSC is not set
Actual value:     

Value requested for CONFIG_M586MMX not in final .config
Requested value:  # CONFIG_M586MMX is not set
Actual value:     

Value requested for CONFIG_M686 not in final .config
Requested value:  CONFIG_M686=y
Actual value:     

Value requested for CONFIG_MPENTIUMII not in final .config
Requested value:  # CONFIG_MPENTIUMII is not set
Actual value:     

Value requested for CONFIG_MPENTIUMIII not in final .config
Requested value:  # CONFIG_MPENTIUMIII is not set
Actual value:     

Value requested for CONFIG_MPENTIUMM not in final .config
Requested value:  # CONFIG_MPENTIUMM is not set
Actual value:     

Value requested for CONFIG_MPENTIUM4 not in final .config
Requested value:  # CONFIG_MPENTIUM4 is not set
Actual value:     

Value requested for CONFIG_MK6 not in final .config
Requested value:  # CONFIG_MK6 is not set
Actual value:     

Value requested for CONFIG_MK7 not in final .config
Requested value:  # CONFIG_MK7 is not set
Actual value:     

Value requested for CONFIG_MCRUSOE not in final .config
Requested value:  # CONFIG_MCRUSOE is not set
Actual value:     

Value requested for CONFIG_MEFFICEON not in final .config
Requested value:  # CONFIG_MEFFICEON is not set
Actual value:     

Value requested for CONFIG_MWINCHIPC6 not in final .config
Requested value:  # CONFIG_MWINCHIPC6 is not set
Actual value:     

Value requested for CONFIG_MWINCHIP3D not in final .config
Requested value:  # CONFIG_MWINCHIP3D is not set
Actual value:     

Value requested for CONFIG_MELAN not in final .config
Requested value:  # CONFIG_MELAN is not set
Actual value:     

Value requested for CONFIG_MGEODEGX1 not in final .config
Requested value:  # CONFIG_MGEODEGX1 is not set
Actual value:     

Value requested for CONFIG_MGEODE_LX not in final .config
Requested value:  # CONFIG_MGEODE_LX is not set
Actual value:     

Value requested for CONFIG_MCYRIXIII not in final .config
Requested value:  # CONFIG_MCYRIXIII is not set
Actual value:     

Value requested for CONFIG_MVIAC3_2 not in final .config
Requested value:  # CONFIG_MVIAC3_2 is not set
Actual value:     

Value requested for CONFIG_MVIAC7 not in final .config
Requested value:  # CONFIG_MVIAC7 is not set
Actual value:     

Value requested for CONFIG_X86_GENERIC not in final .config
Requested value:  # CONFIG_X86_GENERIC is not set
Actual value:     

Value requested for CONFIG_X86_INTERNODE_CACHE_SHIFT not in final .config
Requested value:  CONFIG_X86_INTERNODE_CACHE_SHIFT=5
Actual value:     CONFIG_X86_INTERNODE_CACHE_SHIFT=6

Value requested for CONFIG_X86_L1_CACHE_SHIFT not in final .config
Requested value:  CONFIG_X86_L1_CACHE_SHIFT=5
Actual value:     CONFIG_X86_L1_CACHE_SHIFT=6

Value requested for CONFIG_X86_USE_PPRO_CHECKSUM not in final .config
Requested value:  CONFIG_X86_USE_PPRO_CHECKSUM=y
Actual value:     

Value requested for CONFIG_X86_MINIMUM_CPU_FAMILY not in final .config
Requested value:  CONFIG_X86_MINIMUM_CPU_FAMILY=6
Actual value:     CONFIG_X86_MINIMUM_CPU_FAMILY=64

Value requested for CONFIG_CPU_SUP_TRANSMETA_32 not in final .config
Requested value:  CONFIG_CPU_SUP_TRANSMETA_32=y
Actual value:     

Value requested for CONFIG_CPU_SUP_VORTEX_32 not in final .config
Requested value:  CONFIG_CPU_SUP_VORTEX_32=y
Actual value:     

Value requested for CONFIG_HPET_TIMER not in final .config
Requested value:  # CONFIG_HPET_TIMER is not set
Actual value:     CONFIG_HPET_TIMER=y

Value requested for CONFIG_NR_CPUS_RANGE_END not in final .config
Requested value:  CONFIG_NR_CPUS_RANGE_END=8
Actual value:     CONFIG_NR_CPUS_RANGE_END=512

Value requested for CONFIG_NR_CPUS_DEFAULT not in final .config
Requested value:  CONFIG_NR_CPUS_DEFAULT=8
Actual value:     CONFIG_NR_CPUS_DEFAULT=64

Value requested for CONFIG_X86_ANCIENT_MCE not in final .config
Requested value:  # CONFIG_X86_ANCIENT_MCE is not set
Actual value:     

Value requested for CONFIG_X86_LEGACY_VM86 not in final .config
Requested value:  # CONFIG_X86_LEGACY_VM86 is not set
Actual value:     

Value requested for CONFIG_X86_ESPFIX32 not in final .config
Requested value:  CONFIG_X86_ESPFIX32=y
Actual value:     

Value requested for CONFIG_TOSHIBA not in final .config
Requested value:  # CONFIG_TOSHIBA is not set
Actual value:     

Value requested for CONFIG_X86_REBOOTFIXUPS not in final .config
Requested value:  # CONFIG_X86_REBOOTFIXUPS is not set
Actual value:     

Value requested for CONFIG_MICROCODE_INITRD32 not in final .config
Requested value:  CONFIG_MICROCODE_INITRD32=y
Actual value:     

Value requested for CONFIG_NOHIGHMEM not in final .config
Requested value:  # CONFIG_NOHIGHMEM is not set
Actual value:     

Value requested for CONFIG_HIGHMEM4G not in final .config
Requested value:  CONFIG_HIGHMEM4G=y
Actual value:     

Value requested for CONFIG_HIGHMEM64G not in final .config
Requested value:  # CONFIG_HIGHMEM64G is not set
Actual value:     

Value requested for CONFIG_VMSPLIT_3G not in final .config
Requested value:  CONFIG_VMSPLIT_3G=y
Actual value:     

Value requested for CONFIG_VMSPLIT_3G_OPT not in final .config
Requested value:  # CONFIG_VMSPLIT_3G_OPT is not set
Actual value:     

Value requested for CONFIG_VMSPLIT_2G not in final .config
Requested value:  # CONFIG_VMSPLIT_2G is not set
Actual value:     

Value requested for CONFIG_VMSPLIT_2G_OPT not in final .config
Requested value:  # CONFIG_VMSPLIT_2G_OPT is not set
Actual value:     

Value requested for CONFIG_VMSPLIT_1G not in final .config
Requested value:  # CONFIG_VMSPLIT_1G is not set
Actual value:     

Value requested for CONFIG_PAGE_OFFSET not in final .config
Requested value:  CONFIG_PAGE_OFFSET=0xC0000000
Actual value:     

Value requested for CONFIG_HIGHMEM not in final .config
Requested value:  CONFIG_HIGHMEM=y
Actual value:     

Value requested for CONFIG_X86_PAE not in final .config
Requested value:  # CONFIG_X86_PAE is not set
Actual value:     

Value requested for CONFIG_ARCH_FLATMEM_ENABLE not in final .config
Requested value:  CONFIG_ARCH_FLATMEM_ENABLE=y
Actual value:     

Value requested for CONFIG_ARCH_SELECT_MEMORY_MODEL not in final .config
Requested value:  CONFIG_ARCH_SELECT_MEMORY_MODEL=y
Actual value:     

Value requested for CONFIG_ILLEGAL_POINTER_VALUE not in final .config
Requested value:  CONFIG_ILLEGAL_POINTER_VALUE=0
Actual value:     CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000

Value requested for CONFIG_HIGHPTE not in final .config
Requested value:  # CONFIG_HIGHPTE is not set
Actual value:     

Value requested for CONFIG_COMPAT_VDSO not in final .config
Requested value:  # CONFIG_COMPAT_VDSO is not set
Actual value:     

Value requested for CONFIG_FUNCTION_PADDING_CFI not in final .config
Requested value:  CONFIG_FUNCTION_PADDING_CFI=0
Actual value:     CONFIG_FUNCTION_PADDING_CFI=11

Value requested for CONFIG_FUNCTION_PADDING_BYTES not in final .config
Requested value:  CONFIG_FUNCTION_PADDING_BYTES=4
Actual value:     CONFIG_FUNCTION_PADDING_BYTES=16

Value requested for CONFIG_APM not in final .config
Requested value:  # CONFIG_APM is not set
Actual value:     

Value requested for CONFIG_X86_POWERNOW_K6 not in final .config
Requested value:  # CONFIG_X86_POWERNOW_K6 is not set
Actual value:     

Value requested for CONFIG_X86_POWERNOW_K7 not in final .config
Requested value:  # CONFIG_X86_POWERNOW_K7 is not set
Actual value:     

Value requested for CONFIG_X86_GX_SUSPMOD not in final .config
Requested value:  # CONFIG_X86_GX_SUSPMOD is not set
Actual value:     

Value requested for CONFIG_X86_SPEEDSTEP_ICH not in final .config
Requested value:  # CONFIG_X86_SPEEDSTEP_ICH is not set
Actual value:     

Value requested for CONFIG_X86_SPEEDSTEP_SMI not in final .config
Requested value:  # CONFIG_X86_SPEEDSTEP_SMI is not set
Actual value:     

Value requested for CONFIG_X86_CPUFREQ_NFORCE2 not in final .config
Requested value:  # CONFIG_X86_CPUFREQ_NFORCE2 is not set
Actual value:     

Value requested for CONFIG_X86_LONGRUN not in final .config
Requested value:  # CONFIG_X86_LONGRUN is not set
Actual value:     

Value requested for CONFIG_X86_LONGHAUL not in final .config
Requested value:  # CONFIG_X86_LONGHAUL is not set
Actual value:     

Value requested for CONFIG_X86_E_POWERSAVER not in final .config
Requested value:  # CONFIG_X86_E_POWERSAVER is not set
Actual value:     

Value requested for CONFIG_PCI_GOBIOS not in final .config
Requested value:  # CONFIG_PCI_GOBIOS is not set
Actual value:     

Value requested for CONFIG_PCI_GOMMCONFIG not in final .config
Requested value:  # CONFIG_PCI_GOMMCONFIG is not set
Actual value:     

Value requested for CONFIG_PCI_GODIRECT not in final .config
Requested value:  # CONFIG_PCI_GODIRECT is not set
Actual value:     

Value requested for CONFIG_PCI_GOANY not in final .config
Requested value:  CONFIG_PCI_GOANY=y
Actual value:     

Value requested for CONFIG_PCI_BIOS not in final .config
Requested value:  CONFIG_PCI_BIOS=y
Actual value:     

Value requested for CONFIG_ISA not in final .config
Requested value:  # CONFIG_ISA is not set
Actual value:     

Value requested for CONFIG_SCx200 not in final .config
Requested value:  # CONFIG_SCx200 is not set
Actual value:     

Value requested for CONFIG_OLPC not in final .config
Requested value:  # CONFIG_OLPC is not set
Actual value:     

Value requested for CONFIG_ALIX not in final .config
Requested value:  # CONFIG_ALIX is not set
Actual value:     

Value requested for CONFIG_NET5501 not in final .config
Requested value:  # CONFIG_NET5501 is not set
Actual value:     

Value requested for CONFIG_GEOS not in final .config
Requested value:  # CONFIG_GEOS is not set
Actual value:     

Value requested for CONFIG_COMPAT_32 not in final .config
Requested value:  CONFIG_COMPAT_32=y
Actual value:     

Value requested for CONFIG_HAVE_ATOMIC_IOMAP not in final .config
Requested value:  CONFIG_HAVE_ATOMIC_IOMAP=y
Actual value:     

Value requested for CONFIG_ARCH_32BIT_OFF_T not in final .config
Requested value:  CONFIG_ARCH_32BIT_OFF_T=y
Actual value:     

Value requested for CONFIG_ARCH_WANT_IPC_PARSE_VERSION not in final .config
Requested value:  CONFIG_ARCH_WANT_IPC_PARSE_VERSION=y
Actual value:     

Value requested for CONFIG_MODULES_USE_ELF_REL not in final .config
Requested value:  CONFIG_MODULES_USE_ELF_REL=y
Actual value:     

Value requested for CONFIG_ARCH_MMAP_RND_BITS not in final .config
Requested value:  CONFIG_ARCH_MMAP_RND_BITS=8
Actual value:     CONFIG_ARCH_MMAP_RND_BITS=28

Value requested for CONFIG_CLONE_BACKWARDS not in final .config
Requested value:  CONFIG_CLONE_BACKWARDS=y
Actual value:     

Value requested for CONFIG_OLD_SIGSUSPEND3 not in final .config
Requested value:  CONFIG_OLD_SIGSUSPEND3=y
Actual value:     

Value requested for CONFIG_OLD_SIGACTION not in final .config
Requested value:  CONFIG_OLD_SIGACTION=y
Actual value:     

Value requested for CONFIG_ARCH_SPLIT_ARG64 not in final .config
Requested value:  CONFIG_ARCH_SPLIT_ARG64=y
Actual value:     

Value requested for CONFIG_FUNCTION_ALIGNMENT not in final .config
Requested value:  CONFIG_FUNCTION_ALIGNMENT=4
Actual value:     CONFIG_FUNCTION_ALIGNMENT=16

Value requested for CONFIG_SELECT_MEMORY_MODEL not in final .config
Requested value:  CONFIG_SELECT_MEMORY_MODEL=y
Actual value:     

Value requested for CONFIG_FLATMEM_MANUAL not in final .config
Requested value:  CONFIG_FLATMEM_MANUAL=y
Actual value:     

Value requested for CONFIG_SPARSEMEM_MANUAL not in final .config
Requested value:  # CONFIG_SPARSEMEM_MANUAL is not set
Actual value:     

Value requested for CONFIG_FLATMEM not in final .config
Requested value:  CONFIG_FLATMEM=y
Actual value:     

Value requested for CONFIG_SPARSEMEM_STATIC not in final .config
Requested value:  CONFIG_SPARSEMEM_STATIC=y
Actual value:     

Value requested for CONFIG_BOUNCE not in final .config
Requested value:  CONFIG_BOUNCE=y
Actual value:     

Value requested for CONFIG_KMAP_LOCAL not in final .config
Requested value:  CONFIG_KMAP_LOCAL=y
Actual value:     

Value requested for CONFIG_HOTPLUG_PCI_COMPAQ not in final .config
Requested value:  # CONFIG_HOTPLUG_PCI_COMPAQ is not set
Actual value:     

Value requested for CONFIG_HOTPLUG_PCI_IBM not in final .config
Requested value:  # CONFIG_HOTPLUG_PCI_IBM is not set
Actual value:     

Value requested for CONFIG_EFI_CAPSULE_QUIRK_QUARK_CSH not in final .config
Requested value:  CONFIG_EFI_CAPSULE_QUIRK_QUARK_CSH=y
Actual value:     

Value requested for CONFIG_PCH_PHUB not in final .config
Requested value:  # CONFIG_PCH_PHUB is not set
Actual value:     

Value requested for CONFIG_SCSI_NSP32 not in final .config
Requested value:  # CONFIG_SCSI_NSP32 is not set
Actual value:     

Value requested for CONFIG_PATA_CS5520 not in final .config
Requested value:  # CONFIG_PATA_CS5520 is not set
Actual value:     

Value requested for CONFIG_PATA_CS5530 not in final .config
Requested value:  # CONFIG_PATA_CS5530 is not set
Actual value:     

Value requested for CONFIG_PATA_CS5535 not in final .config
Requested value:  # CONFIG_PATA_CS5535 is not set
Actual value:     

Value requested for CONFIG_PATA_CS5536 not in final .config
Requested value:  # CONFIG_PATA_CS5536 is not set
Actual value:     

Value requested for CONFIG_PATA_SC1200 not in final .config
Requested value:  # CONFIG_PATA_SC1200 is not set
Actual value:     

Value requested for CONFIG_PCH_GBE not in final .config
Requested value:  # CONFIG_PCH_GBE is not set
Actual value:     

Value requested for CONFIG_INPUT_WISTRON_BTNS not in final .config
Requested value:  # CONFIG_INPUT_WISTRON_BTNS is not set
Actual value:     

Value requested for CONFIG_SERIAL_TIMBERDALE not in final .config
Requested value:  # CONFIG_SERIAL_TIMBERDALE is not set
Actual value:     

Value requested for CONFIG_SERIAL_PCH_UART not in final .config
Requested value:  # CONFIG_SERIAL_PCH_UART is not set
Actual value:     

Value requested for CONFIG_HW_RANDOM_GEODE not in final .config
Requested value:  CONFIG_HW_RANDOM_GEODE=y
Actual value:     

Value requested for CONFIG_SONYPI not in final .config
Requested value:  # CONFIG_SONYPI is not set
Actual value:     

Value requested for CONFIG_PC8736x_GPIO not in final .config
Requested value:  # CONFIG_PC8736x_GPIO is not set
Actual value:     

Value requested for CONFIG_NSC_GPIO not in final .config
Requested value:  # CONFIG_NSC_GPIO is not set
Actual value:     

Value requested for CONFIG_I2C_EG20T not in final .config
Requested value:  # CONFIG_I2C_EG20T is not set
Actual value:     

Value requested for CONFIG_SCx200_ACB not in final .config
Requested value:  # CONFIG_SCx200_ACB is not set
Actual value:     

Value requested for CONFIG_PTP_1588_CLOCK_PCH not in final .config
Requested value:  # CONFIG_PTP_1588_CLOCK_PCH is not set
Actual value:     

Value requested for CONFIG_SBC8360_WDT not in final .config
Requested value:  # CONFIG_SBC8360_WDT is not set
Actual value:     

Value requested for CONFIG_SBC7240_WDT not in final .config
Requested value:  # CONFIG_SBC7240_WDT is not set
Actual value:     

Value requested for CONFIG_MFD_CS5535 not in final .config
Requested value:  # CONFIG_MFD_CS5535 is not set
Actual value:     

Value requested for CONFIG_AGP_ALI not in final .config
Requested value:  # CONFIG_AGP_ALI is not set
Actual value:     

Value requested for CONFIG_AGP_ATI not in final .config
Requested value:  # CONFIG_AGP_ATI is not set
Actual value:     

Value requested for CONFIG_AGP_AMD not in final .config
Requested value:  # CONFIG_AGP_AMD is not set
Actual value:     

Value requested for CONFIG_AGP_NVIDIA not in final .config
Requested value:  # CONFIG_AGP_NVIDIA is not set
Actual value:     

Value requested for CONFIG_AGP_SWORKS not in final .config
Requested value:  # CONFIG_AGP_SWORKS is not set
Actual value:     

Value requested for CONFIG_AGP_EFFICEON not in final .config
Requested value:  # CONFIG_AGP_EFFICEON is not set
Actual value:     

Value requested for CONFIG_SND_PCM not in final .config
Requested value:  CONFIG_SND_PCM=y
Actual value:     CONFIG_SND_PCM=m

Value requested for CONFIG_SND_HWDEP not in final .config
Requested value:  CONFIG_SND_HWDEP=y
Actual value:     CONFIG_SND_HWDEP=m

Value requested for CONFIG_SND_DYNAMIC_MINORS not in final .config
Requested value:  # CONFIG_SND_DYNAMIC_MINORS is not set
Actual value:     CONFIG_SND_DYNAMIC_MINORS=y

Value requested for CONFIG_SND_CS5530 not in final .config
Requested value:  # CONFIG_SND_CS5530 is not set
Actual value:     

Value requested for CONFIG_SND_CS5535AUDIO not in final .config
Requested value:  # CONFIG_SND_CS5535AUDIO is not set
Actual value:     

Value requested for CONFIG_SND_SIS7019 not in final .config
Requested value:  # CONFIG_SND_SIS7019 is not set
Actual value:     

Value requested for CONFIG_SND_HDA not in final .config
Requested value:  CONFIG_SND_HDA=y
Actual value:     CONFIG_SND_HDA=m

Value requested for CONFIG_SND_HDA_CORE not in final .config
Requested value:  CONFIG_SND_HDA_CORE=y
Actual value:     CONFIG_SND_HDA_CORE=m

Value requested for CONFIG_SND_INTEL_DSP_CONFIG not in final .config
Requested value:  CONFIG_SND_INTEL_DSP_CONFIG=y
Actual value:     CONFIG_SND_INTEL_DSP_CONFIG=m

Value requested for CONFIG_SND_INTEL_SOUNDWIRE_ACPI not in final .config
Requested value:  CONFIG_SND_INTEL_SOUNDWIRE_ACPI=y
Actual value:     CONFIG_SND_INTEL_SOUNDWIRE_ACPI=m

Value requested for CONFIG_LEDS_OT200 not in final .config
Requested value:  # CONFIG_LEDS_OT200 is not set
Actual value:     

Value requested for CONFIG_PCH_DMA not in final .config
Requested value:  # CONFIG_PCH_DMA is not set
Actual value:     

Value requested for CONFIG_CLKSRC_I8253 not in final .config
Requested value:  CONFIG_CLKSRC_I8253=y
Actual value:     

Value requested for CONFIG_MAILBOX not in final .config
Requested value:  # CONFIG_MAILBOX is not set
Actual value:     CONFIG_MAILBOX=y

Value requested for CONFIG_CRYPTO_SERPENT_SSE2_586 not in final .config
Requested value:  # CONFIG_CRYPTO_SERPENT_SSE2_586 is not set
Actual value:     

Value requested for CONFIG_CRYPTO_TWOFISH_586 not in final .config
Requested value:  # CONFIG_CRYPTO_TWOFISH_586 is not set
Actual value:     

Value requested for CONFIG_CRYPTO_DEV_GEODE not in final .config
Requested value:  # CONFIG_CRYPTO_DEV_GEODE is not set
Actual value:     

Value requested for CONFIG_CRYPTO_DEV_HIFN_795X not in final .config
Requested value:  # CONFIG_CRYPTO_DEV_HIFN_795X is not set
Actual value:     

Value requested for CONFIG_CRYPTO_LIB_POLY1305_RSIZE not in final .config
Requested value:  CONFIG_CRYPTO_LIB_POLY1305_RSIZE=1
Actual value:     CONFIG_CRYPTO_LIB_POLY1305_RSIZE=11

Value requested for CONFIG_AUDIT_GENERIC not in final .config
Requested value:  CONFIG_AUDIT_GENERIC=y
Actual value:     

Value requested for CONFIG_GENERIC_VDSO_32 not in final .config
Requested value:  CONFIG_GENERIC_VDSO_32=y
Actual value:     

Value requested for CONFIG_DEBUG_KMAP_LOCAL not in final .config
Requested value:  # CONFIG_DEBUG_KMAP_LOCAL is not set
Actual value:     

Value requested for CONFIG_DEBUG_HIGHMEM not in final .config
Requested value:  # CONFIG_DEBUG_HIGHMEM is not set
Actual value:     

Value requested for CONFIG_HAVE_DEBUG_STACKOVERFLOW not in final .config
Requested value:  CONFIG_HAVE_DEBUG_STACKOVERFLOW=y
Actual value:     

Value requested for CONFIG_DEBUG_STACKOVERFLOW not in final .config
Requested value:  # CONFIG_DEBUG_STACKOVERFLOW is not set
Actual value:     

Value requested for CONFIG_HAVE_FUNCTION_GRAPH_TRACER not in final .config
Requested value:  CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
Actual value:     

Value requested for CONFIG_HAVE_FUNCTION_GRAPH_RETVAL not in final .config
Requested value:  CONFIG_HAVE_FUNCTION_GRAPH_RETVAL=y
Actual value:     

Value requested for CONFIG_DRM_KUNIT_TEST not in final .config
Requested value:  CONFIG_DRM_KUNIT_TEST=m
Actual value:     

Value requested for CONFIG_DRM_XE_WERROR not in final .config
Requested value:  CONFIG_DRM_XE_WERROR=y
Actual value:     

Value requested for CONFIG_DRM_XE_DEBUG not in final .config
Requested value:  CONFIG_DRM_XE_DEBUG=y
Actual value:     

Value requested for CONFIG_DRM_XE_DEBUG_MEM not in final .config
Requested value:  CONFIG_DRM_XE_DEBUG_MEM=y
Actual value:     

Value requested for CONFIG_DRM_XE_KUNIT_TEST not in final .config
Requested value:  CONFIG_DRM_XE_KUNIT_TEST=m
Actual value:     

++ nproc
+ make -j48 ARCH=i386 olddefconfig
  GEN     Makefile

WARNING: unmet direct dependencies detected for FB_IOMEM_HELPERS
  Depends on [n]: HAS_IOMEM [=y] && FB_CORE [=n]
  Selected by [m]:
  - DRM_XE_DISPLAY [=y] && HAS_IOMEM [=y] && DRM [=y] && DRM_XE [=m] && DRM_XE [=m]=m [=m]
#
# configuration written to .config
#
++ nproc
+ make -j48 ARCH=i386
  SYNC    include/config/auto.conf.cmd
  GEN     Makefile

WARNING: unmet direct dependencies detected for FB_IOMEM_HELPERS
  Depends on [n]: HAS_IOMEM [=y] && FB_CORE [=n]
  Selected by [m]:
  - DRM_XE_DISPLAY [=y] && HAS_IOMEM [=y] && DRM [=y] && DRM_XE [=m] && DRM_XE [=m]=m [=m]

WARNING: unmet direct dependencies detected for FB_IOMEM_HELPERS
  Depends on [n]: HAS_IOMEM [=y] && FB_CORE [=n]
  Selected by [m]:
  - DRM_XE_DISPLAY [=y] && HAS_IOMEM [=y] && DRM [=y] && DRM_XE [=m] && DRM_XE [=m]=m [=m]

WARNING: unmet direct dependencies detected for FB_IOMEM_HELPERS
  Depends on [n]: HAS_IOMEM [=y] && FB_CORE [=n]
  Selected by [m]:
  - DRM_XE_DISPLAY [=y] && HAS_IOMEM [=y] && DRM [=y] && DRM_XE [=m] && DRM_XE [=m]=m [=m]
  GEN     Makefile
  WRAP    arch/x86/include/generated/uapi/asm/bpf_perf_event.h
  UPD     include/generated/uapi/linux/version.h
  WRAP    arch/x86/include/generated/uapi/asm/errno.h
  WRAP    arch/x86/include/generated/uapi/asm/fcntl.h
  WRAP    arch/x86/include/generated/uapi/asm/ioctl.h
  WRAP    arch/x86/include/generated/uapi/asm/ipcbuf.h
  WRAP    arch/x86/include/generated/uapi/asm/ioctls.h
  WRAP    arch/x86/include/generated/uapi/asm/param.h
  WRAP    arch/x86/include/generated/uapi/asm/poll.h
  WRAP    arch/x86/include/generated/uapi/asm/socket.h
  WRAP    arch/x86/include/generated/uapi/asm/resource.h
  WRAP    arch/x86/include/generated/uapi/asm/sockios.h
  SYSHDR  arch/x86/include/generated/uapi/asm/unistd_32.h
  WRAP    arch/x86/include/generated/uapi/asm/termbits.h
  SYSHDR  arch/x86/include/generated/uapi/asm/unistd_64.h
  SYSHDR  arch/x86/include/generated/uapi/asm/unistd_x32.h
  WRAP    arch/x86/include/generated/uapi/asm/termios.h
  WRAP    arch/x86/include/generated/uapi/asm/types.h
  SYSTBL  arch/x86/include/generated/asm/syscalls_32.h
  UPD     include/generated/compile.h
  WRAP    arch/x86/include/generated/asm/early_ioremap.h
  HOSTCC  arch/x86/tools/relocs_32.o
  WRAP    arch/x86/include/generated/asm/mcs_spinlock.h
  WRAP    arch/x86/include/generated/asm/irq_regs.h
  HOSTCC  arch/x86/tools/relocs_64.o
  WRAP    arch/x86/include/generated/asm/kmap_size.h
  WRAP    arch/x86/include/generated/asm/local64.h
  HOSTCC  arch/x86/tools/relocs_common.o
  WRAP    arch/x86/include/generated/asm/mmiowb.h
  WRAP    arch/x86/include/generated/asm/module.lds.h
  WRAP    arch/x86/include/generated/asm/rwonce.h
  WRAP    arch/x86/include/generated/asm/unaligned.h
  HOSTCC  scripts/kallsyms
  HOSTCC  scripts/sorttable
  HOSTCC  scripts/asn1_compiler
  HOSTCC  scripts/selinux/genheaders/genheaders
  HOSTCC  scripts/selinux/mdp/mdp
  HOSTLD  arch/x86/tools/relocs
  UPD     include/config/kernel.release
  UPD     include/generated/utsrelease.h
  CC      scripts/mod/empty.o
  HOSTCC  scripts/mod/mk_elfconfig
  CC      scripts/mod/devicetable-offsets.s
  UPD     scripts/mod/devicetable-offsets.h
  MKELF   scripts/mod/elfconfig.h
  HOSTCC  scripts/mod/modpost.o
  HOSTCC  scripts/mod/file2alias.o
  HOSTCC  scripts/mod/sumversion.o
  HOSTCC  scripts/mod/symsearch.o
  HOSTLD  scripts/mod/modpost
  CC      kernel/bounds.s
  CHKSHA1 /workspace/kernel/include/linux/atomic/atomic-arch-fallback.h
  CHKSHA1 /workspace/kernel/include/linux/atomic/atomic-instrumented.h
  CHKSHA1 /workspace/kernel/include/linux/atomic/atomic-long.h
  UPD     include/generated/timeconst.h
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
  UPD     include/generated/asm-offsets.h
  CALL    /workspace/kernel/scripts/checksyscalls.sh
  LDS     scripts/module.lds
  CC      ipc/util.o
  CC      ipc/msgutil.o
  CC      ipc/msg.o
  CC      ipc/sem.o
  CC      ipc/shm.o
  CC      certs/system_keyring.o
  CC      init/main.o
  CC      ipc/syscall.o
  CC      block/bdev.o
  HOSTCC  usr/gen_init_cpio
  CC      ipc/ipc_sysctl.o
  AS      arch/x86/lib/atomic64_cx8_32.o
  CC      io_uring/io_uring.o
  CC      ipc/mqueue.o
  CC      block/fops.o
  CC      io_uring/opdef.o
  AS      arch/x86/lib/checksum_32.o
  CC      io_uring/kbuf.o
  CC      init/do_mounts.o
  CC      ipc/namespace.o
  CC      block/bio.o
  UPD     init/utsversion-tmp.h
  CC      arch/x86/video/video-common.o
  CC      arch/x86/pci/i386.o
  CC      mm/filemap.o
  CC      arch/x86/power/cpu.o
  CC      block/partitions/core.o
  CC      arch/x86/power/hibernate_32.o
  AR      virt/lib/built-in.a
  CC      arch/x86/realmode/init.o
  CC      security/keys/gc.o
  AR      arch/x86/net/built-in.a
  GEN     security/selinux/flask.h security/selinux/av_permissions.h
  AR      arch/x86/crypto/built-in.a
  CC      sound/core/seq/seq.o
  AR      sound/drivers/opl3/built-in.a
  AR      virt/built-in.a
  AR      drivers/cache/built-in.a
  AR      arch/x86/platform/atom/built-in.a
  AR      arch/x86/virt/svm/built-in.a
  CC      security/selinux/avc.o
  CC      lib/math/div64.o
  CC      arch/x86/kernel/fpu/init.o
  CC      net/core/sock.o
  CC      arch/x86/events/amd/core.o
  CC      lib/math/gcd.o
  AR      sound/i2c/other/built-in.a
  CC      fs/notify/dnotify/dnotify.o
  CC      arch/x86/events/amd/lbr.o
  CC      arch/x86/mm/pat/set_memory.o
  CC      fs/notify/inotify/inotify_fsnotify.o
  CC      arch/x86/events/amd/ibs.o
  CC      arch/x86/lib/cmdline.o
  AR      arch/x86/virt/vmx/built-in.a
  AR      sound/i2c/built-in.a
  AR      drivers/irqchip/built-in.a
  AR      arch/x86/platform/ce4100/built-in.a
  AR      arch/x86/virt/built-in.a
  CC      arch/x86/entry/vdso/vma.o
  AR      sound/drivers/opl4/built-in.a
  AS      arch/x86/lib/cmpxchg8b_emu.o
  CC      arch/x86/platform/efi/memmap.o
  CC      kernel/sched/core.o
  AR      sound/drivers/vx/built-in.a
  AR      sound/drivers/mpu401/built-in.a
  AR      drivers/bus/mhi/built-in.a
  CC      io_uring/rsrc.o
  AR      drivers/bus/built-in.a
  AR      sound/drivers/pcsp/built-in.a
  CC      net/core/request_sock.o
  CC      fs/nfs_common/nfsacl.o
  AR      sound/drivers/built-in.a
  AR      drivers/pwm/built-in.a
  CC      crypto/asymmetric_keys/asymmetric_type.o
  CC      crypto/asymmetric_keys/restrict.o
  CC      drivers/pci/msi/pcidev_msi.o
  CC      lib/math/lcm.o
  CC      arch/x86/lib/cpu.o
  CC      lib/math/int_log.o
  GEN     usr/initramfs_data.cpio
  COPY    usr/initramfs_inc_data
  AS      usr/initramfs_data.o
  CC      lib/math/int_pow.o
  AR      usr/built-in.a
  CC      arch/x86/platform/efi/quirks.o
  CC      arch/x86/kernel/fpu/bugs.o
  HOSTCC  certs/extract-cert
  CC      lib/math/int_sqrt.o
  CC      arch/x86/kernel/fpu/core.o
  CC      lib/math/reciprocal_div.o
  CC      ipc/mq_sysctl.o
  CC      sound/core/seq/seq_lock.o
  CC      fs/notify/inotify/inotify_user.o
  AS      arch/x86/realmode/rm/header.o
  CC      arch/x86/lib/delay.o
  CC      lib/math/rational.o
  AS      arch/x86/realmode/rm/trampoline_32.o
  AS      arch/x86/realmode/rm/stack.o
  CC      kernel/sched/fair.o
  AS      arch/x86/realmode/rm/reboot.o
  AR      arch/x86/video/built-in.a
  CC      lib/crypto/mpi/generic_mpih-lshift.o
  CC      lib/crypto/memneq.o
  CERT    certs/x509_certificate_list
  AS      arch/x86/realmode/rm/wakeup_asm.o
  CERT    certs/signing_key.x509
  AS      certs/system_certificates.o
  CC      lib/crypto/mpi/generic_mpih-mul1.o
  CC      arch/x86/realmode/rm/wakemain.o
  CC      lib/crypto/mpi/generic_mpih-mul2.o
  AR      certs/built-in.a
  CC      io_uring/notif.o
  CC      arch/x86/pci/init.o
  AS      arch/x86/power/hibernate_asm_32.o
  CC      drivers/pci/msi/api.o
  CC      security/keys/key.o
  CC      sound/core/sound.o
  CC      arch/x86/entry/vdso/extable.o
  CC      fs/nfs_common/grace.o
  CC      crypto/asymmetric_keys/signature.o
  CC      arch/x86/power/hibernate.o
  CC      arch/x86/realmode/rm/video-mode.o
  AR      fs/notify/dnotify/built-in.a
  CC      mm/mempool.o
  CC      block/partitions/msdos.o
  CC      block/partitions/efi.o
  CC      arch/x86/events/amd/uncore.o
  CC      lib/zlib_inflate/inffast.o
  CC      net/core/skbuff.o
  AS      arch/x86/lib/getuser.o
  AS      arch/x86/realmode/rm/copy.o
  GEN     arch/x86/lib/inat-tables.c
  AS      arch/x86/realmode/rm/bioscall.o
  CC      sound/core/seq/seq_clientmgr.o
  CC      sound/core/seq/seq_memory.o
  AR      lib/math/built-in.a
  CC      arch/x86/realmode/rm/regs.o
  AR      fs/notify/fanotify/built-in.a
  CC      arch/x86/lib/insn-eval.o
  CC      arch/x86/lib/insn.o
  CC      arch/x86/lib/kaslr.o
  CC      drivers/pci/msi/msi.o
  CC      lib/zlib_inflate/inflate.o
  CC      arch/x86/realmode/rm/video-vga.o
  CC      net/core/datagram.o
  CC      arch/x86/realmode/rm/video-vesa.o
  CC      init/do_mounts_initrd.o
  CC      kernel/sched/build_policy.o
  CC      kernel/sched/build_utility.o
  CC      arch/x86/realmode/rm/video-bios.o
  CC      security/selinux/hooks.o
  CC      arch/x86/platform/efi/efi.o
  CC      crypto/api.o
  CC      init/initramfs.o
  CC      crypto/cipher.o
  CC      sound/core/seq/seq_queue.o
  CC      crypto/compress.o
  PASYMS  arch/x86/realmode/rm/pasyms.h
  CC      arch/x86/mm/pat/memtype.o
  LDS     arch/x86/realmode/rm/realmode.lds
  CC      arch/x86/platform/efi/efi_32.o
  CC      lib/crypto/mpi/generic_mpih-mul3.o
  LD      arch/x86/realmode/rm/realmode.elf
  RELOCS  arch/x86/realmode/rm/realmode.relocs
  OBJCOPY arch/x86/realmode/rm/realmode.bin
  CC      crypto/asymmetric_keys/public_key.o
  AS      arch/x86/realmode/rmpiggy.o
  CC      arch/x86/pci/pcbios.o
  AR      arch/x86/realmode/built-in.a
  CC      net/core/stream.o
  CC      init/calibrate.o
  CC      lib/crypto/mpi/generic_mpih-rshift.o
  CC      init/init_task.o
  CC      sound/core/init.o
  CC      drivers/pci/msi/irqdomain.o
  AR      fs/notify/inotify/built-in.a
  CC      fs/notify/fsnotify.o
  AR      fs/nfs_common/built-in.a
  AR      arch/x86/power/built-in.a
  CC      io_uring/tctx.o
  CC      fs/iomap/trace.o
  CC      fs/quota/dquot.o
  ASN.1   crypto/asymmetric_keys/x509.asn1.[ch]
  AR      ipc/built-in.a
  LDS     arch/x86/entry/vdso/vdso32/vdso32.lds
  ASN.1   crypto/asymmetric_keys/x509_akid.asn1.[ch]
  CC      mm/oom_kill.o
  CC      crypto/asymmetric_keys/x509_loader.o
  CC      fs/proc/task_mmu.o
  AS      arch/x86/entry/vdso/vdso32/note.o
  CC      lib/zlib_inflate/infutil.o
  AS      arch/x86/entry/vdso/vdso32/system_call.o
  CC      crypto/asymmetric_keys/x509_public_key.o
  CC      fs/notify/notification.o
  AS      arch/x86/entry/vdso/vdso32/sigreturn.o
  CC      arch/x86/kernel/fpu/regset.o
  CC      arch/x86/entry/vdso/vdso32/vclock_gettime.o
  AR      block/partitions/built-in.a
  CC      security/keys/keyring.o
  CC      lib/crypto/utils.o
  CC      block/elevator.o
  CC      io_uring/filetable.o
  CC      arch/x86/lib/memcpy_32.o
  CC      sound/core/memory.o
  CC      lib/crypto/mpi/generic_mpih-sub1.o
  AR      arch/x86/events/amd/built-in.a
  AR      sound/isa/ad1816a/built-in.a
  AS      arch/x86/lib/memmove_32.o
  CC      arch/x86/kernel/cpu/mce/core.o
  CC      arch/x86/events/intel/core.o
  CC      io_uring/rw.o
  AR      sound/isa/cs423x/built-in.a
  AR      sound/isa/ad1848/built-in.a
  CC      arch/x86/lib/misc.o
  CC      arch/x86/events/intel/bts.o
  CC      lib/crypto/chacha.o
  CC      lib/zlib_inflate/inftrees.o
  AR      sound/isa/es1688/built-in.a
  CC      arch/x86/kernel/acpi/boot.o
  CC      arch/x86/kernel/cpu/mtrr/mtrr.o
  AR      sound/isa/galaxy/built-in.a
  CC      arch/x86/lib/pc-conf-reg.o
  AR      sound/isa/gus/built-in.a
  CC      arch/x86/events/intel/ds.o
  CC      arch/x86/pci/mmconfig_32.o
  AR      sound/isa/msnd/built-in.a
  CC      lib/crypto/mpi/generic_mpih-add1.o
  CC      sound/core/seq/seq_fifo.o
  AR      sound/isa/opti9xx/built-in.a
  AR      sound/isa/sb/built-in.a
  CC      crypto/algapi.o
  CC      sound/core/seq/seq_prioq.o
  AR      sound/isa/wavefront/built-in.a
  CC      sound/core/control.o
  AR      sound/isa/wss/built-in.a
  CC      lib/zlib_inflate/inflate_syms.o
  AR      sound/isa/built-in.a
  CC      arch/x86/mm/pat/memtype_interval.o
  AS      arch/x86/platform/efi/efi_stub_32.o
  CC      sound/core/seq/seq_timer.o
  CC      arch/x86/platform/efi/runtime-map.o
  AR      sound/pci/ac97/built-in.a
  AS      arch/x86/lib/putuser.o
  AR      sound/pci/ali5451/built-in.a
  AR      sound/pci/asihpi/built-in.a
  AS      arch/x86/lib/retpoline.o
  AR      sound/pci/au88x0/built-in.a
  ASN.1   crypto/asymmetric_keys/pkcs7.asn1.[ch]
  CC      crypto/asymmetric_keys/pkcs7_trust.o
  AR      drivers/pci/msi/built-in.a
  AR      sound/pci/aw2/built-in.a
  CC      init/version.o
  CC      drivers/pci/pcie/portdrv.o
  CC      fs/notify/group.o
  CC      arch/x86/lib/string_32.o
  AR      sound/pci/ctxfi/built-in.a
  CC      drivers/pci/pcie/rcec.o
  AR      sound/pci/ca0106/built-in.a
  AR      sound/pci/cs46xx/built-in.a
  CC      arch/x86/entry/vdso/vdso32/vgetcpu.o
  CC      sound/core/misc.o
  CC      arch/x86/lib/strstr_32.o
  AR      sound/pci/cs5535audio/built-in.a
  AR      sound/pci/lola/built-in.a
  AR      sound/pci/lx6464es/built-in.a
  CC      arch/x86/lib/usercopy.o
  AR      sound/pci/echoaudio/built-in.a
  CC      arch/x86/lib/usercopy_32.o
  HOSTCC  arch/x86/entry/vdso/vdso2c
  AR      sound/pci/emu10k1/built-in.a
  AR      sound/pci/hda/built-in.a
  CC [M]  sound/pci/hda/hda_bind.o
  AR      lib/zlib_inflate/built-in.a
  CC      arch/x86/entry/vdso/vdso32-setup.o
  CC      arch/x86/kernel/fpu/signal.o
  AR      init/built-in.a
  CC      crypto/asymmetric_keys/pkcs7_verify.o
  CC      arch/x86/kernel/apic/apic.o
  CC      arch/x86/events/zhaoxin/core.o
  CC      arch/x86/events/core.o
  CC      arch/x86/kernel/cpu/mce/severity.o
  CC      lib/crypto/mpi/ec.o
  CC      security/keys/keyctl.o
  CC      arch/x86/pci/direct.o
  CC      arch/x86/kernel/apic/apic_common.o
  VDSO    arch/x86/entry/vdso/vdso32.so.dbg
  CC      sound/core/seq/seq_system.o
  CC      arch/x86/lib/msr-smp.o
  CC      io_uring/net.o
  CC      fs/iomap/iter.o
  CC      io_uring/poll.o
  CC      fs/iomap/buffered-io.o
  CC      arch/x86/kernel/cpu/mtrr/if.o
  AR      arch/x86/mm/pat/built-in.a
  OBJCOPY arch/x86/entry/vdso/vdso32.so
  VDSO2C  arch/x86/entry/vdso/vdso-image-32.c
  CC      arch/x86/entry/vdso/vdso-image-32.o
  CC      arch/x86/mm/init.o
  CC      sound/core/seq/seq_ports.o
  AR      arch/x86/platform/efi/built-in.a
  CC      arch/x86/kernel/cpu/mtrr/generic.o
  CC      lib/crypto/aes.o
  AR      arch/x86/platform/geode/built-in.a
  CC      block/blk-core.o
  CC      crypto/asymmetric_keys/x509.asn1.o
  AR      arch/x86/platform/iris/built-in.a
  CC      crypto/asymmetric_keys/x509_akid.asn1.o
  CC      arch/x86/platform/intel/iosf_mbi.o
  CC      arch/x86/lib/cache-smp.o
  CC      fs/notify/mark.o
  CC      block/blk-sysfs.o
  CC      io_uring/eventfd.o
  CC      crypto/asymmetric_keys/x509_cert_parser.o
  CC      arch/x86/kernel/acpi/sleep.o
  AR      arch/x86/entry/vdso/built-in.a
  AR      arch/x86/entry/vsyscall/built-in.a
  AS      arch/x86/entry/entry.o
  CC      arch/x86/kernel/cpu/mtrr/cleanup.o
  AS      arch/x86/entry/entry_32.o
  CC      drivers/pci/pcie/aspm.o
  CC      arch/x86/events/intel/knc.o
  CC      arch/x86/entry/syscall_32.o
  CC      mm/fadvise.o
  CC      arch/x86/lib/msr.o
  CC [M]  sound/pci/hda/hda_codec.o
  CC      arch/x86/entry/common.o
  CC      fs/proc/inode.o
  CC      fs/proc/root.o
  CC      sound/core/seq/seq_info.o
  CC      kernel/locking/mutex.o
  CC      arch/x86/pci/mmconfig-shared.o
  CC      kernel/locking/semaphore.o
  CC      arch/x86/kernel/cpu/mtrr/amd.o
  CC      arch/x86/kernel/fpu/xstate.o
  AR      arch/x86/events/zhaoxin/built-in.a
  CC      fs/quota/quota_v2.o
  CC      arch/x86/events/probe.o
  CC      arch/x86/kernel/cpu/mce/genpool.o
  CC      lib/crypto/arc4.o
  AS      arch/x86/lib/msr-reg.o
  CC      crypto/asymmetric_keys/pkcs7.asn1.o
  CC      arch/x86/lib/msr-reg-export.o
  CC      crypto/asymmetric_keys/pkcs7_parser.o
  CC      sound/core/seq/seq_dummy.o
  CC      crypto/scatterwalk.o
  CC      fs/proc/base.o
  CC      lib/crypto/mpi/mpicoder.o
  CC      drivers/pci/pcie/pme.o
  CC      crypto/proc.o
  CC      security/keys/permission.o
  CC      arch/x86/mm/init_32.o
  AR      arch/x86/platform/intel/built-in.a
  AS      arch/x86/kernel/acpi/wakeup_32.o
  AR      arch/x86/platform/intel-mid/built-in.a
  CC      security/keys/process_keys.o
  CC      lib/crypto/gf128mul.o
  CC      security/keys/request_key.o
  AR      arch/x86/platform/intel-quark/built-in.a
  CC      arch/x86/kernel/acpi/cstate.o
  AR      arch/x86/platform/olpc/built-in.a
  CC      kernel/locking/rwsem.o
  AR      arch/x86/platform/scx200/built-in.a
  CC      kernel/locking/percpu-rwsem.o
  CC      arch/x86/mm/fault.o
  AR      arch/x86/platform/ts5500/built-in.a
  AR      arch/x86/platform/uv/built-in.a
  CC      fs/notify/fdinfo.o
  AR      arch/x86/platform/built-in.a
  CC      mm/maccess.o
  CC      arch/x86/mm/ioremap.o
  CC      arch/x86/kernel/cpu/mtrr/cyrix.o
  CC      io_uring/uring_cmd.o
  AS      arch/x86/entry/thunk.o
  AS      arch/x86/lib/hweight.o
  CC      fs/proc/generic.o
  CC      arch/x86/kernel/cpu/mtrr/centaur.o
  CC      arch/x86/mm/extable.o
  CC      arch/x86/lib/iomem.o
  CC      fs/quota/quota_tree.o
  AR      crypto/asymmetric_keys/built-in.a
  CC      arch/x86/kernel/cpu/mce/intel.o
  CC      arch/x86/kernel/cpu/mce/amd.o
  CC      kernel/locking/spinlock.o
  CC      lib/zlib_deflate/deflate.o
  AR      sound/core/seq/built-in.a
  CC      sound/core/device.o
  CC      arch/x86/kernel/apic/apic_noop.o
  CC      lib/zlib_deflate/deftree.o
  CC      arch/x86/pci/fixup.o
  CC      arch/x86/mm/mmap.o
  CC      crypto/aead.o
  AR      arch/x86/kernel/acpi/built-in.a
  CC      lib/crypto/blake2s.o
  CC      io_uring/openclose.o
  AR      arch/x86/entry/built-in.a
  CC      block/blk-flush.o
  CC      arch/x86/kernel/cpu/mce/threshold.o
  CC      kernel/power/qos.o
  CC      arch/x86/lib/atomic64_32.o
  AR      drivers/pci/pcie/built-in.a
  CC      fs/proc/array.o
  AR      drivers/pci/pwrctl/built-in.a
  CC      fs/proc/fd.o
  CC      arch/x86/lib/inat.o
  CC      drivers/pci/hotplug/pci_hotplug_core.o
  CC      fs/proc/proc_tty.o
  CC      lib/crypto/blake2s-generic.o
  CC      drivers/pci/hotplug/acpi_pcihp.o
  CC      lib/crypto/mpi/mpi-add.o
  AR      fs/notify/built-in.a
  AR      arch/x86/lib/built-in.a
  CC      fs/proc/cmdline.o
  CC      arch/x86/kernel/cpu/mtrr/legacy.o
  CC      lib/crypto/mpi/mpi-bit.o
  AR      arch/x86/kernel/fpu/built-in.a
  CC      kernel/locking/osq_lock.o
  AR      arch/x86/lib/lib.a
  CC      kernel/locking/qspinlock.o
  CC      arch/x86/mm/pgtable.o
  CC      mm/page-writeback.o
  CC      arch/x86/mm/physaddr.o
  CC      fs/proc/consoles.o
  CC      arch/x86/kernel/apic/ipi.o
  CC      sound/core/info.o
  CC      security/selinux/selinuxfs.o
  CC      arch/x86/mm/tlb.o
  CC      block/blk-settings.o
  AR      drivers/pci/controller/dwc/built-in.a
  CC      security/keys/request_key_auth.o
  AR      drivers/pci/controller/mobiveil/built-in.a
  CC      fs/iomap/direct-io.o
  CC      fs/iomap/fiemap.o
  AR      drivers/pci/controller/plda/built-in.a
  CC      lib/zlib_deflate/deflate_syms.o
  AR      drivers/pci/controller/built-in.a
  CC      arch/x86/mm/cpu_entry_area.o
  CC      arch/x86/mm/maccess.o
  CC      crypto/geniv.o
  CC      arch/x86/pci/acpi.o
  CC      fs/kernfs/mount.o
  CC      kernel/printk/printk.o
  CC      fs/quota/quota.o
  CC      kernel/locking/rtmutex_api.o
  AR      arch/x86/kernel/cpu/mtrr/built-in.a
  CC      arch/x86/kernel/cpu/microcode/core.o
  CC      arch/x86/pci/legacy.o
  CC      arch/x86/kernel/cpu/microcode/intel.o
  CC      fs/proc/cpuinfo.o
  CC      kernel/irq/irqdesc.o
  CC [M]  sound/pci/hda/hda_jack.o
  CC      arch/x86/events/intel/lbr.o
  CC      kernel/irq/handle.o
  CC      kernel/locking/qrwlock.o
  CC      arch/x86/mm/pgprot.o
  CC      kernel/irq/manage.o
  CC      arch/x86/pci/irq.o
  AR      lib/zlib_deflate/built-in.a
  CC      arch/x86/events/intel/p4.o
  CC      lib/crypto/sha1.o
  CC      arch/x86/kernel/apic/vector.o
  CC      kernel/rcu/update.o
  CC      kernel/rcu/sync.o
  CC      lib/crypto/mpi/mpi-cmp.o
  CC      arch/x86/events/intel/p6.o
  CC      lib/crypto/sha256.o
  CC      lib/crypto/mpi/mpi-sub-ui.o
  AR      kernel/livepatch/built-in.a
  CC      arch/x86/kernel/apic/init.o
  CC      kernel/dma/mapping.o
  AR      drivers/pci/hotplug/built-in.a
  AR      drivers/pci/switch/built-in.a
  CC      drivers/pci/access.o
  CC      io_uring/sqpoll.o
  CC      kernel/dma/direct.o
  CC      security/keys/user_defined.o
  CC      kernel/dma/ops_helpers.o
  CC      kernel/irq/spurious.o
  CC      kernel/rcu/srcutree.o
  CC      crypto/lskcipher.o
  CC      crypto/skcipher.o
  CC      net/core/scm.o
  CC      sound/core/isadma.o
  CC      sound/core/vmaster.o
  CC      fs/proc/devices.o
  CC      kernel/power/main.o
  CC      kernel/power/console.o
  CC      kernel/power/process.o
  CC      crypto/seqiv.o
  AR      kernel/sched/built-in.a
  CC      fs/iomap/seek.o
  CC      kernel/rcu/tree.o
  CC      kernel/irq/resend.o
  CC      arch/x86/mm/pgtable_32.o
  CC      fs/kernfs/inode.o
  CC      block/blk-ioc.o
  CC      crypto/echainiv.o
  CC      arch/x86/kernel/kprobes/core.o
  CC      crypto/ahash.o
  CC      arch/x86/kernel/kprobes/opt.o
  CC      arch/x86/kernel/cpu/microcode/amd.o
  CC      arch/x86/pci/common.o
  CC      arch/x86/kernel/cpu/cacheinfo.o
  AR      kernel/locking/built-in.a
  CC      lib/crypto/mpi/mpi-div.o
  CC      sound/core/ctljack.o
  AR      arch/x86/kernel/cpu/mce/built-in.a
  AR      sound/ppc/built-in.a
  CC      arch/x86/kernel/cpu/scattered.o
  AR      sound/arm/built-in.a
  CC      crypto/shash.o
  CC      fs/proc/interrupts.o
  CC      lib/crypto/mpi/mpi-inv.o
  CC      security/keys/proc.o
  CC      fs/quota/kqid.o
  CC      fs/sysfs/file.o
  AR      sound/pci/ice1712/built-in.a
  CC [M]  sound/pci/hda/hda_auto_parser.o
  CC      fs/sysfs/dir.o
  CC      kernel/dma/dummy.o
  CC      fs/sysfs/symlink.o
  CC      drivers/pci/bus.o
  CC      arch/x86/events/utils.o
  CC      arch/x86/events/intel/pt.o
  CC      lib/crypto/mpi/mpi-mod.o
  CC      arch/x86/events/intel/uncore.o
  CC      net/core/gen_stats.o
  CC      security/keys/sysctl.o
  CC      sound/core/jack.o
  CC      lib/lzo/lzo1x_compress.o
  CC      fs/iomap/swapfile.o
  CC      kernel/printk/printk_safe.o
  CC      arch/x86/kernel/cpu/topology_common.o
  CC      arch/x86/mm/iomap_32.o
  CC      arch/x86/pci/early.o
  CC      crypto/akcipher.o
  CC      crypto/sig.o
  CC      fs/kernfs/dir.o
  CC      fs/proc/loadavg.o
  CC      security/selinux/netlink.o
  CC      mm/folio-compat.o
  CC      kernel/irq/chip.o
  CC      fs/quota/netlink.o
  CC      security/selinux/nlmsgtab.o
  CC      block/blk-map.o
  CC      drivers/pci/probe.o
  CC      kernel/rcu/rcu_segcblist.o
  CC      lib/crypto/mpi/mpi-mul.o
  CC      fs/devpts/inode.o
  CC      lib/crypto/mpi/mpih-cmp.o
  CC      kernel/dma/remap.o
  LDS     arch/x86/kernel/vmlinux.lds
  AR      arch/x86/kernel/cpu/microcode/built-in.a
  CC      net/core/gen_estimator.o
  CC      kernel/power/suspend.o
  CC      crypto/kpp.o
  CC      security/integrity/iint.o
  ASN.1   crypto/rsapubkey.asn1.[ch]
  CC      security/keys/keyctl_pkey.o
  AR      arch/x86/kernel/kprobes/built-in.a
  CC      kernel/power/hibernate.o
  CC      fs/netfs/buffered_read.o
  CC      fs/netfs/buffered_write.o
  CC      lib/lzo/lzo1x_decompress_safe.o
  CC      fs/netfs/direct_read.o
  CC      fs/kernfs/file.o
  CC      fs/netfs/direct_write.o
  CC      arch/x86/kernel/apic/hw_nmi.o
  CC      block/blk-merge.o
  CC      io_uring/xattr.o
  CC      arch/x86/kernel/cpu/topology_ext.o
  CC      block/blk-timeout.o
  CC      fs/sysfs/mount.o
  CC      arch/x86/mm/hugetlbpage.o
  CC      fs/netfs/io.o
  CC      kernel/printk/nbcon.o
  CC      arch/x86/pci/bus_numa.o
  CC      arch/x86/events/rapl.o
  CC      fs/proc/meminfo.o
  CC      arch/x86/events/msr.o
  CC      sound/core/timer.o
  AR      fs/iomap/built-in.a
  CC [M]  sound/pci/hda/hda_sysfs.o
  CC      arch/x86/kernel/apic/io_apic.o
  CC      mm/readahead.o
  CC      io_uring/nop.o
  CC      io_uring/fs.o
  CC      kernel/entry/common.o
  CC      drivers/pci/host-bridge.o
  CC      lib/crypto/mpi/mpih-div.o
  CC      lib/crypto/mpi/mpih-mul.o
  ASN.1   crypto/rsaprivkey.asn1.[ch]
  AR      kernel/dma/built-in.a
  CC      kernel/module/main.o
  CC      mm/swap.o
  AR      fs/quota/built-in.a
  AR      lib/lzo/built-in.a
  CC      io_uring/splice.o
  CC      arch/x86/kernel/cpu/topology_amd.o
  CC      lib/lz4/lz4_decompress.o
  AR      security/keys/built-in.a
  CC      kernel/printk/printk_ringbuffer.o
  CC      security/integrity/integrity_audit.o
  CC      fs/ext4/balloc.o
  AR      fs/devpts/built-in.a
  CC      security/commoncap.o
  CC      kernel/irq/dummychip.o
  CC      security/selinux/netif.o
  CC      fs/ext4/bitmap.o
  CC      arch/x86/pci/amd_bus.o
  AS      arch/x86/kernel/head_32.o
  CC      crypto/rsa.o
  CC      arch/x86/kernel/head32.o
  CC      arch/x86/kernel/ebda.o
  CC      block/blk-lib.o
  CC      kernel/irq/devres.o
  CC      arch/x86/mm/dump_pagetables.o
  CC      fs/sysfs/group.o
  CC      fs/jbd2/transaction.o
  CC      fs/proc/stat.o
  CC      net/core/net_namespace.o
  CC      fs/jbd2/commit.o
  CC      arch/x86/events/intel/uncore_nhmex.o
  CC      fs/jbd2/recovery.o
  CC      arch/x86/kernel/cpu/common.o
  CC      block/blk-mq.o
  CC      block/blk-mq-tag.o
  CC [M]  sound/pci/hda/hda_controller.o
  CC      block/blk-stat.o
  CC      fs/kernfs/symlink.o
  CC      drivers/pci/remove.o
  CC      arch/x86/events/intel/uncore_snb.o
  CC      arch/x86/kernel/cpu/rdrand.o
  CC      kernel/printk/sysctl.o
  CC      arch/x86/events/intel/uncore_snbep.o
  CC      kernel/irq/autoprobe.o
  CC      lib/crypto/mpi/mpi-pow.o
  CC      fs/netfs/iterator.o
  CC      block/blk-mq-sysfs.o
  CC      fs/netfs/locking.o
  CC      io_uring/sync.o
  CC      kernel/irq/irqdomain.o
  CC      kernel/power/snapshot.o
  CC      fs/proc/uptime.o
  CC      io_uring/msg_ring.o
  AR      security/integrity/built-in.a
  CC      kernel/module/strict_rwx.o
  CC      crypto/rsa_helper.o
  CC      kernel/entry/syscall_user_dispatch.o
  CC      fs/proc/util.o
  AR      kernel/printk/built-in.a
  CC      arch/x86/kernel/platform-quirks.o
  CC      arch/x86/events/intel/uncore_discovery.o
  CC      kernel/module/kmod.o
  CC      kernel/irq/proc.o
  AR      arch/x86/pci/built-in.a
  CC      crypto/rsa-pkcs1pad.o
  CC      lib/crypto/mpi/mpiutil.o
  CC      fs/netfs/main.o
  AR      fs/sysfs/built-in.a
  CC      mm/truncate.o
  CC      arch/x86/mm/highmem_32.o
  CC      sound/core/hrtimer.o
  CC      block/blk-mq-cpumap.o
  CC      arch/x86/kernel/cpu/match.o
  AR      lib/lz4/built-in.a
  CC      drivers/pci/pci.o
  CC      sound/core/seq_device.o
  CC      kernel/irq/migration.o
  CC      fs/ext4/block_validity.o
  CC      security/selinux/netnode.o
  AR      fs/kernfs/built-in.a
  CC      fs/ext4/dir.o
  CC      kernel/irq/cpuhotplug.o
  CC      fs/jbd2/checkpoint.o
  CC      arch/x86/kernel/apic/msi.o
  CC      block/blk-mq-sched.o
  CC      block/ioctl.o
  CC      fs/proc/version.o
  CC      kernel/module/tree_lookup.o
  CC      arch/x86/kernel/apic/probe_32.o
  AR      kernel/entry/built-in.a
  CC      kernel/irq/pm.o
  CC      fs/ext4/ext4_jbd2.o
  CC      block/genhd.o
  CC      fs/netfs/misc.o
  CC      lib/zstd/zstd_decompress_module.o
  CC      arch/x86/events/intel/cstate.o
  CC      lib/xz/xz_dec_syms.o
  CC      block/ioprio.o
  AR      lib/crypto/mpi/built-in.a
  AR      lib/crypto/built-in.a
  CC      lib/zstd/decompress/huf_decompress.o
  CC [M]  sound/core/hwdep.o
  CC      lib/dim/dim.o
  CC      lib/fonts/fonts.o
  CC      lib/argv_split.o
  CC      fs/jbd2/revoke.o
  CC [M]  sound/core/pcm.o
  AR      arch/x86/mm/built-in.a
  CC      fs/netfs/objects.o
  CC      io_uring/advise.o
  CC [M]  sound/core/pcm_native.o
  CC      block/badblocks.o
  CC      crypto/acompress.o
  CC      kernel/irq/msi.o
  CC      kernel/module/kallsyms.o
  CC      net/core/secure_seq.o
  CC      fs/proc/softirqs.o
  CC      kernel/module/procfs.o
  CC      fs/proc/namespaces.o
  CC [M]  sound/pci/hda/hda_proc.o
  CC      lib/xz/xz_dec_stream.o
  CC      fs/jbd2/journal.o
  CC      drivers/video/console/dummycon.o
  CC [M]  sound/pci/hda/hda_hwdep.o
  CC      mm/vmscan.o
  AR      drivers/idle/built-in.a
  CC      lib/zstd/decompress/zstd_ddict.o
  CC      kernel/irq/affinity.o
  CC      arch/x86/kernel/cpu/bugs.o
  CC      drivers/video/backlight/backlight.o
  CC      drivers/video/aperture.o
  CC      lib/fonts/font_8x16.o
  AR      drivers/video/fbdev/core/built-in.a
  CC      lib/dim/net_dim.o
  AR      drivers/video/fbdev/omap/built-in.a
  AR      arch/x86/kernel/apic/built-in.a
  CC      block/blk-rq-qos.o
  CC      arch/x86/kernel/process_32.o
  AR      drivers/video/fbdev/omap2/omapfb/dss/built-in.a
  AR      kernel/rcu/built-in.a
  AR      drivers/video/fbdev/omap2/omapfb/displays/built-in.a
  CC      lib/zstd/decompress/zstd_decompress.o
  AR      drivers/video/fbdev/omap2/omapfb/built-in.a
  CC      lib/zstd/decompress/zstd_decompress_block.o
  CC      block/disk-events.o
  AR      drivers/video/fbdev/omap2/built-in.a
  AR      drivers/video/fbdev/built-in.a
  AR      drivers/char/ipmi/built-in.a
  CC      drivers/video/cmdline.o
  CC      drivers/acpi/acpica/dsargs.o
  CC      drivers/pnp/pnpacpi/core.o
  CC      security/selinux/netport.o
  CC      kernel/power/swap.o
  CC      drivers/pnp/pnpacpi/rsparser.o
  CC      drivers/pnp/core.o
  CC [M]  sound/core/pcm_lib.o
  CC      lib/xz/xz_dec_lzma2.o
  AR      lib/fonts/built-in.a
  CC      lib/zstd/zstd_common_module.o
  CC      lib/bug.o
  CC      lib/zstd/common/debug.o
  CC      lib/zstd/common/entropy_common.o
  CC      lib/buildid.o
  CC      lib/clz_tab.o
  CC      block/blk-ia-ranges.o
  CC      fs/proc/self.o
  CC      drivers/pnp/card.o
  CC      block/early-lookup.o
  CC      drivers/video/console/vgacon.o
  CC      io_uring/epoll.o
  CC      kernel/module/sysfs.o
  CC      crypto/scompress.o
  CC      block/bounce.o
  CC      block/bsg.o
  CC      security/selinux/status.o
  CC      lib/zstd/common/error_private.o
  CC      drivers/acpi/acpica/dscontrol.o
  CC [M]  sound/core/pcm_misc.o
  CC      io_uring/statx.o
  CC      fs/netfs/write_collect.o
  CC      block/blk-cgroup.o
  AR      arch/x86/events/intel/built-in.a
  AR      arch/x86/events/built-in.a
  CC      block/blk-ioprio.o
  CC      kernel/power/user.o
  CC      lib/zstd/common/fse_decompress.o
  CC      drivers/acpi/acpica/dsdebug.o
  CC      kernel/power/poweroff.o
  AR      drivers/video/backlight/built-in.a
  CC      kernel/irq/matrix.o
  CC      drivers/video/nomodeset.o
  CC      lib/zstd/common/zstd_common.o
  CC      drivers/video/hdmi.o
  CC      block/blk-iolatency.o
  CC      block/blk-iocost.o
  CC      fs/netfs/write_issue.o
  CC [M]  sound/pci/hda/patch_hdmi.o
  CC      block/mq-deadline.o
  CC      net/core/flow_dissector.o
  CC      block/kyber-iosched.o
  CC      drivers/acpi/acpica/dsfield.o
  CC      lib/dim/rdma_dim.o
  CC      fs/proc/thread_self.o
  CC      lib/xz/xz_dec_bcj.o
  CC      net/core/sysctl_net_core.o
  CC      arch/x86/kernel/signal.o
  CC      net/ethernet/eth.o
  AR      net/802/built-in.a
  CC      lib/cmdline.o
  CC      lib/cpumask.o
  AR      drivers/amba/built-in.a
  CC      drivers/acpi/acpica/dsinit.o
  CC      lib/ctype.o
  AR      drivers/pnp/pnpacpi/built-in.a
  AR      drivers/clk/actions/built-in.a
  CC      drivers/pnp/driver.o
  CC      io_uring/timeout.o
  AR      drivers/clk/analogbits/built-in.a
  CC      io_uring/fdinfo.o
  AR      drivers/clk/bcm/built-in.a
  CC [M]  sound/core/pcm_memory.o
  AR      drivers/clk/imgtec/built-in.a
  AR      kernel/module/built-in.a
  CC      lib/dec_and_lock.o
  AR      drivers/clk/imx/built-in.a
  CC      kernel/time/time.o
  AR      sound/sh/built-in.a
  AR      drivers/clk/ingenic/built-in.a
  CC      net/core/dev.o
  CC      kernel/time/timer.o
  AR      drivers/clk/mediatek/built-in.a
  CC      kernel/time/hrtimer.o
  CC      kernel/time/timekeeping.o
  CC      fs/ext4/extents.o
  AR      drivers/clk/microchip/built-in.a
  AR      lib/dim/built-in.a
  AR      drivers/clk/mstar/built-in.a
  CC      kernel/time/ntp.o
  AR      drivers/clk/mvebu/built-in.a
  AR      drivers/clk/ralink/built-in.a
  CC      security/selinux/ss/ebitmap.o
  CC      drivers/acpi/acpica/dsmethod.o
  AR      drivers/clk/renesas/built-in.a
  CC      crypto/algboss.o
  AR      drivers/clk/socfpga/built-in.a
  AR      lib/xz/built-in.a
  CC      lib/decompress.o
  AR      drivers/clk/sophgo/built-in.a
  CC      crypto/testmgr.o
  AR      kernel/power/built-in.a
  AR      drivers/clk/sprd/built-in.a
  CC      crypto/cmac.o
  AR      drivers/video/console/built-in.a
  CC      drivers/pnp/resource.o
  CC      fs/ext4/extents_status.o
  AR      drivers/clk/starfive/built-in.a
  CC      fs/ext4/file.o
  AR      drivers/clk/sunxi-ng/built-in.a
  CC      fs/ext4/fsmap.o
  AR      drivers/clk/ti/built-in.a
  CC      net/core/dev_addr_lists.o
  AR      drivers/clk/versatile/built-in.a
  CC      drivers/pci/pci-driver.o
  AR      drivers/clk/xilinx/built-in.a
  AR      drivers/clk/built-in.a
  CC      fs/proc/proc_sysctl.o
  CC      arch/x86/kernel/cpu/aperfmperf.o
  CC      fs/proc/proc_net.o
  CC      security/selinux/ss/hashtab.o
  CC      security/selinux/ss/symtab.o
  AR      drivers/video/built-in.a
  CC      drivers/dma/dw/core.o
  CC      drivers/dma/dw/dw.o
  CC      drivers/dma/hsu/hsu.o
  AR      drivers/dma/idxd/built-in.a
  AR      drivers/dma/mediatek/built-in.a
  CC      drivers/acpi/acpica/dsmthdat.o
  CC      fs/proc/kcore.o
  CC      drivers/pnp/manager.o
  CC      arch/x86/kernel/signal_32.o
  CC [M]  sound/core/memalloc.o
  CC      arch/x86/kernel/cpu/cpuid-deps.o
  CC      block/blk-mq-pci.o
  AR      drivers/soc/apple/built-in.a
  AR      drivers/dma/qcom/built-in.a
  AR      kernel/irq/built-in.a
  AR      drivers/soc/aspeed/built-in.a
  AR      drivers/soc/bcm/built-in.a
  CC [M]  sound/pci/hda/hda_eld.o
  AR      drivers/soc/fsl/built-in.a
  AR      sound/synth/emux/built-in.a
  AR      sound/synth/built-in.a
  AR      drivers/soc/fujitsu/built-in.a
  CC      security/selinux/ss/sidtab.o
  AR      drivers/dma/stm32/built-in.a
  CC      security/selinux/ss/avtab.o
  AR      drivers/soc/hisilicon/built-in.a
  CC      security/selinux/ss/policydb.o
  AR      drivers/soc/imx/built-in.a
  AR      drivers/soc/ixp4xx/built-in.a
  CC      drivers/dma/dw/idma32.o
  CC      drivers/dma/dw/acpi.o
  AR      fs/netfs/built-in.a
  AR      drivers/soc/loongson/built-in.a
  CC      net/sched/sch_generic.o
  AR      lib/zstd/built-in.a
  CC      net/netlink/af_netlink.o
  AR      drivers/soc/mediatek/built-in.a
  CC      lib/decompress_bunzip2.o
  AR      drivers/soc/microchip/built-in.a
  AR      drivers/soc/nuvoton/built-in.a
  CC      kernel/time/clocksource.o
  CC      drivers/acpi/acpica/dsobject.o
  AR      drivers/soc/pxa/built-in.a
  CC      net/core/dst.o
  AR      net/bpf/built-in.a
  AR      drivers/soc/amlogic/built-in.a
  CC      net/core/netevent.o
  CC      io_uring/cancel.o
  AR      drivers/soc/qcom/built-in.a
  CC      net/core/neighbour.o
  CC      arch/x86/kernel/traps.o
  AR      drivers/soc/renesas/built-in.a
  AR      drivers/soc/rockchip/built-in.a
  CC      crypto/hmac.o
  AR      drivers/soc/sunxi/built-in.a
  CC      kernel/time/jiffies.o
  AR      drivers/soc/ti/built-in.a
  CC      arch/x86/kernel/cpu/umwait.o
  AR      drivers/soc/xilinx/built-in.a
  CC      block/blk-mq-virtio.o
  AR      drivers/soc/built-in.a
  AR      fs/jbd2/built-in.a
  CC      io_uring/waitid.o
  CC      io_uring/register.o
  CC      drivers/pnp/support.o
  CC      lib/decompress_inflate.o
  CC      kernel/futex/core.o
  CC      drivers/virtio/virtio.o
  AR      net/ethernet/built-in.a
  AR      drivers/dma/hsu/built-in.a
  CC      mm/shrinker.o
  CC      kernel/futex/syscalls.o
  CC      security/selinux/ss/services.o
  CC      drivers/virtio/virtio_ring.o
  CC      arch/x86/kernel/idt.o
  CC      drivers/acpi/acpica/dsopcode.o
  CC      drivers/acpi/acpica/dspkginit.o
  CC      drivers/acpi/acpica/dsutils.o
  AR      drivers/dma/ti/built-in.a
  CC [M]  sound/pci/hda/hda_intel.o
  CC      fs/ext4/fsync.o
  CC      fs/ext4/hash.o
  CC      drivers/pci/search.o
  CC      security/selinux/ss/conditional.o
  CC      net/netlink/genetlink.o
  CC      fs/proc/vmcore.o
  CC      block/blk-mq-debugfs.o
  AR      drivers/dma/dw/built-in.a
  CC [M]  sound/core/pcm_timer.o
  AR      drivers/dma/xilinx/built-in.a
  CC      drivers/dma/dmaengine.o
  CC      mm/shmem.o
  LD [M]  sound/pci/hda/snd-hda-codec.o
  CC      drivers/acpi/acpica/dswexec.o
  CC      net/core/rtnetlink.o
  CC      drivers/pnp/interface.o
  CC      lib/decompress_unlz4.o
  CC      crypto/crypto_null.o
  MKCAP   arch/x86/kernel/cpu/capflags.c
  CC      drivers/acpi/acpica/dswload.o
  CC      arch/x86/kernel/cpu/powerflags.o
  CC      kernel/cgroup/cgroup.o
  CC      drivers/pnp/quirks.o
  CC      drivers/pnp/system.o
  CC      kernel/cgroup/rstat.o
  CC      fs/proc/kmsg.o
  CC      crypto/md5.o
  CC      arch/x86/kernel/cpu/topology.o
  CC      drivers/pci/rom.o
  CC      kernel/time/timer_list.o
  CC      kernel/time/timeconv.o
  CC      io_uring/truncate.o
  AR      sound/usb/misc/built-in.a
  AR      sound/usb/usx2y/built-in.a
  AR      sound/usb/caiaq/built-in.a
  CC      drivers/pci/setup-res.o
  AR      sound/usb/6fire/built-in.a
  AR      sound/firewire/built-in.a
  CC      kernel/time/timecounter.o
  CC      drivers/pci/irq.o
  AR      sound/usb/hiface/built-in.a
  CC      mm/util.o
  AR      sound/usb/bcd2000/built-in.a
  CC      crypto/sha256_generic.o
  CC      fs/ramfs/inode.o
  AR      sound/usb/built-in.a
  CC      drivers/acpi/acpica/dswload2.o
  CC      drivers/pci/vpd.o
  CC      fs/ext4/ialloc.o
  AR      sound/sparc/built-in.a
  CC      fs/ramfs/file-mmu.o
  AR      sound/spi/built-in.a
  CC      mm/mmzone.o
  AR      sound/parisc/built-in.a
  LD [M]  sound/core/snd-hwdep.o
  CC      drivers/pci/setup-bus.o
  LD [M]  sound/core/snd-pcm.o
  AR      sound/core/built-in.a
  CC      lib/decompress_unlzma.o
  CC      kernel/cgroup/namespace.o
  CC      kernel/cgroup/cgroup-v1.o
  CC      kernel/futex/pi.o
  CC      security/lsm_syscalls.o
  CC      security/min_addr.o
  CC      kernel/time/alarmtimer.o
  CC      security/security.o
  CC      crypto/sha512_generic.o
  CC      crypto/sha3_generic.o
  CC      block/blk-pm.o
  CC      lib/decompress_unlzo.o
  CC      drivers/acpi/acpica/dswscope.o
  AR      drivers/pnp/built-in.a
  CC      fs/proc/page.o
  CC      drivers/tty/vt/vt_ioctl.o
  CC      drivers/char/hw_random/core.o
  CC      drivers/char/agp/backend.o
  AR      drivers/iommu/amd/built-in.a
  AR      drivers/iommu/intel/built-in.a
  AR      drivers/iommu/arm/arm-smmu/built-in.a
  AR      drivers/iommu/arm/arm-smmu-v3/built-in.a
  AR      drivers/iommu/arm/built-in.a
  AR      drivers/iommu/iommufd/built-in.a
  CC      drivers/iommu/iommu.o
  CC      drivers/char/agp/generic.o
  CC      drivers/pci/vc.o
  CC      drivers/pci/mmap.o
  CC      net/sched/sch_mq.o
  CC      drivers/dma/virt-dma.o
  CC      drivers/virtio/virtio_anchor.o
  CC      drivers/tty/vt/vc_screen.o
  CC      security/lsm_audit.o
  AR      drivers/gpu/host1x/built-in.a
  CC      drivers/pci/devres.o
  AR      fs/ramfs/built-in.a
  LD [M]  sound/pci/hda/snd-hda-codec-hdmi.o
  CC      drivers/acpi/acpica/dswstate.o
  LD [M]  sound/pci/hda/snd-hda-intel.o
  CC      drivers/acpi/acpica/evevent.o
  AR      sound/pci/korg1212/built-in.a
  AR      sound/pci/mixart/built-in.a
  AR      sound/pci/nm256/built-in.a
  AR      sound/pci/oxygen/built-in.a
  CC      kernel/cgroup/freezer.o
  CC      drivers/pci/proc.o
  CC      io_uring/memmap.o
  AR      sound/pci/pcxhr/built-in.a
  CC      arch/x86/kernel/cpu/proc.o
  AR      sound/pci/riptide/built-in.a
  AR      sound/pci/rme9652/built-in.a
  CC      drivers/connector/cn_queue.o
  CC      kernel/cgroup/legacy_freezer.o
  CC      kernel/futex/requeue.o
  AR      sound/pci/trident/built-in.a
  CC      drivers/connector/connector.o
  AR      drivers/gpu/drm/tests/built-in.a
  AR      sound/pci/ymfpci/built-in.a
  AR      drivers/gpu/drm/arm/built-in.a
  AR      sound/pci/vx222/built-in.a
  AR      sound/pci/built-in.a
  CC      drivers/gpu/drm/display/drm_display_helper_mod.o
  CC      drivers/base/power/sysfs.o
  CC      drivers/tty/vt/selection.o
  CC      lib/decompress_unxz.o
  CC      block/holder.o
  AR      sound/pcmcia/vx/built-in.a
  CC      fs/hugetlbfs/inode.o
  AR      sound/pcmcia/pdaudiocf/built-in.a
  CC      fs/fat/cache.o
  AR      sound/pcmcia/built-in.a
  AR      sound/mips/built-in.a
  AR      sound/soc/built-in.a
  AR      sound/atmel/built-in.a
  AR      sound/hda/built-in.a
  CC      security/selinux/ss/mls.o
  CC      net/netlink/policy.o
  CC [M]  sound/hda/hda_bus_type.o
  CC [M]  sound/hda/hdac_bus.o
  CC      crypto/ecb.o
  AR      fs/proc/built-in.a
  CC      drivers/virtio/virtio_pci_modern_dev.o
  CC      drivers/acpi/acpica/evgpe.o
  CC      kernel/cgroup/pids.o
  CC      drivers/char/hw_random/intel-rng.o
  CC      crypto/cbc.o
  CC      crypto/ctr.o
  CC      kernel/time/posix-timers.o
  CC      drivers/dma/acpi-dma.o
  CC      drivers/gpu/drm/display/drm_dp_dual_mode_helper.o
  CC      drivers/gpu/drm/display/drm_dp_helper.o
  CC      drivers/gpu/drm/display/drm_dp_mst_topology.o
  CC      drivers/gpu/drm/display/drm_dsc_helper.o
  CC      drivers/gpu/drm/display/drm_hdcp_helper.o
  CC      drivers/gpu/drm/display/drm_hdmi_helper.o
  CC      lib/decompress_unzstd.o
  CC      drivers/base/firmware_loader/builtin/main.o
  CC      arch/x86/kernel/irq.o
  CC      net/core/utils.o
  CC      drivers/base/power/generic_ops.o
  CC      io_uring/io-wq.o
  CC      net/core/link_watch.o
  CC      drivers/base/regmap/regmap.o
  CC      kernel/futex/waitwake.o
  CC      crypto/gcm.o
  CC      drivers/char/agp/isoch.o
  CC      drivers/acpi/acpica/evgpeblk.o
  CC      net/sched/sch_frag.o
  CC      drivers/tty/vt/keyboard.o
  AR      block/built-in.a
  CC      drivers/pci/pci-sysfs.o
  CC      drivers/char/agp/amd64-agp.o
  CC      io_uring/futex.o
  CC      arch/x86/kernel/cpu/feat_ctl.o
  CC      mm/vmstat.o
  CC      security/selinux/ss/context.o
  CC      kernel/time/posix-cpu-timers.o
  AR      drivers/base/firmware_loader/builtin/built-in.a
  CC      drivers/gpu/drm/ttm/ttm_tt.o
  CC      drivers/base/firmware_loader/main.o
  CC      drivers/gpu/drm/ttm/ttm_bo.o
  CC      fs/fat/dir.o
  CC      fs/fat/fatent.o
  CC [M]  sound/hda/hdac_device.o
  CC      drivers/char/hw_random/amd-rng.o
  CC      drivers/virtio/virtio_pci_legacy_dev.o
  CC      drivers/connector/cn_proc.o
  AR      net/netlink/built-in.a
  CC      drivers/base/power/common.o
  CC      fs/fat/file.o
  CC      lib/dump_stack.o
  AR      drivers/dma/built-in.a
  CC      drivers/char/agp/intel-agp.o
  CC      drivers/acpi/acpica/evgpeinit.o
  CC      mm/backing-dev.o
  CC      net/ethtool/ioctl.o
  CC      drivers/block/loop.o
  CC      net/ethtool/common.o
  CC      drivers/gpu/drm/display/drm_scdc_helper.o
  CC      net/ethtool/netlink.o
  CC      drivers/iommu/iommu-traces.o
  CC      mm/mm_init.o
  AR      kernel/futex/built-in.a
  AR      fs/hugetlbfs/built-in.a
  CC      drivers/gpu/drm/ttm/ttm_bo_util.o
  AR      drivers/base/test/built-in.a
  CC      net/ethtool/bitset.o
  CC      drivers/block/virtio_blk.o
  CC      security/selinux/netlabel.o
  CC      fs/ext4/indirect.o
  CC      drivers/acpi/acpica/evgpeutil.o
  CC      drivers/base/power/qos.o
  CC      drivers/char/agp/intel-gtt.o
  CC      drivers/char/hw_random/geode-rng.o
  CC      drivers/virtio/virtio_pci_modern.o
  CC      crypto/ccm.o
  CC      drivers/char/hw_random/via-rng.o
  CC      net/core/filter.o
  CC      crypto/aes_generic.o
  CC      lib/earlycpio.o
  CC      fs/isofs/namei.o
  CC      fs/nfs/client.o
  CC      fs/exportfs/expfs.o
  CC      lib/extable.o
  CC      net/sched/sch_api.o
  AR      drivers/base/firmware_loader/built-in.a
  CC      lib/flex_proportions.o
  CC [M]  sound/hda/hdac_sysfs.o
  CC      fs/lockd/clntlock.o
  CC      crypto/crc32c_generic.o
  CC      drivers/pci/slot.o
  CC      drivers/acpi/acpica/evglock.o
  CC      kernel/time/posix-clock.o
  CC      drivers/tty/hvc/hvc_console.o
  CC      drivers/tty/serial/8250/8250_core.o
  CC      io_uring/napi.o
  CC      drivers/base/power/runtime.o
  CC      net/core/sock_diag.o
  AR      drivers/gpu/vga/built-in.a
  CC      drivers/tty/serial/8250/8250_platform.o
  AR      drivers/misc/eeprom/built-in.a
  AR      drivers/misc/cb710/built-in.a
  AR      drivers/misc/ti-st/built-in.a
  AR      drivers/acpi/pmic/built-in.a
  AR      drivers/misc/lis3lv02d/built-in.a
  CC      drivers/acpi/dptf/int340x_thermal.o
  AR      drivers/connector/built-in.a
  AR      drivers/misc/cardreader/built-in.a
  CC      drivers/tty/serial/8250/8250_pnp.o
  AR      drivers/misc/keba/built-in.a
  AR      drivers/misc/built-in.a
  CC      fs/nls/nls_base.o
  AR      drivers/char/hw_random/built-in.a
  CC      drivers/gpu/drm/ttm/ttm_bo_vm.o
  CC      drivers/acpi/x86/apple.o
  CC      arch/x86/kernel/irq_32.o
  CC      arch/x86/kernel/dumpstack_32.o
  CC      fs/nls/nls_cp437.o
  CC      drivers/tty/vt/vt.o
  CC      lib/idr.o
  CC      drivers/iommu/iommu-sysfs.o
  CC      fs/nls/nls_ascii.o
  CC      fs/fat/inode.o
  CC      fs/nls/nls_iso8859-1.o
  CC      drivers/acpi/acpica/evhandler.o
  CC      kernel/cgroup/rdma.o
  CC      drivers/virtio/virtio_pci_common.o
  AR      fs/exportfs/built-in.a
  CC      drivers/virtio/virtio_pci_legacy.o
  CC      fs/isofs/inode.o
  CC      drivers/acpi/x86/cmos_rtc.o
  CC      drivers/iommu/dma-iommu.o
  CC      crypto/authenc.o
  COPY    drivers/tty/vt/defkeymap.c
  CC      fs/isofs/dir.o
  CC      drivers/acpi/acpica/evmisc.o
  AR      drivers/char/agp/built-in.a
  CC      drivers/char/mem.o
  CC      mm/percpu.o
  AR      drivers/acpi/dptf/built-in.a
  AR      drivers/block/built-in.a
  AR      security/selinux/built-in.a
  CC      drivers/acpi/acpica/evregion.o
  CC      security/device_cgroup.o
  CC      net/sched/sch_blackhole.o
  CC      fs/nls/nls_utf8.o
  CC      drivers/pci/pci-acpi.o
  CC      net/sched/cls_api.o
  CC      net/sched/act_api.o
  CC [M]  sound/hda/hdac_regmap.o
  CC      kernel/time/itimer.o
  CC      net/sched/sch_fifo.o
  CC      net/ethtool/strset.o
  CC      net/netfilter/core.o
  CC      kernel/cgroup/cpuset.o
  CC      drivers/acpi/acpica/evrgnini.o
  CC      lib/irq_regs.o
  CC      net/netfilter/nf_log.o
  AR      drivers/tty/hvc/built-in.a
  CC      drivers/gpu/drm/ttm/ttm_module.o
  CC      net/ipv4/netfilter/nf_defrag_ipv4.o
  CC      drivers/gpu/drm/ttm/ttm_execbuf_util.o
  CC      drivers/gpu/drm/ttm/ttm_range_manager.o
  AR      fs/unicode/built-in.a
  CC      drivers/tty/serial/8250/8250_rsa.o
  CC      drivers/acpi/acpica/evsci.o
  CC      net/ipv4/route.o
  AR      drivers/gpu/drm/display/built-in.a
  CC      net/xfrm/xfrm_policy.o
  CC      lib/is_single_threaded.o
  AR      fs/nls/built-in.a
  CC      net/unix/af_unix.o
  CC      lib/klist.o
  CC      drivers/base/regmap/regcache.o
  CC      lib/kobject.o
  CC      fs/lockd/clntproc.o
  CC      net/unix/garbage.o
  CC      drivers/acpi/x86/lpss.o
  CC      kernel/time/clockevents.o
  CC      drivers/base/power/wakeirq.o
  AR      sound/x86/built-in.a
  CC      drivers/tty/serial/serial_core.o
  AR      sound/xen/built-in.a
  CC      drivers/tty/serial/serial_base_bus.o
  CC      net/ethtool/linkinfo.o
  CC      drivers/acpi/acpica/evxface.o
  CC      fs/nfs/dir.o
  CC      fs/nfs/file.o
  CC      drivers/virtio/virtio_pci_admin_legacy_io.o
  CC      drivers/acpi/acpica/evxfevnt.o
  CC      crypto/authencesn.o
  AR      io_uring/built-in.a
  CC      arch/x86/kernel/cpu/intel.o
  CC [M]  sound/hda/hdac_controller.o
  CC      arch/x86/kernel/time.o
  CC      drivers/char/random.o
  AR      sound/virtio/built-in.a
  CC      fs/ext4/inline.o
  CC      fs/nfs/getroot.o
  CC      drivers/acpi/acpica/evxfgpe.o
  CC      drivers/gpu/drm/ttm/ttm_resource.o
  CC      fs/isofs/util.o
  CC      drivers/base/power/main.o
  CC      kernel/time/tick-common.o
  CC      drivers/char/misc.o
  CC      fs/fat/misc.o
  CC      drivers/tty/serial/8250/8250_port.o
  CC      lib/kobject_uevent.o
  AR      security/built-in.a
  CC      drivers/virtio/virtio_input.o
  CC      net/sched/cls_cgroup.o
  CC      kernel/time/tick-broadcast.o
  CC      lib/logic_pio.o
  CC      drivers/pci/iomap.o
  CC      arch/x86/kernel/cpu/tsx.o
  CC      drivers/iommu/iova.o
  CC      mm/slab_common.o
  CC      drivers/tty/serial/serial_ctrl.o
  CC      drivers/acpi/x86/s2idle.o
  CC      drivers/tty/serial/serial_port.o
  CC      drivers/tty/serial/earlycon.o
  CC      drivers/pci/quirks.o
  CC      drivers/base/regmap/regcache-rbtree.o
  CC      net/ipv4/netfilter/nf_reject_ipv4.o
  CC      drivers/base/regmap/regcache-flat.o
  CC      drivers/acpi/acpica/evxfregn.o
  CC      net/netfilter/nf_queue.o
  CC      net/ethtool/linkmodes.o
  CC      fs/isofs/rock.o
  CC      net/unix/sysctl_net_unix.o
  CC      arch/x86/kernel/cpu/intel_epb.o
  CC      arch/x86/kernel/cpu/amd.o
  CC      arch/x86/kernel/cpu/hygon.o
  CC [M]  sound/hda/hdac_stream.o
  CC      crypto/lzo.o
  CC      kernel/time/tick-broadcast-hrtimer.o
  CC [M]  sound/hda/array.o
  CC [M]  sound/hda/hdmi_chmap.o
  CC      fs/fat/nfs.o
  CC      fs/lockd/clntxdr.o
  CC      drivers/acpi/acpica/exconcat.o
  CC      drivers/gpu/drm/ttm/ttm_pool.o
  CC      drivers/acpi/acpica/exconfig.o
  CC      drivers/tty/vt/consolemap.o
  CC      drivers/virtio/virtio_dma_buf.o
  CC      drivers/pci/pci-label.o
  CC      net/ipv4/inetpeer.o
  CC      net/ipv4/netfilter/ip_tables.o
  CC      mm/compaction.o
  CC      drivers/acpi/acpica/exconvrt.o
  CC      net/ipv4/protocol.o
  AR      drivers/iommu/built-in.a
  CC      net/ipv4/ip_input.o
  CC      drivers/base/regmap/regcache-maple.o
  AR      drivers/mfd/built-in.a
  CC      net/ipv4/ip_fragment.o
  CC      fs/isofs/export.o
  CC      drivers/char/virtio_console.o
  CC      drivers/acpi/x86/utils.o
  CC      mm/show_mem.o
  CC      fs/nfs/inode.o
  CC      drivers/pci/vgaarb.o
  CC      kernel/time/tick-oneshot.o
  HOSTCC  drivers/tty/vt/conmakehash
  CC      lib/maple_tree.o
  CC      drivers/acpi/acpica/excreate.o
  CC      crypto/lzo-rle.o
  CC      kernel/cgroup/misc.o
  AR      drivers/nfc/built-in.a
  CC      kernel/trace/trace_clock.o
  CC      fs/ext4/inode.o
  CC      drivers/tty/serial/8250/8250_dma.o
  CC      fs/ext4/ioctl.o
  CC      drivers/acpi/acpica/exdebug.o
  CC      net/ethtool/rss.o
  AR      drivers/virtio/built-in.a
  CC      kernel/cgroup/debug.o
  CC      fs/isofs/joliet.o
  CC      fs/fat/namei_vfat.o
  AR      drivers/dax/hmem/built-in.a
  CC      arch/x86/kernel/cpu/centaur.o
  CC      drivers/base/power/wakeup.o
  AR      drivers/dax/built-in.a
  AR      drivers/tty/ipwireless/built-in.a
  CC      drivers/base/power/wakeup_stats.o
  CC      kernel/trace/ring_buffer.o
  CC      net/sched/ematch.o
  CC      kernel/time/tick-sched.o
  CC      drivers/acpi/x86/blacklist.o
  CC      drivers/dma-buf/dma-buf.o
  AR      drivers/cxl/core/built-in.a
  CC      net/netfilter/nf_sockopt.o
  AR      drivers/cxl/built-in.a
  CC      net/netfilter/utils.o
  CC [M]  sound/hda/trace.o
  CC      drivers/tty/vt/defkeymap.o
  CC      drivers/base/regmap/regmap-debugfs.o
  CC [M]  sound/hda/hdac_component.o
  CC      net/ipv4/netfilter/iptable_filter.o
  AR      net/unix/built-in.a
  CC      kernel/trace/trace.o
  CC      drivers/gpu/drm/ttm/ttm_device.o
  CC      drivers/base/power/trace.o
  CC      net/ipv4/ip_forward.o
  CC      net/netfilter/nfnetlink.o
  CC      fs/lockd/host.o
  CC      drivers/acpi/acpica/exdump.o
  CC      drivers/dma-buf/dma-fence.o
  CONMK   drivers/tty/vt/consolemap_deftbl.c
  CC      drivers/tty/vt/consolemap_deftbl.o
  CC      fs/isofs/compress.o
  CC      fs/nfs/super.o
  AR      drivers/tty/vt/built-in.a
  CC      drivers/tty/tty_io.o
  CC      crypto/rng.o
  CC      crypto/drbg.o
  CC      crypto/jitterentropy.o
  CC      arch/x86/kernel/cpu/transmeta.o
  CC      mm/shmem_quota.o
  CC      drivers/tty/serial/8250/8250_dwlib.o
  AR      drivers/acpi/x86/built-in.a
  CC      drivers/acpi/tables.o
  CC      drivers/tty/n_tty.o
  CC      fs/nfs/io.o
  CC      fs/fat/namei_msdos.o
  CC      drivers/acpi/acpica/exfield.o
  CC      drivers/acpi/acpica/exfldio.o
  AR      drivers/pci/built-in.a
  CC      drivers/tty/serial/8250/8250_pcilib.o
  AR      kernel/cgroup/built-in.a
  CC      net/ipv4/netfilter/iptable_mangle.o
  CC      drivers/dma-buf/dma-fence-array.o
  CC      drivers/char/hpet.o
  CC      fs/ext4/mballoc.o
  CC [M]  sound/hda/hdac_i915.o
  CC      fs/nfs/direct.o
  CC      net/ethtool/linkstate.o
  CC      drivers/gpu/drm/ttm/ttm_sys_manager.o
  CC      drivers/macintosh/mac_hid.o
  AR      drivers/base/regmap/built-in.a
  CC      net/xfrm/xfrm_state.o
  AR      drivers/scsi/pcmcia/built-in.a
  CC      arch/x86/kernel/cpu/zhaoxin.o
  CC      drivers/scsi/scsi.o
  CC      kernel/time/timer_migration.o
  CC      arch/x86/kernel/cpu/vortex.o
  CC      kernel/bpf/core.o
  AR      net/sched/built-in.a
  CC      kernel/events/core.o
  CC      drivers/base/component.o
  CC      kernel/events/ring_buffer.o
  AR      fs/isofs/built-in.a
  CC      net/ipv6/netfilter/ip6_tables.o
  CC      net/ipv4/netfilter/ipt_REJECT.o
  CC      lib/memcat_p.o
  CC      drivers/dma-buf/dma-fence-chain.o
  CC      net/packet/af_packet.o
  CC      drivers/acpi/acpica/exmisc.o
  CC      crypto/jitterentropy-kcapi.o
  AR      drivers/base/power/built-in.a
  CC      drivers/tty/serial/8250/8250_early.o
  AR      net/dsa/built-in.a
  CC      drivers/dma-buf/dma-fence-unwrap.o
  CC      kernel/fork.o
  CC      fs/autofs/init.o
  CC      net/netfilter/nfnetlink_log.o
  CC      fs/nfs/pagelist.o
  CC      drivers/gpu/drm/ttm/ttm_agp_backend.o
  CC      drivers/dma-buf/dma-resv.o
  CC      arch/x86/kernel/cpu/perfctr-watchdog.o
  CC      drivers/gpu/drm/i915/i915_config.o
  CC      lib/nmi_backtrace.o
  CC      fs/lockd/svc.o
  AR      drivers/macintosh/built-in.a
  CC [M]  net/ipv4/netfilter/iptable_nat.o
  CC [M]  sound/hda/intel-dsp-config.o
  CC      net/ipv4/ip_options.o
  CC      kernel/trace/trace_output.o
  CC      kernel/trace/trace_seq.o
  AR      fs/fat/built-in.a
  CC      lib/objpool.o
  CC      drivers/acpi/acpica/exmutex.o
  CC      arch/x86/kernel/cpu/vmware.o
  CC      drivers/char/nvram.o
  CC      drivers/gpu/drm/i915/i915_driver.o
  CC      net/sunrpc/auth_gss/auth_gss.o
  CC      crypto/ghash-generic.o
  CC      mm/interval_tree.o
  CC      net/sunrpc/auth_gss/gss_generic_token.o
  CC      drivers/dma-buf/sync_file.o
  CC      drivers/base/core.o
  CC      lib/plist.o
  CC      net/ethtool/debug.o
  CC      drivers/tty/serial/8250/8250_exar.o
  CC      lib/radix-tree.o
  CC      drivers/tty/tty_ioctl.o
  CC      net/sunrpc/clnt.o
  CC      fs/autofs/inode.o
  CC      drivers/acpi/acpica/exnames.o
  CC      arch/x86/kernel/cpu/hypervisor.o
  AR      drivers/gpu/drm/ttm/built-in.a
  CC      fs/9p/vfs_super.o
  AR      fs/hostfs/built-in.a
  AR      drivers/gpu/drm/omapdrm/built-in.a
  AR      drivers/gpu/drm/renesas/rcar-du/built-in.a
  AR      drivers/gpu/drm/tilcdc/built-in.a
  CC      crypto/hash_info.o
  AR      drivers/gpu/drm/renesas/rz-du/built-in.a
  CC      net/netfilter/nf_conntrack_core.o
  AR      drivers/gpu/drm/renesas/built-in.a
  CC      drivers/gpu/drm/virtio/virtgpu_drv.o
  AR      drivers/gpu/drm/imx/built-in.a
  CC      net/ipv6/af_inet6.o
  CC      net/ipv6/anycast.o
  CC      fs/autofs/root.o
  AR      net/wireless/tests/built-in.a
  CC [M]  sound/hda/intel-nhlt.o
  CC      crypto/rsapubkey.asn1.o
  CC      net/wireless/core.o
  CC      fs/autofs/symlink.o
  CC      crypto/rsaprivkey.asn1.o
  AR      crypto/built-in.a
  CC      drivers/scsi/hosts.o
  CC      fs/autofs/waitq.o
  CC      fs/9p/vfs_inode.o
  CC      drivers/acpi/acpica/exoparg1.o
  CC      arch/x86/kernel/cpu/mshyperv.o
  CC      net/wireless/sysfs.o
  CC      drivers/acpi/acpica/exoparg2.o
  AR      drivers/dma-buf/built-in.a
  AR      drivers/char/built-in.a
  CC      net/ipv6/ip6_output.o
  CC      fs/autofs/expire.o
  CC      fs/ext4/migrate.o
  CC      mm/list_lru.o
  AR      net/ipv4/netfilter/built-in.a
  CC      fs/lockd/svclock.o
  CC      drivers/gpu/drm/i915/i915_drm_client.o
  CC      net/ipv6/netfilter/ip6table_filter.o
  CC      kernel/time/vsyscall.o
  CC      net/ipv4/ip_output.o
  CC [M]  sound/hda/intel-sdw-acpi.o
  CC      net/wireless/radiotap.o
  CC      fs/9p/vfs_inode_dotl.o
  CC      net/ipv6/netfilter/ip6table_mangle.o
  CC      net/ipv6/netfilter/nf_defrag_ipv6_hooks.o
  CC      net/sunrpc/auth_gss/gss_mech_switch.o
  CC      net/ethtool/wol.o
  CC      drivers/tty/serial/8250/8250_lpss.o
  CC      drivers/gpu/drm/virtio/virtgpu_kms.o
  CC      net/ethtool/features.o
  CC      drivers/tty/tty_ldisc.o
  CC      drivers/acpi/acpica/exoparg3.o
  CC      net/ipv4/ip_sockglue.o
  CC      fs/autofs/dev-ioctl.o
  CC      mm/workingset.o
  CC      kernel/time/timekeeping_debug.o
  CC      drivers/scsi/scsi_ioctl.o
  CC      fs/nfs/read.o
  CC      net/core/dev_ioctl.o
  CC      drivers/gpu/drm/i915/i915_getparam.o
  CC      kernel/time/namespace.o
  LD [M]  sound/hda/snd-hda-core.o
  LD [M]  sound/hda/snd-intel-dspcfg.o
  LD [M]  sound/hda/snd-intel-sdw-acpi.o
  AR      drivers/gpu/drm/i2c/built-in.a
  CC      sound/sound_core.o
  CC      drivers/acpi/osi.o
  CC      arch/x86/kernel/cpu/debugfs.o
  CC      drivers/acpi/acpica/exoparg6.o
  AR      drivers/gpu/drm/panel/built-in.a
  CC      fs/9p/vfs_addr.o
  CC      fs/nfs/symlink.o
  CC      kernel/events/callchain.o
  CC      kernel/exec_domain.o
  CC      drivers/tty/serial/8250/8250_mid.o
  AR      kernel/bpf/built-in.a
  CC      kernel/panic.o
  CC      kernel/trace/trace_stat.o
  CC      fs/ext4/mmp.o
  CC      drivers/gpu/drm/i915/i915_ioctl.o
  CC      sound/last.o
  CC      drivers/gpu/drm/virtio/virtgpu_gem.o
  CC      kernel/cpu.o
  CC      net/wireless/util.o
  CC      kernel/exit.o
  CC      drivers/acpi/acpica/exprep.o
  CC      kernel/softirq.o
  CC      kernel/resource.o
  CC      net/ipv6/ip6_input.o
  CC      net/xfrm/xfrm_hash.o
  CC      fs/lockd/svcshare.o
  CC      net/ipv6/addrconf.o
  CC      mm/debug.o
  CC      net/ethtool/privflags.o
  CC      net/ipv6/addrlabel.o
  CC      fs/ext4/move_extent.o
  AR      fs/autofs/built-in.a
  CC      fs/9p/vfs_file.o
  CC      drivers/scsi/scsicam.o
  CC      arch/x86/kernel/cpu/capflags.o
  AR      arch/x86/kernel/cpu/built-in.a
  AR      kernel/time/built-in.a
  CC      net/ethtool/rings.o
  CC      net/ipv6/netfilter/nf_conntrack_reasm.o
  CC      net/ethtool/channels.o
  CC      arch/x86/kernel/ioport.o
  CC      fs/ext4/namei.o
  AR      net/packet/built-in.a
  AR      sound/built-in.a
  CC      drivers/acpi/osl.o
  AR      net/mac80211/tests/built-in.a
  CC      net/mac80211/main.o
  CC      drivers/acpi/acpica/exregion.o
  CC      net/mac80211/status.o
  CC      net/sunrpc/auth_gss/svcauth_gss.o
  CC      net/sunrpc/auth_gss/gss_rpc_upcall.o
  CC      drivers/base/bus.o
  CC      net/mac80211/driver-ops.o
  CC      net/ipv6/route.o
  CC      drivers/tty/serial/8250/8250_pci.o
  CC      net/netfilter/nf_conntrack_standalone.o
  CC      net/xfrm/xfrm_input.o
  CC      kernel/trace/trace_printk.o
  CC      fs/ext4/page-io.o
  CC      drivers/gpu/drm/i915/i915_irq.o
  CC      drivers/gpu/drm/i915/i915_mitigations.o
  CC      drivers/gpu/drm/virtio/virtgpu_vram.o
  CC      drivers/acpi/acpica/exresnte.o
  CC      lib/ratelimit.o
  CC      net/core/tso.o
  CC      drivers/scsi/scsi_error.o
  CC      fs/9p/vfs_dir.o
  CC      drivers/gpu/drm/i915/i915_module.o
  CC      mm/gup.o
  CC      arch/x86/kernel/dumpstack.o
  CC      lib/rbtree.o
  CC      net/ethtool/coalesce.o
  CC      fs/lockd/svcproc.o
  CC      fs/nfs/unlink.o
  CC      fs/ext4/readpage.o
  CC      kernel/sysctl.o
  CC      net/ipv4/inet_hashtables.o
  CC      drivers/acpi/acpica/exresolv.o
  CC      kernel/capability.o
  CC      kernel/ptrace.o
  CC      drivers/base/dd.o
  CC      net/sunrpc/auth_gss/gss_rpc_xdr.o
  CC      net/ethtool/pause.o
  CC      drivers/gpu/drm/i915/i915_params.o
  CC      mm/mmap_lock.o
  CC      lib/seq_buf.o
  CC      kernel/trace/pid_list.o
  CC      net/sunrpc/xprt.o
  CC      fs/lockd/svcsubs.o
  CC      net/ipv4/inet_timewait_sock.o
  CC      net/ipv6/netfilter/nf_reject_ipv6.o
  CC      drivers/gpu/drm/virtio/virtgpu_display.o
  CC      net/sunrpc/auth_gss/trace.o
  CC      drivers/acpi/acpica/exresop.o
  CC      fs/lockd/mon.o
  CC      fs/9p/vfs_dentry.o
  CC      net/sunrpc/socklib.o
  CC      net/netfilter/nf_conntrack_expect.o
  CC      drivers/gpu/drm/virtio/virtgpu_vq.o
  CC      net/ipv6/ip6_fib.o
  CC      arch/x86/kernel/nmi.o
  CC      net/ipv6/netfilter/ip6t_ipv6header.o
  CC      net/netfilter/nf_conntrack_helper.o
  CC      drivers/tty/serial/8250/8250_pericom.o
  CC      net/core/sock_reuseport.o
  CC      fs/lockd/trace.o
  CC      lib/siphash.o
  CC      kernel/events/hw_breakpoint.o
  CC      mm/highmem.o
  CC      drivers/acpi/acpica/exserial.o
  CC      drivers/base/syscore.o
  CC      lib/string.o
  CC      net/netlabel/netlabel_user.o
  CC      drivers/scsi/scsi_lib.o
  CC      net/rfkill/core.o
  CC      drivers/scsi/constants.o
  CC      net/xfrm/xfrm_output.o
  CC      net/rfkill/input.o
  CC      fs/9p/v9fs.o
  CC      net/ethtool/eee.o
  CC      kernel/trace/trace_sched_switch.o
  CC      drivers/gpu/drm/i915/i915_pci.o
  CC      fs/lockd/xdr.o
  CC      kernel/user.o
  CC      net/ipv4/inet_connection_sock.o
  CC      kernel/signal.o
  CC      net/sunrpc/auth_gss/gss_krb5_mech.o
  CC      drivers/acpi/acpica/exstore.o
  CC      lib/timerqueue.o
  AR      drivers/tty/serial/8250/built-in.a
  CC      kernel/sys.o
  AR      drivers/tty/serial/built-in.a
  CC      net/ipv6/netfilter/ip6t_REJECT.o
  CC      kernel/umh.o
  CC      drivers/tty/tty_buffer.o
  CC      net/sunrpc/auth_gss/gss_krb5_seal.o
  CC      lib/vsprintf.o
  CC      net/mac80211/sta_info.o
  CC      net/ipv6/ipv6_sockglue.o
  CC      arch/x86/kernel/ldt.o
  CC      drivers/gpu/drm/virtio/virtgpu_fence.o
  CC      arch/x86/kernel/setup.o
  CC      kernel/events/uprobes.o
  CC      drivers/acpi/acpica/exstoren.o
  CC      net/core/fib_notifier.o
  CC      net/core/xdp.o
  CC      net/core/flow_offload.o
  CC      drivers/gpu/drm/i915/i915_scatterlist.o
  CC      arch/x86/kernel/x86_init.o
  CC      mm/memory.o
  CC      fs/9p/fid.o
  CC      net/netlabel/netlabel_kapi.o
  CC      fs/nfs/write.o
  AR      net/rfkill/built-in.a
  CC      net/9p/mod.o
  CC      net/wireless/reg.o
  CC      net/dns_resolver/dns_key.o
  CC      drivers/base/driver.o
  CC      net/dns_resolver/dns_query.o
  CC      net/9p/client.o
  CC      net/ethtool/tsinfo.o
  CC      net/ethtool/cabletest.o
  CC      net/netlabel/netlabel_domainhash.o
  CC      net/netfilter/nf_conntrack_proto.o
  CC      drivers/gpu/drm/i915/i915_suspend.o
  CC      drivers/acpi/acpica/exstorob.o
  CC      drivers/tty/tty_port.o
  CC      fs/lockd/clnt4xdr.o
  CC      fs/lockd/xdr4.o
  CC      fs/ext4/resize.o
  CC      drivers/acpi/utils.o
  CC      kernel/trace/trace_nop.o
  CC      kernel/workqueue.o
  CC      net/sunrpc/auth_gss/gss_krb5_unseal.o
  CC      drivers/base/class.o
  CC      fs/nfs/namespace.o
  CC      drivers/gpu/drm/i915/i915_switcheroo.o
  CC      arch/x86/kernel/i8259.o
  CC      net/xfrm/xfrm_sysctl.o
  CC      net/ipv6/ndisc.o
  CC      drivers/gpu/drm/virtio/virtgpu_object.o
  CC      net/mac80211/wep.o
  CC      drivers/acpi/acpica/exsystem.o
  CC      fs/ext4/super.o
  CC      drivers/gpu/drm/virtio/virtgpu_debugfs.o
  CC      fs/9p/xattr.o
  CC      fs/debugfs/inode.o
  CC      drivers/scsi/scsi_lib_dma.o
  CC      drivers/gpu/drm/virtio/virtgpu_plane.o
  CC      fs/debugfs/file.o
  AR      net/ipv6/netfilter/built-in.a
  CC      net/ipv6/udp.o
  CC      arch/x86/kernel/irqinit.o
  AR      net/dns_resolver/built-in.a
  CC      drivers/base/platform.o
  CC      net/core/gro.o
  CC      drivers/acpi/acpica/extrace.o
  CC      lib/win_minmax.o
  CC      net/xfrm/xfrm_replay.o
  CC      drivers/tty/tty_mutex.o
  CC      kernel/trace/blktrace.o
  CC      drivers/gpu/drm/i915/i915_sysfs.o
  CC      net/netlabel/netlabel_addrlist.o
  CC      net/ipv4/tcp.o
  CC      net/sunrpc/auth_gss/gss_krb5_wrap.o
  CC      drivers/gpu/drm/virtio/virtgpu_ioctl.o
  CC      net/core/netdev-genl.o
  CC      net/ethtool/tunnels.o
  AR      drivers/nvme/common/built-in.a
  CC      kernel/pid.o
  CC      drivers/scsi/scsi_scan.o
  AR      drivers/nvme/host/built-in.a
  AR      fs/9p/built-in.a
  AR      drivers/nvme/target/built-in.a
  CC      kernel/task_work.o
  AR      drivers/nvme/built-in.a
  CC      drivers/ata/libata-core.o
  CC      drivers/ata/libata-scsi.o
  CC      lib/xarray.o
  CC      net/xfrm/xfrm_device.o
  CC      drivers/acpi/acpica/exutils.o
  AR      kernel/events/built-in.a
  CC      drivers/gpu/drm/virtio/virtgpu_prime.o
  CC      drivers/acpi/reboot.o
  CC      net/xfrm/xfrm_nat_keepalive.o
  CC      fs/lockd/svc4proc.o
  CC      net/devres.o
  CC      net/handshake/alert.o
  CC      fs/lockd/procfs.o
  CC      net/sunrpc/auth_gss/gss_krb5_crypto.o
  CC      drivers/gpu/drm/virtio/virtgpu_trace_points.o
  CC      net/netfilter/nf_conntrack_proto_generic.o
  CC      net/handshake/genl.o
  CC      net/netlabel/netlabel_mgmt.o
  CC      fs/tracefs/inode.o
  CC [M]  fs/efivarfs/inode.o
  CC      drivers/tty/tty_ldsem.o
  CC      arch/x86/kernel/jump_label.o
  CC      net/netlabel/netlabel_unlabeled.o
  AR      fs/debugfs/built-in.a
  CC [M]  fs/efivarfs/file.o
  CC      net/9p/error.o
  CC      drivers/acpi/acpica/hwacpi.o
  CC      lib/lockref.o
  CC      kernel/extable.o
  CC      net/sunrpc/auth_gss/gss_krb5_keys.o
  CC      arch/x86/kernel/irq_work.o
  CC      drivers/base/cpu.o
  CC      drivers/tty/tty_baudrate.o
  CC      net/ipv6/udplite.o
  CC      drivers/gpu/drm/virtio/virtgpu_submit.o
  CC      arch/x86/kernel/probe_roms.o
  CC      drivers/gpu/drm/i915/i915_utils.o
  CC      net/9p/protocol.o
  CC      drivers/acpi/acpica/hwesleep.o
  CC      net/netlabel/netlabel_cipso_v4.o
  CC      drivers/base/firmware.o
  CC      net/ipv6/raw.o
  CC      fs/ext4/symlink.o
  CC      drivers/ata/libata-eh.o
  CC      drivers/acpi/nvs.o
  CC      drivers/acpi/wakeup.o
  CC      drivers/acpi/acpica/hwgpe.o
  CC      drivers/base/init.o
  CC [M]  fs/efivarfs/super.o
  CC      drivers/gpu/drm/i915/intel_clock_gating.o
  CC      net/mac80211/aead_api.o
  CC      fs/tracefs/event_inode.o
  GEN     drivers/scsi/scsi_devinfo_tbl.c
  CC      net/ethtool/fec.o
  CC      drivers/scsi/scsi_devinfo.o
  CC      kernel/params.o
  CC      net/netfilter/nf_conntrack_proto_tcp.o
  CC      fs/nfs/mount_clnt.o
  CC      lib/bcd.o
  CC      net/xfrm/xfrm_algo.o
  CC      drivers/scsi/scsi_sysctl.o
  CC      lib/sort.o
  CC      drivers/scsi/scsi_proc.o
  CC      net/ipv6/icmp.o
  CC      net/9p/trans_common.o
  CC      kernel/trace/trace_events.o
  AR      fs/lockd/built-in.a
  CC      net/socket.o
  CC      lib/parser.o
  CC      net/core/netdev-genl-gen.o
  CC      drivers/tty/tty_jobctrl.o
  CC      net/handshake/netlink.o
  CC      drivers/acpi/acpica/hwregs.o
  AR      net/sunrpc/auth_gss/built-in.a
  CC      drivers/acpi/sleep.o
  CC      fs/open.o
  CC      net/sunrpc/xprtsock.o
  CC      arch/x86/kernel/sys_ia32.o
  CC      arch/x86/kernel/ksysfs.o
  CC      net/xfrm/xfrm_user.o
  CC      mm/mincore.o
  CC      drivers/acpi/device_sysfs.o
  CC      drivers/base/map.o
  AR      drivers/gpu/drm/virtio/built-in.a
  CC      net/netlabel/netlabel_calipso.o
  CC      net/mac80211/wpa.o
  CC      net/sunrpc/sched.o
  AR      drivers/gpu/drm/bridge/analogix/built-in.a
  CC      net/mac80211/scan.o
  CC      net/mac80211/offchannel.o
  AR      drivers/gpu/drm/bridge/cadence/built-in.a
  CC      net/mac80211/ht.o
  AR      drivers/gpu/drm/bridge/imx/built-in.a
  AR      drivers/gpu/drm/bridge/synopsys/built-in.a
  CC      net/mac80211/agg-tx.o
  AR      drivers/gpu/drm/bridge/built-in.a
  CC      lib/debug_locks.o
  CC      net/mac80211/agg-rx.o
  CC      kernel/kthread.o
  CC [M]  fs/efivarfs/vars.o
  AR      drivers/net/phy/qcom/built-in.a
  CC      drivers/scsi/scsi_debugfs.o
  CC      drivers/net/phy/mdio-boardinfo.o
  CC      drivers/acpi/acpica/hwsleep.o
  CC      net/9p/trans_fd.o
  CC      lib/random32.o
  CC      drivers/net/phy/stubs.o
  CC      net/9p/trans_virtio.o
  CC      drivers/base/devres.o
  CC      net/core/gso.o
  CC      drivers/tty/n_null.o
  AR      fs/tracefs/built-in.a
  CC      net/ethtool/eeprom.o
  CC      net/mac80211/vht.o
  CC      kernel/sys_ni.o
  CC      drivers/acpi/acpica/hwvalid.o
  CC      kernel/nsproxy.o
  CC      net/sunrpc/auth.o
  CC      drivers/gpu/drm/i915/intel_device_info.o
  CC      mm/mlock.o
  CC      fs/nfs/nfstrace.o
  CC      arch/x86/kernel/bootflag.o
  AR      drivers/net/pse-pd/built-in.a
  CC      lib/bust_spinlocks.o
  CC      drivers/net/mdio/acpi_mdio.o
  CC      drivers/acpi/acpica/hwxface.o
  CC      drivers/net/mdio/fwnode_mdio.o
  CC      net/mac80211/he.o
  CC      kernel/notifier.o
  CC      drivers/acpi/device_pm.o
  CC      net/handshake/request.o
  CC      drivers/scsi/scsi_trace.o
  LD [M]  fs/efivarfs/efivarfs.o
  CC      drivers/acpi/proc.o
  CC      drivers/tty/pty.o
  CC      drivers/net/phy/mdio_devres.o
  CC      drivers/firewire/init_ohci1394_dma.o
  CC      drivers/acpi/acpica/hwxfsleep.o
  CC      drivers/tty/tty_audit.o
  CC      drivers/acpi/bus.o
  CC      net/wireless/scan.o
  AR      net/netlabel/built-in.a
  CC      drivers/base/attribute_container.o
  CC      drivers/tty/sysrq.o
  CC      net/netfilter/nf_conntrack_proto_udp.o
  CC      net/ipv6/mcast.o
  CC      drivers/gpu/drm/i915/intel_memory_region.o
  CC      lib/kasprintf.o
  CC      arch/x86/kernel/e820.o
  CC      net/mac80211/s1g.o
  CC      arch/x86/kernel/pci-dma.o
  CC      drivers/acpi/acpica/hwpci.o
  CC      net/mac80211/ibss.o
  CC      net/ethtool/stats.o
  CC      net/netfilter/nf_conntrack_proto_icmp.o
  CC      net/ipv4/tcp_input.o
  AR      net/9p/built-in.a
  CC      net/core/net-sysfs.o
  CC      drivers/acpi/acpica/nsaccess.o
  CC      drivers/acpi/acpica/nsalloc.o
  CC      kernel/trace/trace_export.o
  CC      lib/bitmap.o
  AR      drivers/net/mdio/built-in.a
  CC      drivers/base/transport_class.o
  AR      drivers/net/pcs/built-in.a
  AR      drivers/firewire/built-in.a
  CC      drivers/acpi/acpica/nsarguments.o
  CC      lib/scatterlist.o
  AR      drivers/net/ethernet/3com/built-in.a
  CC      drivers/cdrom/cdrom.o
  AR      drivers/auxdisplay/built-in.a
  CC      drivers/ata/libata-transport.o
  CC      drivers/ata/libata-trace.o
  CC      drivers/net/ethernet/8390/ne2k-pci.o
  AR      drivers/net/ethernet/adaptec/built-in.a
  CC      drivers/scsi/scsi_logging.o
  CC      drivers/scsi/scsi_pm.o
  CC      lib/list_sort.o
  CC      net/core/hotdata.o
  CC      mm/mmap.o
  CC      mm/mmu_gather.o
  CC      drivers/net/phy/phy.o
  CC      drivers/pcmcia/cs.o
  CC      net/handshake/tlshd.o
  CC      kernel/ksysfs.o
  CC      mm/mprotect.o
  CC      drivers/net/phy/phy-c45.o
  CC      drivers/acpi/glue.o
  CC      drivers/ata/libata-sata.o
  CC      drivers/acpi/acpica/nsconvert.o
  CC      drivers/base/topology.o
  CC      drivers/usb/common/common.o
  AR      drivers/tty/built-in.a
  CC      drivers/input/serio/serio.o
  CC      drivers/usb/core/usb.o
  CC      drivers/gpu/drm/i915/intel_pcode.o
  CC      drivers/usb/core/hub.o
  AR      net/xfrm/built-in.a
  CC      drivers/usb/core/hcd.o
  CC      arch/x86/kernel/quirks.o
  CC      kernel/trace/trace_event_perf.o
  CC      net/sysctl_net.o
  CC      net/core/net-procfs.o
  CC      drivers/acpi/acpica/nsdump.o
  CC      mm/mremap.o
  CC      drivers/scsi/scsi_bsg.o
  CC      drivers/gpu/drm/i915/intel_region_ttm.o
  CC      drivers/usb/core/urb.o
  CC      net/netfilter/nf_conntrack_extend.o
  CC      net/ethtool/phc_vclocks.o
  AR      drivers/net/ethernet/agere/built-in.a
  AR      drivers/net/wireless/admtek/built-in.a
  CC      net/mac80211/iface.o
  CC      kernel/cred.o
  AR      drivers/net/wireless/ath/built-in.a
  AR      drivers/net/wireless/atmel/built-in.a
  AR      drivers/net/wireless/broadcom/built-in.a
  CC      lib/uuid.o
  CC      drivers/pcmcia/socket_sysfs.o
  AR      drivers/net/wireless/intel/built-in.a
  CC      drivers/base/container.o
  AR      drivers/net/wireless/intersil/built-in.a
  AR      drivers/net/usb/built-in.a
  CC      drivers/base/property.o
  AR      drivers/net/wireless/marvell/built-in.a
  CC      drivers/net/ethernet/8390/8390.o
  CC      net/netfilter/nf_conntrack_acct.o
  AR      drivers/net/wireless/mediatek/built-in.a
  CC      drivers/usb/common/debug.o
  AR      drivers/net/wireless/microchip/built-in.a
  CC      net/core/netpoll.o
  CC      drivers/input/serio/i8042.o
  CC      drivers/acpi/acpica/nseval.o
  CC      drivers/pcmcia/cardbus.o
  AR      drivers/net/wireless/purelifi/built-in.a
  CC      lib/iov_iter.o
  CC      net/ethtool/mm.o
  AR      drivers/net/wireless/quantenna/built-in.a
  AR      drivers/net/wireless/ralink/built-in.a
  AR      drivers/net/wireless/realtek/built-in.a
  CC      drivers/input/serio/serport.o
  AR      drivers/net/wireless/rsi/built-in.a
  AR      drivers/net/wireless/silabs/built-in.a
  CC      net/handshake/trace.o
  AR      drivers/usb/common/built-in.a
  AR      drivers/net/wireless/st/built-in.a
  CC      net/ipv4/tcp_output.o
  AR      drivers/net/wireless/ti/built-in.a
  AR      drivers/net/wireless/zydas/built-in.a
  CC      mm/msync.o
  AR      drivers/net/wireless/virtual/built-in.a
  AR      drivers/net/wireless/built-in.a
  CC      net/sunrpc/auth_null.o
  CC      drivers/net/mii.o
  CC      arch/x86/kernel/kdebugfs.o
  CC      drivers/ata/libata-sff.o
  CC      drivers/ata/libata-pmp.o
  CC      drivers/usb/core/message.o
  CC      arch/x86/kernel/alternative.o
  AR      drivers/usb/phy/built-in.a
  CC      arch/x86/kernel/i8253.o
  CC      drivers/net/phy/phy-core.o
  CC      drivers/pcmcia/ds.o
  CC      drivers/acpi/acpica/nsinit.o
  CC      drivers/scsi/scsi_common.o
  CC      drivers/acpi/acpica/nsload.o
  CC      kernel/trace/trace_events_filter.o
  AR      drivers/cdrom/built-in.a
  CC      fs/read_write.o
  CC      drivers/pcmcia/pcmcia_resource.o
  CC      drivers/net/phy/phy_device.o
  CC      mm/page_vma_mapped.o
  CC      drivers/pcmcia/cistpl.o
  CC      net/sunrpc/auth_tls.o
  CC      drivers/gpu/drm/i915/intel_runtime_pm.o
  CC      drivers/input/serio/libps2.o
  CC      mm/pagewalk.o
  CC      drivers/usb/core/driver.o
  CC      net/core/fib_rules.o
  CC      drivers/acpi/acpica/nsnames.o
  CC      drivers/usb/mon/mon_main.o
  CC      drivers/acpi/acpica/nsobject.o
  CC      drivers/usb/host/pci-quirks.o
  CC      drivers/scsi/scsi_transport_spi.o
  CC      drivers/usb/host/ehci-hcd.o
  CC      drivers/acpi/acpica/nsparse.o
  CC      net/ipv6/reassembly.o
  CC      drivers/usb/class/usblp.o
  CC      net/ethtool/module.o
  AR      drivers/net/ethernet/8390/built-in.a
  AR      drivers/net/ethernet/alacritech/built-in.a
  CC      net/netfilter/nf_conntrack_seqadj.o
  AR      drivers/net/ethernet/alteon/built-in.a
  AR      drivers/net/ethernet/amazon/built-in.a
  CC      net/netfilter/nf_conntrack_proto_icmpv6.o
  AR      drivers/net/ethernet/amd/built-in.a
  AR      drivers/net/ethernet/aquantia/built-in.a
  CC      drivers/base/cacheinfo.o
  CC      net/ethtool/cmis_fw_update.o
  AR      drivers/net/ethernet/arc/built-in.a
  AR      drivers/net/ethernet/asix/built-in.a
  CC      drivers/acpi/acpica/nspredef.o
  AR      drivers/net/ethernet/atheros/built-in.a
  AR      drivers/net/ethernet/cadence/built-in.a
  CC      drivers/net/ethernet/broadcom/bnx2.o
  CC      drivers/acpi/acpica/nsprepkg.o
  CC      net/ethtool/cmis_cdb.o
  CC      drivers/net/loopback.o
  AR      drivers/net/ethernet/brocade/built-in.a
  CC      drivers/acpi/acpica/nsrepair.o
  CC      drivers/usb/mon/mon_stat.o
  CC      fs/file_table.o
  CC      drivers/net/netconsole.o
  AR      drivers/input/serio/built-in.a
  CC      mm/pgtable-generic.o
  CC      drivers/usb/storage/scsiglue.o
  CC      mm/rmap.o
  CC      drivers/usb/storage/protocol.o
  CC      net/ethtool/pse-pd.o
  CC      arch/x86/kernel/hw_breakpoint.o
  CC      drivers/usb/storage/transport.o
  AR      net/handshake/built-in.a
  CC      drivers/usb/host/ehci-pci.o
  CC      drivers/input/keyboard/atkbd.o
  CC      drivers/pcmcia/pcmcia_cis.o
  CC      drivers/input/mouse/psmouse-base.o
  CC      fs/nfs/export.o
  CC      drivers/rtc/lib.o
  AR      drivers/input/joystick/built-in.a
  CC      drivers/usb/core/config.o
  AR      drivers/net/ethernet/cavium/common/built-in.a
  CC      drivers/acpi/acpica/nsrepair2.o
  AR      drivers/net/ethernet/cavium/thunder/built-in.a
  AR      drivers/net/ethernet/cavium/liquidio/built-in.a
  AR      drivers/net/ethernet/chelsio/built-in.a
  AR      drivers/net/ethernet/cavium/octeon/built-in.a
  CC      drivers/base/swnode.o
  CC      drivers/rtc/class.o
  AR      drivers/net/ethernet/cavium/built-in.a
  CC      drivers/usb/core/file.o
  AR      drivers/usb/class/built-in.a
  CC      lib/clz_ctz.o
  CC      drivers/usb/core/buffer.o
  CC      drivers/ata/libata-acpi.o
  AR      drivers/usb/misc/built-in.a
  CC      kernel/trace/trace_events_trigger.o
  CC      drivers/ata/libata-pata-timings.o
  CC      drivers/gpu/drm/i915/intel_sbi.o
  CC      drivers/usb/mon/mon_text.o
  CC      lib/bsearch.o
  CC      fs/super.o
  CC      net/ethtool/plca.o
  CC      net/ipv4/tcp_timer.o
  CC      drivers/usb/mon/mon_bin.o
  CC      net/ipv4/tcp_ipv4.o
  CC      drivers/usb/core/sysfs.o
  CC      fs/char_dev.o
  CC      drivers/rtc/interface.o
  CC      net/ipv6/tcp_ipv6.o
  AR      drivers/input/tablet/built-in.a
  CC      net/ipv6/ping.o
  CC      drivers/acpi/acpica/nssearch.o
  CC      net/core/net-traces.o
  CC      net/ipv4/tcp_minisocks.o
  CC      drivers/scsi/virtio_scsi.o
  CC      drivers/usb/storage/usb.o
  CC      drivers/net/phy/linkmode.o
  CC      drivers/net/virtio_net.o
  CC      net/netfilter/nf_conntrack_netlink.o
  CC      drivers/net/net_failover.o
  CC      mm/vmalloc.o
  CC      arch/x86/kernel/tsc.o
  CC      drivers/net/phy/mdio_bus.o
  CC      drivers/usb/storage/initializers.o
  AR      drivers/input/touchscreen/built-in.a
  CC      drivers/ata/ahci.o
  AR      drivers/input/keyboard/built-in.a
  CC      drivers/usb/core/endpoint.o
  CC      drivers/pcmcia/rsrc_mgr.o
  AR      drivers/input/misc/built-in.a
  CC      drivers/input/input.o
  CC      drivers/usb/core/devio.o
  CC      drivers/input/mouse/synaptics.o
  CC      lib/find_bit.o
  CC      drivers/base/auxiliary.o
  CC      net/wireless/nl80211.o
  CC      arch/x86/kernel/tsc_msr.o
  CC      drivers/acpi/acpica/nsutils.o
  CC      drivers/ata/libahci.o
  CC      fs/ext4/sysfs.o
  CC      drivers/ata/ata_piix.o
  CC      drivers/gpu/drm/i915/intel_step.o
  CC      net/netfilter/nf_conntrack_ftp.o
  CC      lib/llist.o
  CC      net/ipv4/tcp_cong.o
  CC      fs/stat.o
  CC      drivers/ata/pata_amd.o
  CC      lib/lwq.o
  CC      kernel/trace/trace_eprobe.o
  CC      drivers/base/devtmpfs.o
  AR      net/ethtool/built-in.a
  CC      drivers/ata/pata_oldpiix.o
  CC      drivers/usb/host/ohci-hcd.o
  CC      drivers/pcmcia/rsrc_nonstatic.o
  CC      fs/exec.o
  CC      drivers/usb/core/notify.o
  CC      net/mac80211/link.o
  CC      drivers/acpi/acpica/nswalk.o
  CC      drivers/ata/pata_sch.o
  AR      drivers/usb/mon/built-in.a
  CC      drivers/usb/storage/sierra_ms.o
  CC      drivers/usb/core/generic.o
  CC      mm/process_vm_access.o
  CC      lib/memweight.o
  CC      drivers/i2c/algos/i2c-algo-bit.o
  CC      lib/kfifo.o
  CC      drivers/i2c/busses/i2c-i801.o
  CC      drivers/rtc/nvmem.o
  CC      drivers/scsi/sd.o
  CC      net/sunrpc/auth_unix.o
  CC      arch/x86/kernel/io_delay.o
  CC      drivers/rtc/dev.o
  CC      fs/nfs/sysfs.o
  CC      fs/pipe.o
  CC      net/core/selftests.o
  CC      drivers/acpi/acpica/nsxfeval.o
  CC      fs/ext4/xattr.o
  CC      drivers/usb/host/ohci-pci.o
  CC      drivers/rtc/proc.o
  CC      fs/ext4/xattr_hurd.o
  CC      drivers/net/phy/mdio_device.o
  CC      drivers/input/mouse/focaltech.o
  CC      net/wireless/mlme.o
  CC      drivers/gpu/drm/i915/intel_uncore.o
  CC      arch/x86/kernel/rtc.o
  CC      net/core/ptp_classifier.o
  CC      fs/namei.o
  CC      drivers/usb/core/quirks.o
  CC      drivers/input/mouse/alps.o
  CC      drivers/base/module.o
  CC      drivers/ata/pata_mpiix.o
  CC      net/ipv6/exthdrs.o
  CC      drivers/usb/host/uhci-hcd.o
  CC      drivers/usb/storage/option_ms.o
  CC      drivers/acpi/acpica/nsxfname.o
  CC      drivers/usb/host/xhci.o
  CC      net/ipv6/datagram.o
  CC      drivers/input/mouse/byd.o
  CC      drivers/pcmcia/yenta_socket.o
  CC      net/netfilter/nf_conntrack_irc.o
  CC      drivers/rtc/sysfs.o
  CC      lib/percpu-refcount.o
  AR      drivers/i2c/algos/built-in.a
  CC      net/ipv6/ip6_flowlabel.o
  CC      drivers/base/auxiliary_sysfs.o
  CC      kernel/trace/trace_kprobe.o
  CC      fs/ext4/xattr_trusted.o
  CC      net/ipv4/tcp_metrics.o
  CC      fs/fcntl.o
  CC      mm/page_alloc.o
  CC      net/core/netprio_cgroup.o
  CC      lib/rhashtable.o
  CC      arch/x86/kernel/resource.o
  CC      net/sunrpc/svc.o
  CC      drivers/acpi/acpica/nsxfobj.o
  AR      drivers/i2c/busses/built-in.a
  CC      net/netfilter/nf_conntrack_sip.o
  CC      drivers/usb/core/devices.o
  CC      fs/nfs/fs_context.o
  AR      drivers/i2c/muxes/built-in.a
  CC      drivers/i2c/i2c-boardinfo.o
  CC      fs/nfs/nfsroot.o
  CC      drivers/net/phy/swphy.o
  AS      arch/x86/kernel/irqflags.o
  CC      fs/ioctl.o
  CC      arch/x86/kernel/static_call.o
  CC      net/sunrpc/svcsock.o
  CC      drivers/ata/ata_generic.o
  CC      drivers/base/devcoredump.o
  CC      drivers/usb/core/phy.o
  CC      drivers/input/input-compat.o
  CC      drivers/usb/storage/usual-tables.o
  CC      drivers/rtc/rtc-mc146818-lib.o
  CC      drivers/input/input-mt.o
  CC      drivers/acpi/acpica/psargs.o
  CC      drivers/gpu/drm/i915/intel_wakeref.o
  CC      net/ipv4/tcp_fastopen.o
  CC      lib/base64.o
  CC      net/ipv4/tcp_rate.o
  CC      fs/readdir.o
  CC      lib/once.o
  CC      arch/x86/kernel/process.o
  CC      mm/init-mm.o
  CC      fs/select.o
  CC      drivers/i2c/i2c-core-base.o
  CC      drivers/input/input-poller.o
  AR      drivers/usb/storage/built-in.a
  CC      net/core/netclassid_cgroup.o
  CC      drivers/usb/early/ehci-dbgp.o
  AR      drivers/pcmcia/built-in.a
  CC      drivers/acpi/acpica/psloop.o
  CC      net/netfilter/nf_nat_core.o
  CC      lib/refcount.o
  CC      drivers/base/platform-msi.o
  CC      drivers/input/mouse/logips2pp.o
  CC      lib/rcuref.o
  CC      drivers/net/phy/fixed_phy.o
  AR      drivers/ata/built-in.a
  CC      drivers/usb/core/port.o
  CC      drivers/i2c/i2c-core-smbus.o
  CC      drivers/net/ethernet/broadcom/tg3.o
  CC      drivers/acpi/scan.o
  CC      drivers/usb/core/hcd-pci.o
  CC      drivers/net/phy/realtek.o
  CC      drivers/scsi/sr.o
  CC      drivers/rtc/rtc-cmos.o
  CC      net/ipv6/inet6_connection_sock.o
  CC      lib/usercopy.o
  CC      net/ipv6/udp_offload.o
  AR      drivers/i3c/built-in.a
  CC      mm/memblock.o
  CC      fs/nfs/sysctl.o
  CC      drivers/i2c/i2c-core-acpi.o
  CC      net/mac80211/rate.o
  AR      drivers/media/i2c/built-in.a
  CC      fs/ext4/xattr_user.o
  CC      net/ipv6/seg6.o
  AR      drivers/media/tuners/built-in.a
  AR      drivers/media/rc/keymaps/built-in.a
  AR      drivers/media/rc/built-in.a
  AR      drivers/media/common/b2c2/built-in.a
  AR      drivers/media/common/saa7146/built-in.a
  AR      drivers/media/common/siano/built-in.a
  CC      fs/ext4/fast_commit.o
  AR      drivers/media/common/v4l2-tpg/built-in.a
  AR      drivers/media/common/videobuf2/built-in.a
  AR      drivers/media/common/built-in.a
  CC      drivers/acpi/acpica/psobject.o
  AR      drivers/media/platform/allegro-dvt/built-in.a
  CC      fs/nfs/nfs3super.o
  AR      drivers/media/pci/ttpci/built-in.a
  CC      lib/errseq.o
  AR      drivers/media/platform/amlogic/meson-ge2d/built-in.a
  AR      drivers/media/platform/amlogic/built-in.a
  AR      drivers/media/pci/b2c2/built-in.a
  CC      drivers/base/physical_location.o
  CC      drivers/base/trace.o
  AR      drivers/media/pci/pluto2/built-in.a
  AR      drivers/media/platform/aspeed/built-in.a
  AR      drivers/media/platform/amphion/built-in.a
  AR      drivers/media/pci/dm1105/built-in.a
  CC      drivers/acpi/acpica/psopcode.o
  AR      drivers/media/platform/atmel/built-in.a
  AR      drivers/media/pci/pt1/built-in.a
  AR      drivers/media/pci/pt3/built-in.a
  AR      drivers/media/platform/broadcom/built-in.a
  AR      drivers/media/platform/cadence/built-in.a
  AR      drivers/media/pci/mantis/built-in.a
  CC      lib/bucket_locks.o
  AR      drivers/media/platform/chips-media/coda/built-in.a
  AR      drivers/media/pci/ngene/built-in.a
  AR      drivers/media/platform/chips-media/wave5/built-in.a
  AR      drivers/media/pci/ddbridge/built-in.a
  AR      drivers/media/platform/chips-media/built-in.a
  CC      drivers/input/mouse/lifebook.o
  CC      drivers/input/mouse/trackpoint.o
  AR      drivers/media/pci/saa7146/built-in.a
  AR      drivers/media/platform/imagination/built-in.a
  AR      drivers/media/pci/smipcie/built-in.a
  AR      drivers/media/platform/intel/built-in.a
  AR      drivers/media/pci/netup_unidvb/built-in.a
  CC      drivers/gpu/drm/i915/vlv_sideband.o
  AR      drivers/media/platform/marvell/built-in.a
  CC      kernel/trace/error_report-traces.o
  AR      drivers/media/pci/intel/ipu3/built-in.a
  AR      drivers/media/pci/intel/ivsc/built-in.a
  AR      drivers/media/pci/intel/built-in.a
  AR      drivers/media/platform/mediatek/jpeg/built-in.a
  CC      drivers/input/mouse/cypress_ps2.o
  AR      drivers/media/pci/built-in.a
  AR      drivers/media/platform/mediatek/mdp/built-in.a
  AR      drivers/usb/early/built-in.a
  CC      drivers/gpu/drm/i915/vlv_suspend.o
  AR      drivers/media/platform/mediatek/vcodec/common/built-in.a
  AR      drivers/media/platform/mediatek/vcodec/encoder/built-in.a
  AR      drivers/media/platform/mediatek/vcodec/decoder/built-in.a
  AR      drivers/media/usb/b2c2/built-in.a
  AR      drivers/media/platform/mediatek/vcodec/built-in.a
  AR      drivers/media/usb/dvb-usb/built-in.a
  AR      drivers/media/platform/mediatek/vpu/built-in.a
  AR      drivers/media/usb/dvb-usb-v2/built-in.a
  CC      drivers/acpi/acpica/psopinfo.o
  AR      drivers/media/usb/s2255/built-in.a
  AR      drivers/media/platform/mediatek/mdp3/built-in.a
  CC      net/ipv4/tcp_recovery.o
  AR      drivers/media/platform/mediatek/built-in.a
  AR      drivers/media/usb/siano/built-in.a
  AR      drivers/media/usb/ttusb-budget/built-in.a
  AR      drivers/media/platform/microchip/built-in.a
  AR      drivers/media/usb/ttusb-dec/built-in.a
  AR      drivers/media/platform/nuvoton/built-in.a
  AR      drivers/media/usb/built-in.a
  CC      fs/nfs/nfs3client.o
  CC      net/mac80211/michael.o
  AR      drivers/media/platform/nvidia/tegra-vde/built-in.a
  CC      drivers/acpi/acpica/psparse.o
  AR      drivers/media/platform/nvidia/built-in.a
  AR      drivers/gpu/drm/hisilicon/built-in.a
  AR      drivers/gpu/drm/mxsfb/built-in.a
  CC      drivers/ptp/ptp_clock.o
  AR      drivers/media/platform/nxp/dw100/built-in.a
  AR      drivers/pps/clients/built-in.a
  AR      drivers/media/platform/nxp/imx-jpeg/built-in.a
  AR      drivers/pps/generators/built-in.a
  CC      drivers/pps/pps.o
  AR      drivers/media/platform/qcom/camss/built-in.a
  CC      fs/ext4/orphan.o
  CC      drivers/usb/core/usb-acpi.o
  AR      drivers/media/platform/nxp/imx8-isi/built-in.a
  CC      drivers/scsi/sr_ioctl.o
  AR      drivers/media/platform/qcom/venus/built-in.a
  AR      drivers/media/platform/nxp/built-in.a
  AR      drivers/media/platform/qcom/built-in.a
  AR      drivers/media/mmc/siano/built-in.a
  CC      drivers/power/supply/power_supply_core.o
  AR      drivers/media/platform/raspberrypi/pisp_be/built-in.a
  AR      drivers/media/mmc/built-in.a
  AR      drivers/media/platform/raspberrypi/built-in.a
  AR      drivers/media/firewire/built-in.a
  CC      lib/generic-radix-tree.o
  AR      drivers/media/platform/renesas/rcar-vin/built-in.a
  AR      drivers/rtc/built-in.a
  CC      drivers/hwmon/hwmon.o
  AR      drivers/media/platform/renesas/rzg2l-cru/built-in.a
  CC      net/core/dst_cache.o
  AR      drivers/media/platform/renesas/vsp1/built-in.a
  AR      drivers/media/spi/built-in.a
  AR      drivers/media/platform/renesas/built-in.a
  CC      net/core/gro_cells.o
  CC      lib/bitmap-str.o
  CC      arch/x86/kernel/ptrace.o
  CC      lib/string_helpers.o
  AR      drivers/media/platform/rockchip/rkisp1/built-in.a
  AR      drivers/media/platform/rockchip/rga/built-in.a
  CC      lib/hexdump.o
  AR      drivers/media/platform/rockchip/built-in.a
  AR      drivers/base/built-in.a
  CC      drivers/pps/kapi.o
  CC      fs/dcache.o
  CC      fs/inode.o
  CC      drivers/pps/sysfs.o
  AR      drivers/media/platform/samsung/exynos-gsc/built-in.a
  AR      drivers/net/phy/built-in.a
  AR      drivers/media/platform/samsung/exynos4-is/built-in.a
  CC      fs/nfs/nfs3proc.o
  AR      drivers/media/platform/samsung/s3c-camif/built-in.a
  AR      drivers/media/platform/samsung/s5p-g2d/built-in.a
  CC      net/core/failover.o
  AR      drivers/media/platform/samsung/s5p-jpeg/built-in.a
  CC      fs/nfs/nfs3xdr.o
  AR      drivers/media/platform/samsung/s5p-mfc/built-in.a
  AR      drivers/media/test-drivers/built-in.a
  AR      drivers/media/platform/samsung/built-in.a
  CC      fs/nfs/nfs3acl.o
  CC      net/ipv6/fib6_notifier.o
  CC      drivers/acpi/acpica/psscope.o
  CC      drivers/input/mouse/psmouse-smbus.o
  AR      drivers/media/platform/st/sti/bdisp/built-in.a
  CC      arch/x86/kernel/tls.o
  AR      drivers/media/platform/st/sti/c8sectpfe/built-in.a
  AR      drivers/media/platform/st/sti/delta/built-in.a
  CC      drivers/usb/host/xhci-mem.o
  AR      drivers/media/platform/st/sti/hva/built-in.a
  CC      fs/attr.o
  CC      drivers/acpi/mipi-disco-img.o
  AR      drivers/media/platform/st/stm32/built-in.a
  AR      drivers/media/platform/st/built-in.a
  CC      kernel/trace/power-traces.o
  CC      fs/nfs/nfs4proc.o
  CC      net/netfilter/nf_nat_proto.o
  CC      arch/x86/kernel/step.o
  CC      arch/x86/kernel/i8237.o
  AR      drivers/media/platform/sunxi/sun4i-csi/built-in.a
  AR      drivers/media/platform/sunxi/sun6i-csi/built-in.a
  AR      drivers/media/platform/sunxi/sun6i-mipi-csi2/built-in.a
  AR      drivers/media/platform/ti/am437x/built-in.a
  AR      drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/built-in.a
  AR      drivers/media/platform/ti/cal/built-in.a
  AR      drivers/media/platform/sunxi/sun8i-di/built-in.a
  AR      drivers/media/platform/ti/vpe/built-in.a
  AR      drivers/media/platform/sunxi/sun8i-rotate/built-in.a
  AR      drivers/media/platform/ti/davinci/built-in.a
  AR      drivers/media/platform/verisilicon/built-in.a
  AR      drivers/media/platform/sunxi/built-in.a
  AR      drivers/media/platform/ti/j721e-csi2rx/built-in.a
  CC      drivers/acpi/acpica/pstree.o
  AR      drivers/media/platform/ti/omap/built-in.a
  CC      drivers/acpi/acpica/psutils.o
  CC      net/mac80211/tkip.o
  AR      drivers/media/platform/ti/omap3isp/built-in.a
  CC      drivers/acpi/acpica/pswalk.o
  CC      fs/nfs/nfs4xdr.o
  AR      drivers/media/platform/ti/built-in.a
  CC      drivers/i2c/i2c-smbus.o
  AR      drivers/media/platform/via/built-in.a
  AR      drivers/media/platform/xilinx/built-in.a
  AR      drivers/media/platform/built-in.a
  AR      drivers/media/built-in.a
  CC      net/wireless/ibss.o
  AR      drivers/thermal/broadcom/built-in.a
  CC      drivers/acpi/acpica/psxface.o
  AR      drivers/thermal/renesas/built-in.a
  CC      net/wireless/sme.o
  AR      drivers/thermal/samsung/built-in.a
  AR      drivers/usb/core/built-in.a
  AR      drivers/pps/built-in.a
  CC      drivers/acpi/acpica/rsaddr.o
  CC      mm/slub.o
  CC      drivers/thermal/intel/intel_tcc.o
  CC      mm/madvise.o
  CC      drivers/power/supply/power_supply_sysfs.o
  CC      drivers/thermal/intel/therm_throt.o
  CC      drivers/gpu/drm/i915/soc/intel_dram.o
  CC      drivers/scsi/sr_vendor.o
  CC [M]  drivers/thermal/intel/x86_pkg_temp_thermal.o
  CC      net/wireless/chan.o
  CC      drivers/ptp/ptp_chardev.o
  CC      drivers/power/supply/power_supply_leds.o
  CC      net/mac80211/aes_cmac.o
  CC      net/ipv4/tcp_ulp.o
  CC      net/ipv4/tcp_offload.o
  CC      lib/kstrtox.o
  CC      drivers/acpi/acpica/rscalc.o
  CC      net/sunrpc/svcauth.o
  CC      net/sunrpc/svcauth_unix.o
  AR      drivers/input/mouse/built-in.a
  CC      net/sunrpc/addr.o
  CC      drivers/input/ff-core.o
  CC      drivers/ptp/ptp_sysfs.o
  CC      net/wireless/ethtool.o
  CC      net/wireless/mesh.o
  CC      arch/x86/kernel/stacktrace.o
  CC      drivers/scsi/sg.o
  AR      net/core/built-in.a
  CC      drivers/gpu/drm/i915/soc/intel_gmch.o
  CC      drivers/acpi/resource.o
  CC      fs/ext4/acl.o
  CC      fs/bad_inode.o
  CC      drivers/power/supply/power_supply_hwmon.o
  AR      drivers/i2c/built-in.a
  CC      net/ipv6/rpl.o
  AR      drivers/hwmon/built-in.a
  CC      drivers/input/touchscreen.o
  AR      drivers/watchdog/built-in.a
  CC      net/ipv6/ioam6.o
  CC      fs/nfs/nfs4state.o
  CC      lib/iomap.o
  CC      drivers/acpi/acpi_processor.o
  CC      drivers/acpi/acpica/rscreate.o
  CC      drivers/ptp/ptp_vclock.o
  CC      drivers/scsi/scsi_sysfs.o
  CC      drivers/md/md.o
  CC      net/netfilter/nf_nat_helper.o
  CC      fs/file.o
  AR      drivers/power/supply/built-in.a
  CC      arch/x86/kernel/reboot.o
  AR      drivers/power/built-in.a
  CC      drivers/ptp/ptp_kvm_x86.o
  CC      drivers/cpufreq/cpufreq.o
  AR      drivers/thermal/intel/built-in.a
  AR      drivers/thermal/st/built-in.a
  CC      fs/nfs/nfs4renewd.o
  CC      drivers/acpi/acpica/rsdumpinfo.o
  AR      drivers/thermal/qcom/built-in.a
  AR      drivers/thermal/tegra/built-in.a
  CC      fs/nfs/nfs4super.o
  CC      fs/nfs/nfs4file.o
  CC      fs/filesystems.o
  AR      drivers/thermal/mediatek/built-in.a
  CC      fs/nfs/delegation.o
  CC      drivers/thermal/thermal_core.o
  CC      arch/x86/kernel/msr.o
  CC      fs/ext4/xattr_security.o
  CC      net/wireless/ap.o
  CC      drivers/input/ff-memless.o
  CC      net/mac80211/aes_gmac.o
  CC      net/mac80211/fils_aead.o
  CC      net/mac80211/cfg.o
  AR      drivers/gpu/drm/tiny/built-in.a
  AR      drivers/gpu/drm/xlnx/built-in.a
  CC      drivers/md/md-bitmap.o
  CC      kernel/trace/rpm-traces.o
  CC      lib/iomap_copy.o
  CC      drivers/gpu/drm/i915/soc/intel_pch.o
  CC      mm/page_io.o
  CC      drivers/acpi/acpica/rsinfo.o
  CC      drivers/ptp/ptp_kvm_common.o
  CC      net/sunrpc/rpcb_clnt.o
  CC      lib/devres.o
  CC      drivers/usb/host/xhci-ext-caps.o
  CC      net/ipv4/tcp_plb.o
  CC      net/ipv6/sysctl_net_ipv6.o
  CC      drivers/acpi/acpica/rsio.o
  CC      fs/nfs/nfs4idmap.o
  CC      drivers/acpi/acpica/rsirq.o
  CC      kernel/trace/trace_dynevent.o
  CC      fs/namespace.o
  AR      drivers/scsi/built-in.a
  CC      drivers/cpuidle/cpuidle.o
  CC      drivers/cpuidle/governors/menu.o
  CC      arch/x86/kernel/cpuid.o
  CC      drivers/input/sparse-keymap.o
  CC      lib/check_signature.o
  CC      drivers/cpuidle/driver.o
  AR      fs/ext4/built-in.a
  CC      drivers/cpuidle/governors/haltpoll.o
  CC      net/netfilter/nf_nat_masquerade.o
  AR      drivers/ptp/built-in.a
  CC      net/ipv6/xfrm6_policy.o
  CC      lib/interval_tree.o
  AR      drivers/mmc/built-in.a
  AR      drivers/ufs/built-in.a
  CC      drivers/acpi/acpica/rslist.o
  CC      drivers/input/vivaldi-fmap.o
  CC      arch/x86/kernel/early-quirks.o
  AR      drivers/leds/trigger/built-in.a
  CC      arch/x86/kernel/smp.o
  CC      drivers/usb/host/xhci-ring.o
  AR      drivers/leds/blink/built-in.a
  CC      drivers/usb/host/xhci-hub.o
  AR      drivers/leds/simple/built-in.a
  CC      drivers/leds/led-core.o
  CC      drivers/md/md-autodetect.o
  CC      net/ipv4/datagram.o
  CC      net/sunrpc/timer.o
  CC      fs/seq_file.o
  CC      lib/assoc_array.o
  CC      net/mac80211/ethtool.o
  CC      drivers/gpu/drm/i915/i915_memcpy.o
  CC      fs/xattr.o
  AR      drivers/gpu/drm/gud/built-in.a
  CC      fs/libfs.o
  CC      drivers/acpi/acpica/rsmemory.o
  CC      net/ipv6/xfrm6_state.o
  CC      drivers/usb/host/xhci-dbg.o
  CC      drivers/input/input-leds.o
  CC      drivers/gpu/drm/i915/i915_mm.o
  CC      drivers/thermal/thermal_sysfs.o
  CC      net/mac80211/rx.o
  CC      drivers/input/evdev.o
  CC      kernel/trace/trace_probe.o
  CC      net/wireless/trace.o
  CC      drivers/cpufreq/freq_table.o
  CC      drivers/leds/led-class.o
  CC      fs/nfs/callback.o
  CC      fs/nfs/callback_xdr.o
  CC      net/netfilter/nf_nat_ftp.o
  CC      mm/swap_state.o
  CC      drivers/acpi/acpica/rsmisc.o
  CC      drivers/acpi/processor_core.o
  AR      drivers/cpuidle/governors/built-in.a
  CC      drivers/cpuidle/governor.o
  CC      drivers/cpuidle/sysfs.o
  CC      lib/bitrev.o
  CC      drivers/md/dm.o
  CC      drivers/usb/host/xhci-trace.o
  CC      drivers/usb/host/xhci-debugfs.o
  CC      drivers/acpi/acpica/rsserial.o
  CC      drivers/cpuidle/poll_state.o
  AR      drivers/firmware/arm_ffa/built-in.a
  AR      drivers/firmware/arm_scmi/built-in.a
  AR      drivers/firmware/broadcom/built-in.a
  CC      drivers/thermal/thermal_trip.o
  CC      kernel/trace/trace_uprobe.o
  CC      drivers/cpufreq/cpufreq_performance.o
  AR      drivers/firmware/cirrus/built-in.a
  CC      net/sunrpc/xdr.o
  AR      drivers/firmware/meson/built-in.a
  CC      drivers/md/dm-table.o
  AR      drivers/firmware/microchip/built-in.a
  CC      drivers/acpi/acpica/rsutils.o
  CC      drivers/acpi/acpica/rsxface.o
  CC      drivers/leds/led-triggers.o
  CC      drivers/firmware/efi/efi-bgrt.o
  CC      arch/x86/kernel/smpboot.o
  CC      drivers/cpufreq/cpufreq_userspace.o
  CC      lib/crc-ccitt.o
  CC      net/netfilter/nf_nat_irc.o
  CC      fs/nfs/callback_proc.o
  CC      drivers/firmware/efi/libstub/efi-stub-helper.o
  CC      net/ipv4/raw.o
  CC      arch/x86/kernel/tsc_sync.o
  CC      net/netfilter/nf_nat_sip.o
  CC      net/sunrpc/sunrpc_syms.o
  CC      net/wireless/ocb.o
  CC      drivers/md/dm-target.o
  CC      drivers/gpu/drm/i915/i915_sw_fence.o
  CC      net/ipv6/xfrm6_input.o
  CC      drivers/cpuidle/cpuidle-haltpoll.o
  AR      drivers/input/built-in.a
  CC      drivers/gpu/drm/i915/i915_sw_fence_work.o
  CC      drivers/md/dm-linear.o
  CC      drivers/thermal/thermal_helpers.o
  CC      net/mac80211/spectmgmt.o
  CC      drivers/acpi/acpica/tbdata.o
  CC      drivers/acpi/acpica/tbfadt.o
  CC      drivers/cpufreq/cpufreq_ondemand.o
  CC      lib/crc16.o
  CC      drivers/usb/host/xhci-pci.o
  CC      drivers/firmware/efi/libstub/gop.o
  CC      drivers/clocksource/acpi_pm.o
  AR      drivers/crypto/stm32/built-in.a
  AR      drivers/firmware/imx/built-in.a
  CC      drivers/firmware/efi/libstub/secureboot.o
  AR      drivers/crypto/xilinx/built-in.a
  CC      drivers/clocksource/i8253.o
  AR      drivers/crypto/hisilicon/built-in.a
  CC      arch/x86/kernel/setup_percpu.o
  CC      arch/x86/kernel/mpparse.o
  CC      mm/swapfile.o
  AR      drivers/crypto/intel/keembay/built-in.a
  AR      drivers/crypto/intel/ixp4xx/built-in.a
  AR      drivers/leds/built-in.a
  AR      drivers/crypto/intel/built-in.a
  CC      kernel/reboot.o
  AR      drivers/cpuidle/built-in.a
  CC      fs/fs-writeback.o
  AR      drivers/crypto/starfive/built-in.a
  CC      drivers/firmware/efi/libstub/tpm.o
  AR      drivers/crypto/built-in.a
  CC      drivers/firmware/efi/libstub/file.o
  CC      net/sunrpc/cache.o
  CC      drivers/acpi/acpica/tbfind.o
  CC      drivers/thermal/thermal_hwmon.o
  CC      drivers/hid/usbhid/hid-core.o
  HOSTCC  lib/gen_crc32table
  CC      net/sunrpc/rpc_pipe.o
  CC      drivers/gpu/drm/i915/i915_syncmap.o
  CC      drivers/firmware/efi/libstub/mem.o
  CC      drivers/acpi/acpica/tbinstal.o
  CC      lib/xxhash.o
  CC      fs/pnode.o
  AR      drivers/net/ethernet/cisco/built-in.a
  CC      fs/splice.o
  CC      mm/swap_slots.o
  CC      fs/nfs/nfs4namespace.o
  CC      net/sunrpc/sysfs.o
  CC      drivers/md/dm-stripe.o
  CC      drivers/acpi/acpica/tbprint.o
  CC      drivers/cpufreq/cpufreq_governor.o
  CC      drivers/md/dm-ioctl.o
  AR      drivers/clocksource/built-in.a
  CC      drivers/hid/hid-core.o
  CC      net/wireless/pmsr.o
  CC      drivers/thermal/gov_step_wise.o
  CC      drivers/firmware/efi/libstub/random.o
  CC      drivers/cpufreq/cpufreq_governor_attr_set.o
  CC      net/mac80211/tx.o
  CC      drivers/cpufreq/acpi-cpufreq.o
  CC      arch/x86/kernel/trace_clock.o
  CC      drivers/gpu/drm/i915/i915_user_extensions.o
  AR      drivers/platform/x86/amd/built-in.a
  AR      drivers/platform/x86/intel/built-in.a
  CC      drivers/platform/x86/wmi.o
  CC      net/mac80211/key.o
  CC      net/netfilter/x_tables.o
  CC      net/netfilter/xt_tcpudp.o
  CC      lib/genalloc.o
  CC      fs/nfs/nfs4getroot.o
  CC      net/ipv6/xfrm6_output.o
  CC      fs/nfs/nfs4client.o
  CC      kernel/trace/rethook.o
  CC      drivers/firmware/efi/efi.o
  CC      fs/sync.o
  CC      net/mac80211/util.o
  CC      drivers/acpi/acpica/tbutils.o
  CC      lib/percpu_counter.o
  CC      arch/x86/kernel/trace.o
  CC      fs/utimes.o
  CC      net/ipv4/udp.o
  CC      drivers/firmware/efi/libstub/randomalloc.o
  CC      drivers/thermal/gov_user_space.o
  CC      drivers/gpu/drm/i915/i915_debugfs.o
  CC      kernel/async.o
  CC      drivers/cpufreq/amd-pstate.o
  AR      drivers/gpu/drm/solomon/built-in.a
  CC      fs/nfs/nfs4session.o
  CC      drivers/firmware/efi/libstub/pci.o
  CC      net/mac80211/parse.o
  AR      drivers/usb/host/built-in.a
  AR      drivers/usb/built-in.a
  CC      drivers/gpu/drm/i915/i915_debugfs_params.o
  CC      drivers/acpi/acpica/tbxface.o
  AR      drivers/net/ethernet/cortina/built-in.a
  CC      drivers/gpu/drm/i915/i915_pmu.o
  CC      net/mac80211/wme.o
  CC      lib/audit.o
  CC      drivers/hid/usbhid/hiddev.o
  CC      drivers/hid/usbhid/hid-pidff.o
  CC      drivers/cpufreq/amd-pstate-trace.o
  AR      kernel/trace/built-in.a
  AR      drivers/thermal/built-in.a
  CC      net/sunrpc/svc_xprt.o
  CC      kernel/range.o
  CC      fs/nfs/dns_resolve.o
  CC      drivers/mailbox/mailbox.o
  CC      kernel/smpboot.o
  CC      drivers/platform/x86/wmi-bmof.o
  AR      drivers/net/ethernet/dec/tulip/built-in.a
  CC      arch/x86/kernel/rethook.o
  AR      drivers/net/ethernet/dec/built-in.a
  CC      arch/x86/kernel/vmcore_info_32.o
  AR      drivers/perf/built-in.a
  CC      drivers/md/dm-io.o
  CC      drivers/firmware/efi/libstub/skip_spaces.o
  CC      drivers/acpi/acpica/tbxfload.o
  CC      drivers/mailbox/pcc.o
  AR      drivers/net/ethernet/dlink/built-in.a
  CC      fs/d_path.o
  AR      drivers/net/ethernet/emulex/built-in.a
  AR      drivers/net/ethernet/engleder/built-in.a
  AR      drivers/net/ethernet/ezchip/built-in.a
  CC      fs/nfs/nfs4trace.o
  AR      drivers/net/ethernet/fujitsu/built-in.a
  AR      drivers/net/ethernet/fungible/built-in.a
  AR      drivers/net/ethernet/google/built-in.a
  AR      drivers/net/ethernet/huawei/built-in.a
  CC      drivers/net/ethernet/intel/e1000/e1000_main.o
  CC      drivers/net/ethernet/intel/e1000e/82571.o
  CC      drivers/net/ethernet/intel/e100.o
  CC      drivers/firmware/efi/libstub/lib-cmdline.o
  CC      net/ipv4/udplite.o
  CC      mm/dmapool.o
  CC      lib/syscall.o
  CC      net/ipv6/xfrm6_protocol.o
  CC      drivers/net/ethernet/intel/e1000e/ich8lan.o
  CC      drivers/cpufreq/intel_pstate.o
  CC      drivers/md/dm-kcopyd.o
  CC      drivers/firmware/efi/libstub/lib-ctype.o
  CC      drivers/firmware/efi/libstub/alignedmem.o
  GEN     net/wireless/shipped-certs.c
  CC      mm/hugetlb.o
  CC      net/sunrpc/xprtmultipath.o
  CC      drivers/acpi/acpica/tbxfroot.o
  CC      kernel/ucount.o
  CC      drivers/acpi/acpica/utaddress.o
  CC      drivers/platform/x86/eeepc-laptop.o
  CC      lib/errname.o
  CC      drivers/gpu/drm/i915/gt/gen2_engine_cs.o
  CC      lib/nlattr.o
  CC      arch/x86/kernel/machine_kexec_32.o
  CC      lib/cpu_rmap.o
  CC      drivers/gpu/drm/i915/gt/gen6_engine_cs.o
  CC      drivers/firmware/efi/libstub/relocate.o
  AR      drivers/firmware/psci/built-in.a
  CC      drivers/md/dm-sysfs.o
  AR      drivers/mailbox/built-in.a
  AR      drivers/hwtracing/intel_th/built-in.a
  CC      drivers/md/dm-stats.o
  AR      drivers/android/built-in.a
  AR      drivers/firmware/qcom/built-in.a
  AR      drivers/firmware/smccc/built-in.a
  AR      drivers/firmware/tegra/built-in.a
  AR      drivers/firmware/xilinx/built-in.a
  CC      drivers/firmware/dmi_scan.o
  AR      drivers/hid/usbhid/built-in.a
  CC      drivers/hid/hid-input.o
  AR      drivers/nvmem/layouts/built-in.a
  CC      net/netfilter/xt_CONNSECMARK.o
  CC      drivers/nvmem/core.o
  CC      drivers/firmware/dmi-id.o
  AR      drivers/net/ethernet/broadcom/built-in.a
  CC      fs/stack.o
  CC      drivers/acpi/acpica/utalloc.o
  CC      kernel/regset.o
  AS      arch/x86/kernel/relocate_kernel_32.o
  CC      drivers/acpi/acpica/utascii.o
  CC      drivers/hid/hid-quirks.o
  CC      lib/dynamic_queue_limits.o
  CC      lib/glob.o
  CC      arch/x86/kernel/crash_dump_32.o
  CC      drivers/acpi/acpica/utbuffer.o
  CC      drivers/gpu/drm/i915/gt/gen6_ppgtt.o
  CC      drivers/firmware/efi/vars.o
  CC      drivers/firmware/efi/libstub/printk.o
  CC      drivers/firmware/memmap.o
  CC      drivers/acpi/acpica/utcksum.o
  CC      arch/x86/kernel/crash.o
  CC      kernel/ksyms_common.o
  CC      net/netfilter/xt_NFLOG.o
  CC      net/ipv4/udp_offload.o
  CC      net/mac80211/chan.o
  CC      drivers/acpi/acpica/utcopy.o
  CC      drivers/gpu/drm/i915/gt/gen7_renderclear.o
  CC      drivers/firmware/efi/reboot.o
  CC      drivers/acpi/acpica/utexcep.o
  AR      drivers/net/ethernet/i825xx/built-in.a
  CC      drivers/acpi/acpica/utdebug.o
  CC      drivers/firmware/efi/memattr.o
  CC      net/ipv6/netfilter.o
  CC [M]  drivers/gpu/drm/scheduler/sched_main.o
  CC      drivers/md/dm-rq.o
  CC [M]  drivers/gpu/drm/scheduler/sched_fence.o
  CC      net/sunrpc/stats.o
  CC      net/mac80211/trace.o
  CC      net/sunrpc/sysctl.o
  CC [M]  drivers/gpu/drm/scheduler/sched_entity.o
  CC      drivers/firmware/efi/tpm.o
  CC      drivers/platform/x86/p2sb.o
  CC      drivers/firmware/efi/memmap.o
  CC      drivers/firmware/efi/capsule.o
  CC      fs/fs_struct.o
  CC      drivers/firmware/efi/libstub/vsprintf.o
  CC      drivers/hid/hid-debug.o
  CC      drivers/firmware/efi/esrt.o
  CC      drivers/md/dm-io-rewind.o
  CC      drivers/firmware/efi/libstub/x86-stub.o
  CC      drivers/acpi/acpica/utdecode.o
  CC      fs/statfs.o
  CC      drivers/acpi/acpica/utdelete.o
  CC      lib/strncpy_from_user.o
  CC      net/netfilter/xt_SECMARK.o
  CC      lib/strnlen_user.o
  AR      drivers/nvmem/built-in.a
  CC      net/wireless/shipped-certs.o
  CC      kernel/groups.o
  CC      drivers/firmware/efi/runtime-wrappers.o
  CC      arch/x86/kernel/module.o
  CC      drivers/net/ethernet/intel/e1000e/80003es2lan.o
  HOSTCC  drivers/gpu/drm/xe/xe_gen_wa_oob
  CC      net/ipv6/proc.o
  CC      mm/mmu_notifier.o
  CC      net/ipv4/arp.o
  CC      arch/x86/kernel/doublefault_32.o
  CC      drivers/firmware/efi/libstub/smbios.o
  GEN     xe_wa_oob.c xe_wa_oob.h
  CC [M]  drivers/gpu/drm/xe/xe_bb.o
  CC      drivers/md/dm-builtin.o
  CC      drivers/hid/hidraw.o
  AR      drivers/platform/x86/built-in.a
  AR      drivers/cpufreq/built-in.a
  CC      drivers/acpi/acpica/uterror.o
  AR      drivers/platform/surface/built-in.a
  CC      drivers/hid/hid-generic.o
  AR      drivers/platform/built-in.a
  CC      drivers/firmware/efi/capsule-loader.o
  CC      drivers/firmware/efi/earlycon.o
  CC      net/mac80211/mlme.o
  CC      drivers/net/ethernet/intel/e1000e/mac.o
  CC      drivers/md/dm-raid1.o
  CC      drivers/gpu/drm/i915/gt/gen8_engine_cs.o
  CC      drivers/net/ethernet/intel/e1000e/manage.o
  CC      drivers/gpu/drm/i915/gt/gen8_ppgtt.o
  CC      mm/migrate.o
  CC      drivers/net/ethernet/intel/e1000/e1000_hw.o
  CC      net/netfilter/xt_TCPMSS.o
  AR      drivers/net/ethernet/microsoft/built-in.a
  AR      drivers/net/ethernet/litex/built-in.a
  CC      lib/net_utils.o
  AR      drivers/net/ethernet/marvell/octeon_ep/built-in.a
  AR      drivers/net/ethernet/marvell/octeon_ep_vf/built-in.a
  AR      drivers/net/ethernet/marvell/octeontx2/built-in.a
  AR      drivers/net/ethernet/marvell/prestera/built-in.a
  CC      drivers/net/ethernet/marvell/sky2.o
  CC      drivers/md/dm-log.o
  CC      drivers/net/ethernet/intel/e1000/e1000_ethtool.o
  CC      net/ipv6/syncookies.o
  CC      fs/fs_pin.o
  CC      net/ipv6/calipso.o
  CC      drivers/acpi/acpica/uteval.o
  CC      drivers/acpi/processor_pdc.o
  CC [M]  drivers/gpu/drm/xe/xe_bo.o
  CC      net/netfilter/xt_conntrack.o
  CC      net/ipv4/icmp.o
  CC      drivers/hid/hid-a4tech.o
  CC      kernel/kcmp.o
  LD [M]  drivers/gpu/drm/scheduler/gpu-sched.o
  STUBCPY drivers/firmware/efi/libstub/alignedmem.stub.o
  CC      drivers/hid/hid-apple.o
  STUBCPY drivers/firmware/efi/libstub/efi-stub-helper.stub.o
  STUBCPY drivers/firmware/efi/libstub/file.stub.o
  STUBCPY drivers/firmware/efi/libstub/gop.stub.o
  STUBCPY drivers/firmware/efi/libstub/lib-cmdline.stub.o
  STUBCPY drivers/firmware/efi/libstub/lib-ctype.stub.o
  CC      drivers/gpu/drm/drm_aperture.o
  STUBCPY drivers/firmware/efi/libstub/mem.stub.o
  CC      drivers/hid/hid-belkin.o
  CC      drivers/gpu/drm/drm_atomic.o
  STUBCPY drivers/firmware/efi/libstub/pci.stub.o
  STUBCPY drivers/firmware/efi/libstub/printk.stub.o
  CC      drivers/gpu/drm/drm_atomic_uapi.o
  CC [M]  drivers/gpu/drm/xe/xe_bo_evict.o
  STUBCPY drivers/firmware/efi/libstub/random.stub.o
  CC      net/netfilter/xt_policy.o
  STUBCPY drivers/firmware/efi/libstub/randomalloc.stub.o
  STUBCPY drivers/firmware/efi/libstub/relocate.stub.o
  STUBCPY drivers/firmware/efi/libstub/secureboot.stub.o
  CC      mm/page_counter.o
  CC      arch/x86/kernel/early_printk.o
  STUBCPY drivers/firmware/efi/libstub/skip_spaces.stub.o
  CC      net/ipv6/ah6.o
  STUBCPY drivers/firmware/efi/libstub/smbios.stub.o
  CC      fs/nsfs.o
  STUBCPY drivers/firmware/efi/libstub/tpm.stub.o
  CC      arch/x86/kernel/hpet.o
  STUBCPY drivers/firmware/efi/libstub/vsprintf.stub.o
  STUBCPY drivers/firmware/efi/libstub/x86-stub.stub.o
  CC      drivers/acpi/acpica/utglobal.o
  AR      drivers/firmware/efi/libstub/lib.a
  AR      drivers/firmware/efi/built-in.a
  CC      fs/fs_types.o
  AR      drivers/firmware/built-in.a
  CC      kernel/freezer.o
  CC      drivers/acpi/acpica/uthex.o
  CC      lib/sg_pool.o
  CC      net/ipv6/esp6.o
  CC      drivers/acpi/ec.o
  CC      lib/stackdepot.o
  CC      drivers/net/ethernet/intel/e1000e/nvm.o
  CC      drivers/gpu/drm/drm_auth.o
  CC      drivers/gpu/drm/i915/gt/intel_breadcrumbs.o
  CC      arch/x86/kernel/amd_nb.o
  CC      fs/nfs/nfs4sysctl.o
  CC      drivers/md/dm-region-hash.o
  CC      net/ipv6/sit.o
  CC      net/netfilter/xt_state.o
  CC      drivers/md/dm-zero.o
  CC      net/mac80211/tdls.o
  CC      drivers/acpi/acpica/utids.o
  CC [M]  drivers/gpu/drm/xe/xe_devcoredump.o
  CC      fs/fs_context.o
  CC      net/mac80211/ocb.o
  CC      kernel/profile.o
  CC      drivers/net/ethernet/intel/e1000e/phy.o
  CC      drivers/gpu/drm/i915/gt/intel_context.o
  CC      drivers/hid/hid-cherry.o
  CC      drivers/gpu/drm/i915/gt/intel_context_sseu.o
  CC      drivers/acpi/acpica/utinit.o
  CC      fs/fs_parser.o
  CC [M]  drivers/gpu/drm/xe/xe_device.o
  CC      arch/x86/kernel/kvm.o
  CC      mm/hugetlb_cgroup.o
  CC      arch/x86/kernel/kvmclock.o
  CC [M]  net/netfilter/nf_log_syslog.o
  CC      drivers/acpi/dock.o
  CC      drivers/acpi/acpica/utlock.o
  AR      net/sunrpc/built-in.a
  CC      drivers/net/ethernet/intel/e1000e/param.o
  CC      net/mac80211/airtime.o
  CC      drivers/acpi/acpica/utmath.o
  CC      arch/x86/kernel/paravirt.o
  CC [M]  net/netfilter/xt_mark.o
  CC      mm/early_ioremap.o
  CC      lib/asn1_decoder.o
  CC      mm/secretmem.o
  CC      net/ipv4/devinet.o
  CC      drivers/net/ethernet/intel/e1000e/ethtool.o
  CC      kernel/stacktrace.o
  CC      net/ipv6/addrconf_core.o
  CC      drivers/hid/hid-chicony.o
  CC      drivers/net/ethernet/intel/e1000/e1000_param.o
  CC [M]  net/netfilter/xt_nat.o
  CC [M]  drivers/gpu/drm/xe/xe_device_sysfs.o
  AR      drivers/md/built-in.a
  CC [M]  net/netfilter/xt_LOG.o
  CC      kernel/dma.o
  CC      drivers/acpi/acpica/utmisc.o
  CC [M]  net/netfilter/xt_MASQUERADE.o
  CC      net/mac80211/eht.o
  CC      drivers/net/ethernet/intel/e1000e/netdev.o
  CC [M]  drivers/gpu/drm/xe/xe_dma_buf.o
  GEN     lib/oid_registry_data.c
  AR      fs/nfs/built-in.a
  CC      lib/ucs2_string.o
  CC      drivers/hid/hid-cypress.o
  CC      drivers/acpi/pci_root.o
  CC [M]  net/netfilter/xt_addrtype.o
  CC      net/ipv6/exthdrs_core.o
  CC      drivers/gpu/drm/drm_blend.o
  CC      fs/fsopen.o
  CC      arch/x86/kernel/pvclock.o
  CC      fs/init.o
  CC      drivers/gpu/drm/i915/gt/intel_engine_cs.o
  CC      arch/x86/kernel/pcspeaker.o
  CC      drivers/gpu/drm/i915/gt/intel_engine_heartbeat.o
  CC      drivers/acpi/acpica/utmutex.o
  CC      drivers/acpi/acpica/utnonansi.o
  CC [M]  drivers/gpu/drm/xe/xe_drm_client.o
  CC      kernel/smp.o
  CC      kernel/uid16.o
  CC      fs/kernel_read_file.o
  CC      net/ipv6/ip6_checksum.o
  CC      lib/sbitmap.o
  CC      drivers/hid/hid-ezkey.o
  CC      drivers/gpu/drm/i915/gt/intel_engine_pm.o
  CC      drivers/net/ethernet/intel/e1000e/ptp.o
  AR      drivers/net/ethernet/mellanox/built-in.a
  CC      mm/hmm.o
  AR      drivers/net/ethernet/meta/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_exec.o
  AR      drivers/net/ethernet/microchip/built-in.a
  AR      drivers/net/ethernet/micrel/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_execlist.o
  AR      drivers/net/ethernet/mscc/built-in.a
  CC      lib/group_cpus.o
  CC      net/ipv6/ip6_icmp.o
  CC      drivers/gpu/drm/i915/gt/intel_engine_user.o
  CC      drivers/acpi/acpica/utobject.o
  CC      drivers/acpi/acpica/utosi.o
  CC      net/ipv6/output_core.o
  CC      kernel/kallsyms.o
  CC      arch/x86/kernel/check.o
  CC      drivers/hid/hid-gyration.o
  CC      arch/x86/kernel/uprobes.o
  CC [M]  drivers/gpu/drm/xe/xe_exec_queue.o
  CC      fs/mnt_idmapping.o
  CC [M]  drivers/gpu/drm/xe/xe_force_wake.o
  AR      drivers/net/ethernet/intel/e1000/built-in.a
  CC      net/ipv4/af_inet.o
  CC      kernel/acct.o
  CC      drivers/gpu/drm/i915/gt/intel_execlists_submission.o
  CC      net/ipv6/protocol.o
  CC      drivers/gpu/drm/i915/gt/intel_ggtt.o
  CC [M]  drivers/gpu/drm/xe/xe_ggtt.o
  CC      fs/remap_range.o
  CC      drivers/acpi/pci_link.o
  AR      drivers/net/ethernet/marvell/built-in.a
  CC      net/mac80211/led.o
  CC      net/mac80211/pm.o
  CC      net/ipv4/igmp.o
  CC      net/mac80211/rc80211_minstrel_ht.o
  CC      drivers/acpi/acpica/utownerid.o
  CC      net/ipv4/fib_frontend.o
  CC [M]  drivers/gpu/drm/xe/xe_gpu_scheduler.o
  CC      drivers/hid/hid-ite.o
  CC      kernel/vmcore_info.o
  AR      net/netfilter/built-in.a
  CC      kernel/elfcorehdr.o
  CC      lib/fw_table.o
  CC      mm/memfd.o
  CC      drivers/hid/hid-kensington.o
  CC      net/mac80211/wbrf.o
  AR      drivers/net/ethernet/myricom/built-in.a
  CC      mm/ptdump.o
  CC      drivers/gpu/drm/drm_bridge.o
  CC      drivers/gpu/drm/drm_cache.o
  CC      drivers/gpu/drm/i915/gt/intel_ggtt_fencing.o
  CC [M]  drivers/gpu/drm/xe/xe_gsc.o
  CC      mm/execmem.o
  CC      drivers/acpi/acpica/utpredef.o
  CC      drivers/gpu/drm/i915/gt/intel_gt.o
  CC [M]  drivers/gpu/drm/xe/xe_gsc_debugfs.o
  CC      drivers/gpu/drm/drm_client.o
  CC [M]  drivers/gpu/drm/xe/xe_gsc_proxy.o
  CC [M]  drivers/gpu/drm/xe/xe_gsc_submit.o
  CC      net/ipv6/ip6_offload.o
  AR      drivers/net/ethernet/natsemi/built-in.a
  CC      net/ipv6/tcpv6_offload.o
  AR      drivers/net/ethernet/neterion/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_gt.o
  CC      net/ipv4/fib_semantics.o
  AR      drivers/net/ethernet/netronome/built-in.a
  CC      drivers/acpi/pci_irq.o
  CC      arch/x86/kernel/perf_regs.o
  CC      net/ipv6/exthdrs_offload.o
  CC      net/ipv4/fib_trie.o
  CC      drivers/acpi/acpica/utresdecode.o
  CC      kernel/crash_reserve.o
  CC      drivers/hid/hid-lg.o
  CC      net/ipv6/inet6_hashtables.o
  AR      drivers/net/ethernet/ni/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_gt_ccs_mode.o
  CC      drivers/net/ethernet/nvidia/forcedeth.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_clock.o
  AR      drivers/net/ethernet/oki-semi/built-in.a
  AR      lib/lib.a
  GEN     lib/crc32table.h
  CC [M]  drivers/gpu/drm/xe/xe_gt_freq.o
  AR      drivers/net/ethernet/packetengines/built-in.a
  CC      lib/oid_registry.o
  CC      kernel/kexec_core.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_idle.o
  CC      net/ipv4/fib_notifier.o
  CC      fs/pidfs.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_mcr.o
  CC      drivers/hid/hid-lgff.o
  CC      drivers/gpu/drm/drm_client_modeset.o
  CC      arch/x86/kernel/tracepoint.o
  CC      drivers/acpi/acpica/utresrc.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_pagefault.o
  CC      net/ipv4/inet_fragment.o
  CC      net/ipv4/ping.o
  AR      mm/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_gt_sysfs.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_throttle.o
  CC      drivers/gpu/drm/drm_color_mgmt.o
  CC      lib/crc32.o
  CC      net/ipv4/ip_tunnel_core.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_tlb_invalidation.o
  CC      net/ipv4/gre_offload.o
  AR      drivers/net/ethernet/qlogic/built-in.a
  CC      drivers/hid/hid-lg4ff.o
  CC      drivers/net/ethernet/realtek/8139too.o
  AR      drivers/net/ethernet/qualcomm/emac/built-in.a
  AR      drivers/net/ethernet/qualcomm/built-in.a
  CC      arch/x86/kernel/itmt.o
  CC      fs/buffer.o
  CC      drivers/gpu/drm/drm_connector.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_topology.o
  CC      drivers/acpi/acpica/utstate.o
  CC      net/ipv4/metrics.o
  CC      drivers/acpi/acpi_apd.o
  CC      net/ipv6/mcast_snoop.o
  CC [M]  drivers/gpu/drm/xe/xe_guc.o
  CC      drivers/hid/hid-lg-g15.o
  CC      kernel/crash_core.o
  CC      kernel/kexec.o
  CC      net/ipv4/netlink.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_ads.o
  AR      net/wireless/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.o
  CC      drivers/acpi/acpica/utstring.o
  CC      drivers/net/ethernet/realtek/r8169_main.o
  CC      drivers/net/ethernet/realtek/r8169_firmware.o
  AR      lib/built-in.a
  CC [M]  drivers/gpu/drm/xe/xe_guc_ct.o
  CC      fs/mpage.o
  CC      drivers/net/ethernet/realtek/r8169_phy_config.o
  CC      drivers/acpi/acpica/utstrsuppt.o
  CC      net/ipv4/nexthop.o
  CC      fs/proc_namespace.o
  CC      drivers/acpi/acpica/utstrtoul64.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.o
  CC      fs/direct-io.o
  CC      drivers/acpi/acpica/utxface.o
  CC      drivers/gpu/drm/drm_crtc.o
  CC      drivers/acpi/acpi_platform.o
  CC      arch/x86/kernel/umip.o
  CC      drivers/acpi/acpica/utxfinit.o
  CC      net/ipv4/udp_tunnel_stub.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_clock_utils.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_debugfs.o
  CC      drivers/acpi/acpica/utxferror.o
  CC      drivers/gpu/drm/drm_displayid.o
  CC      drivers/acpi/acpica/utxfmutex.o
  CC      drivers/gpu/drm/drm_drv.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.o
  CC      drivers/hid/hid-microsoft.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_irq.o
  CC      fs/eventpoll.o
  AR      drivers/net/ethernet/intel/e1000e/built-in.a
  CC      fs/anon_inodes.o
  AR      drivers/net/ethernet/intel/built-in.a
  CC      drivers/acpi/acpi_pnp.o
  AR      drivers/net/ethernet/renesas/built-in.a
  CC      kernel/utsname.o
  CC      arch/x86/kernel/unwind_frame.o
  CC      drivers/hid/hid-monterey.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_db_mgr.o
  CC      net/ipv4/ip_tunnel.o
  CC      drivers/gpu/drm/drm_dumb_buffers.o
  CC      net/ipv4/sysctl_net_ipv4.o
  CC      drivers/gpu/drm/drm_edid.o
  CC      net/ipv4/proc.o
  CC      fs/signalfd.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_mcr.o
  CC      fs/timerfd.o
  CC      net/ipv4/fib_rules.o
  AR      drivers/acpi/acpica/built-in.a
  CC      fs/eventfd.o
  AR      net/ipv6/built-in.a
  CC      net/ipv4/ipmr.o
  CC      kernel/pid_namespace.o
  AR      drivers/net/ethernet/rdc/built-in.a
  CC      drivers/gpu/drm/drm_eld.o
  CC      drivers/acpi/power.o
  CC      drivers/gpu/drm/drm_encoder.o
  CC      drivers/acpi/event.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_pm.o
  CC      drivers/hid/hid-ntrig.o
  CC      drivers/gpu/drm/drm_file.o
  CC      drivers/gpu/drm/drm_fourcc.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_hwconfig.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_id_mgr.o
  CC      net/ipv4/ipmr_base.o
  CC      drivers/hid/hid-pl.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_klv_helpers.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_log.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.o
  CC      drivers/gpu/drm/drm_framebuffer.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_pm_irq.o
  CC      fs/aio.o
  CC      fs/locks.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_pc.o
  AR      arch/x86/kernel/built-in.a
  CC      drivers/acpi/evged.o
  CC      fs/binfmt_misc.o
  AR      arch/x86/built-in.a
  AR      drivers/net/ethernet/rocker/built-in.a
  CC      net/ipv4/syncookies.o
  AR      drivers/net/ethernet/samsung/built-in.a
  AR      drivers/net/ethernet/seeq/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_gt_requests.o
  CC      net/ipv4/tunnel4.o
  AR      drivers/net/ethernet/silan/built-in.a
  AR      drivers/net/ethernet/sis/built-in.a
  AR      drivers/net/ethernet/sfc/built-in.a
  CC      drivers/gpu/drm/drm_gem.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_submit.o
  CC      net/ipv4/ipconfig.o
  CC      drivers/gpu/drm/drm_ioctl.o
  CC      net/ipv4/netfilter.o
  CC [M]  drivers/gpu/drm/xe/xe_heci_gsc.o
  CC      fs/binfmt_script.o
  CC      kernel/stop_machine.o
  CC      drivers/acpi/sysfs.o
  CC      fs/binfmt_elf.o
  CC      drivers/hid/hid-petalynx.o
  CC      fs/mbcache.o
  CC      kernel/audit.o
  CC      net/ipv4/tcp_cubic.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_sysfs.o
  CC      drivers/acpi/property.o
  CC      drivers/acpi/debugfs.o
  CC      drivers/hid/hid-redragon.o
  CC [M]  drivers/gpu/drm/xe/xe_hw_engine.o
  CC [M]  drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.o
  CC      fs/posix_acl.o
  CC      drivers/acpi/acpi_lpat.o
  AR      drivers/net/ethernet/nvidia/built-in.a
  CC      drivers/hid/hid-samsung.o
  CC      drivers/hid/hid-sony.o
  AR      net/mac80211/built-in.a
  CC      net/ipv4/tcp_sigpool.o
  CC      drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.o
  CC      drivers/gpu/drm/drm_lease.o
  CC      drivers/gpu/drm/i915/gt/intel_gtt.o
  CC      fs/coredump.o
  CC      fs/drop_caches.o
  CC      net/ipv4/cipso_ipv4.o
  CC      drivers/gpu/drm/drm_managed.o
  CC      kernel/auditfilter.o
  CC      net/ipv4/xfrm4_policy.o
  CC      drivers/acpi/acpi_pcc.o
  CC      drivers/gpu/drm/i915/gt/intel_llc.o
  CC      fs/sysctls.o
  CC [M]  drivers/gpu/drm/xe/xe_hw_engine_group.o
  CC [M]  drivers/gpu/drm/xe/xe_hw_fence.o
  CC      drivers/hid/hid-sunplus.o
  CC      drivers/acpi/ac.o
  CC [M]  drivers/gpu/drm/xe/xe_huc.o
  CC      net/ipv4/xfrm4_state.o
  CC      drivers/acpi/button.o
  CC      drivers/acpi/fan_core.o
  CC      drivers/hid/hid-topseed.o
  CC      net/ipv4/xfrm4_input.o
  CC      fs/fhandle.o
  CC      drivers/gpu/drm/i915/gt/intel_lrc.o
  CC      drivers/gpu/drm/drm_mm.o
  CC      drivers/gpu/drm/i915/gt/intel_migrate.o
  CC      net/ipv4/xfrm4_output.o
  CC      net/ipv4/xfrm4_protocol.o
  CC      drivers/acpi/fan_attr.o
  CC      drivers/gpu/drm/drm_mode_config.o
  AR      drivers/net/ethernet/realtek/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_mocs.o
  AR      drivers/net/ethernet/smsc/built-in.a
  AR      drivers/net/ethernet/socionext/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_ppgtt.o
  AR      drivers/net/ethernet/stmicro/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_rc6.o
  AR      drivers/net/ethernet/sun/built-in.a
  AR      drivers/net/ethernet/tehuti/built-in.a
  CC      drivers/gpu/drm/drm_mode_object.o
  CC      drivers/acpi/fan_hwmon.o
  CC      drivers/acpi/acpi_video.o
  AR      drivers/net/ethernet/ti/built-in.a
  AR      drivers/net/ethernet/vertexcom/built-in.a
  AR      drivers/net/ethernet/via/built-in.a
  AR      drivers/net/ethernet/wangxun/built-in.a
  AR      drivers/net/ethernet/wiznet/built-in.a
  CC      kernel/auditsc.o
  AR      drivers/net/ethernet/xilinx/built-in.a
  AR      drivers/net/ethernet/xircom/built-in.a
  AR      drivers/net/ethernet/synopsys/built-in.a
  CC      drivers/acpi/video_detect.o
  AR      drivers/net/ethernet/pensando/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_region_lmem.o
  CC      kernel/audit_watch.o
  AR      drivers/net/ethernet/built-in.a
  CC      drivers/gpu/drm/drm_modes.o
  CC      drivers/gpu/drm/drm_modeset_lock.o
  CC      drivers/acpi/processor_driver.o
  CC      drivers/gpu/drm/drm_plane.o
  AR      drivers/net/built-in.a
  CC      drivers/gpu/drm/drm_prime.o
  CC [M]  drivers/gpu/drm/xe/xe_irq.o
  CC      kernel/audit_fsnotify.o
  CC      drivers/gpu/drm/drm_print.o
  CC      kernel/audit_tree.o
  CC      kernel/kprobes.o
  CC      kernel/seccomp.o
  AR      drivers/hid/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_renderstate.o
  CC      kernel/relay.o
  CC [M]  drivers/gpu/drm/xe/xe_lrc.o
  CC      drivers/acpi/processor_thermal.o
  CC [M]  drivers/gpu/drm/xe/xe_migrate.o
  CC      drivers/acpi/processor_idle.o
  CC      drivers/acpi/processor_throttling.o
  CC      drivers/gpu/drm/i915/gt/intel_reset.o
  CC      drivers/gpu/drm/i915/gt/intel_ring.o
  CC      drivers/gpu/drm/i915/gt/intel_ring_submission.o
  CC      drivers/acpi/processor_perflib.o
  CC      kernel/utsname_sysctl.o
  CC      kernel/delayacct.o
  CC      drivers/gpu/drm/drm_property.o
  CC      drivers/gpu/drm/drm_syncobj.o
  CC      drivers/gpu/drm/i915/gt/intel_rps.o
  CC [M]  drivers/gpu/drm/xe/xe_mmio.o
  CC [M]  drivers/gpu/drm/xe/xe_mocs.o
  CC      drivers/gpu/drm/drm_sysfs.o
  CC      drivers/acpi/container.o
  CC      kernel/taskstats.o
  CC      kernel/tsacct.o
  CC      drivers/gpu/drm/drm_trace_points.o
  AR      fs/built-in.a
  CC      drivers/gpu/drm/drm_vblank.o
  CC      drivers/gpu/drm/drm_vblank_work.o
  CC      kernel/tracepoint.o
  CC      drivers/gpu/drm/drm_vma_manager.o
  CC      drivers/gpu/drm/i915/gt/intel_sa_media.o
  CC      kernel/irq_work.o
  CC      drivers/gpu/drm/i915/gt/intel_sseu.o
  CC      drivers/gpu/drm/drm_writeback.o
  CC      drivers/gpu/drm/drm_panel.o
  CC      drivers/acpi/thermal_lib.o
  CC      kernel/static_call.o
  CC      kernel/padata.o
  CC      drivers/acpi/thermal.o
  CC      drivers/acpi/nhlt.o
  CC      kernel/jump_label.o
  CC      drivers/gpu/drm/i915/gt/intel_sseu_debugfs.o
  AR      net/ipv4/built-in.a
  AR      net/built-in.a
  CC      drivers/gpu/drm/i915/gt/intel_timeline.o
  CC      drivers/gpu/drm/i915/gt/intel_tlb.o
  CC [M]  drivers/gpu/drm/xe/xe_module.o
  CC      kernel/context_tracking.o
  CC      drivers/gpu/drm/i915/gt/intel_wopcm.o
  CC [M]  drivers/gpu/drm/xe/xe_oa.o
  CC [M]  drivers/gpu/drm/xe/xe_observation.o
  CC      kernel/iomem.o
  CC      drivers/acpi/acpi_memhotplug.o
  CC      drivers/acpi/ioapic.o
  CC      drivers/gpu/drm/drm_pci.o
  CC      kernel/rseq.o
  CC      drivers/gpu/drm/i915/gt/intel_workarounds.o
  CC      drivers/gpu/drm/drm_debugfs.o
  CC      drivers/acpi/battery.o
  CC      drivers/gpu/drm/i915/gt/shmem_utils.o
  CC [M]  drivers/gpu/drm/xe/xe_pat.o
  CC      drivers/gpu/drm/drm_debugfs_crc.o
  CC [M]  drivers/gpu/drm/xe/xe_pci.o
  CC      drivers/gpu/drm/drm_panel_orientation_quirks.o
  CC      drivers/gpu/drm/drm_buddy.o
  CC      drivers/gpu/drm/i915/gt/sysfs_engines.o
  CC      drivers/gpu/drm/drm_gem_shmem_helper.o
  CC      drivers/gpu/drm/drm_atomic_helper.o
  CC      drivers/gpu/drm/drm_atomic_state_helper.o
  CC      drivers/acpi/bgrt.o
  CC      drivers/gpu/drm/drm_crtc_helper.o
  CC      drivers/acpi/spcr.o
  CC [M]  drivers/gpu/drm/xe/xe_pcode.o
  CC      drivers/gpu/drm/drm_damage_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_pm.o
  CC      drivers/gpu/drm/i915/gt/intel_ggtt_gmch.o
  CC      drivers/gpu/drm/i915/gt/gen6_renderstate.o
  CC      drivers/gpu/drm/i915/gt/gen7_renderstate.o
  CC      drivers/gpu/drm/i915/gt/gen8_renderstate.o
  CC      drivers/gpu/drm/i915/gt/gen9_renderstate.o
  CC      drivers/gpu/drm/drm_encoder_slave.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_busy.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_clflush.o
  CC [M]  drivers/gpu/drm/xe/xe_preempt_fence.o
  CC [M]  drivers/gpu/drm/xe/xe_pt.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_context.o
  CC      drivers/gpu/drm/drm_flip_work.o
  CC      drivers/gpu/drm/drm_format_helper.o
  CC      drivers/gpu/drm/drm_gem_atomic_helper.o
  CC      drivers/gpu/drm/drm_gem_framebuffer_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_pt_walk.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_create.o
  CC [M]  drivers/gpu/drm/xe/xe_query.o
  CC      drivers/gpu/drm/drm_kms_helper_common.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_dmabuf.o
  CC [M]  drivers/gpu/drm/xe/xe_range_fence.o
  CC [M]  drivers/gpu/drm/xe/xe_reg_sr.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_domain.o
  CC      drivers/gpu/drm/drm_modeset_helper.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_execbuffer.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_internal.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_lmem.o
  CC      drivers/gpu/drm/drm_plane_helper.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_mman.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_object.o
  AR      kernel/built-in.a
  CC      drivers/gpu/drm/drm_probe_helper.o
  CC      drivers/gpu/drm/drm_rect.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_pages.o
  CC      drivers/gpu/drm/drm_self_refresh_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_reg_whitelist.o
  CC [M]  drivers/gpu/drm/xe/xe_rtp.o
  CC      drivers/gpu/drm/drm_simple_kms_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_ring_ops.o
  CC      drivers/gpu/drm/bridge/panel.o
  CC      drivers/gpu/drm/drm_mipi_dsi.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_phys.o
  CC [M]  drivers/gpu/drm/drm_exec.o
  AR      drivers/acpi/built-in.a
  CC      drivers/gpu/drm/i915/gem/i915_gem_pm.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_region.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_shmem.o
  CC [M]  drivers/gpu/drm/drm_gpuvm.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_shrinker.o
  CC [M]  drivers/gpu/drm/xe/xe_sa.o
  CC [M]  drivers/gpu/drm/drm_suballoc.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_stolen.o
  CC [M]  drivers/gpu/drm/xe/xe_sched_job.o
  CC [M]  drivers/gpu/drm/drm_gem_ttm_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_step.o
  CC [M]  drivers/gpu/drm/xe/xe_sync.o
  CC [M]  drivers/gpu/drm/xe/xe_tile.o
  CC [M]  drivers/gpu/drm/xe/xe_tile_sysfs.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_throttle.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_tiling.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_ttm.o
  CC [M]  drivers/gpu/drm/xe/xe_trace.o
  CC [M]  drivers/gpu/drm/xe/xe_trace_bo.o
  CC [M]  drivers/gpu/drm/xe/xe_trace_guc.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_ttm_move.o
  CC [M]  drivers/gpu/drm/xe/xe_ttm_sys_mgr.o
  CC [M]  drivers/gpu/drm/xe/xe_ttm_stolen_mgr.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_userptr.o
  CC      drivers/gpu/drm/i915/gem/i915_gem_wait.o
  CC      drivers/gpu/drm/i915/gem/i915_gemfs.o
  CC      drivers/gpu/drm/i915/i915_active.o
  CC      drivers/gpu/drm/i915/i915_cmd_parser.o
  CC [M]  drivers/gpu/drm/xe/xe_ttm_vram_mgr.o
  CC [M]  drivers/gpu/drm/xe/xe_tuning.o
  CC [M]  drivers/gpu/drm/xe/xe_uc.o
  CC      drivers/gpu/drm/i915/i915_deps.o
  CC [M]  drivers/gpu/drm/xe/xe_uc_fw.o
  LD [M]  drivers/gpu/drm/drm_suballoc_helper.o
  CC [M]  drivers/gpu/drm/xe/xe_vm.o
  CC [M]  drivers/gpu/drm/xe/xe_vram.o
  CC      drivers/gpu/drm/i915/i915_gem.o
  LD [M]  drivers/gpu/drm/drm_ttm_helper.o
  CC      drivers/gpu/drm/i915/i915_gem_evict.o
  CC      drivers/gpu/drm/i915/i915_gem_gtt.o
  CC [M]  drivers/gpu/drm/xe/xe_vram_freq.o
  CC [M]  drivers/gpu/drm/xe/xe_wait_user_fence.o
  CC [M]  drivers/gpu/drm/xe/xe_wa.o
  CC      drivers/gpu/drm/i915/i915_gem_ww.o
  CC [M]  drivers/gpu/drm/xe/xe_wopcm.o
  CC [M]  drivers/gpu/drm/xe/xe_hmm.o
  CC [M]  drivers/gpu/drm/xe/xe_hwmon.o
  CC      drivers/gpu/drm/i915/i915_query.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_sriov_vf.o
  CC      drivers/gpu/drm/i915/i915_request.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_relay.o
  CC [M]  drivers/gpu/drm/xe/xe_memirq.o
  CC [M]  drivers/gpu/drm/xe/xe_sriov.o
  CC      drivers/gpu/drm/i915/i915_scheduler.o
  CC      drivers/gpu/drm/i915/i915_trace_points.o
  CC [M]  drivers/gpu/drm/xe/display/ext/i915_irq.o
  CC [M]  drivers/gpu/drm/xe/display/ext/i915_utils.o
  CC [M]  drivers/gpu/drm/xe/display/intel_fb_bo.o
  CC      drivers/gpu/drm/i915/i915_ttm_buddy_manager.o
  CC [M]  drivers/gpu/drm/xe/display/intel_fbdev_fb.o
  CC      drivers/gpu/drm/i915/i915_vma.o
  CC [M]  drivers/gpu/drm/xe/display/xe_display.o
  CC [M]  drivers/gpu/drm/xe/display/xe_display_misc.o
  CC [M]  drivers/gpu/drm/xe/display/xe_display_rps.o
  CC      drivers/gpu/drm/i915/i915_vma_resource.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.o
  CC [M]  drivers/gpu/drm/xe/display/xe_display_wa.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.o
  CC [M]  drivers/gpu/drm/xe/display/xe_dsb_buffer.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_debugfs.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_ads.o
  CC [M]  drivers/gpu/drm/xe/display/xe_fb_pin.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_capture.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_ct.o
  CC [M]  drivers/gpu/drm/xe/display/xe_hdcp_gsc.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.o
  CC [M]  drivers/gpu/drm/xe/display/xe_plane_initial.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_fw.o
  CC [M]  drivers/gpu/drm/xe/display/xe_tdf.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.o
  CC [M]  drivers/gpu/drm/xe/i915-soc/intel_dram.o
  CC [M]  drivers/gpu/drm/xe/i915-soc/intel_pch.o
  CC [M]  drivers/gpu/drm/xe/i915-display/icl_dsi.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_log.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_alpm.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_atomic.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_atomic_plane.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_rc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_audio.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_backlight.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_bios.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_bw.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_cdclk.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_guc_submission.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_huc.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_color.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_combo_phy.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_connector.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_huc_fw.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_crtc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_crtc_state_dump.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_cursor.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_cx0_phy.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_uc.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.o
  CC      drivers/gpu/drm/i915/gt/uc/intel_uc_fw.o
  CC      drivers/gpu/drm/i915/gt/intel_gsc.o
  CC      drivers/gpu/drm/i915/i915_hwmon.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_ddi.o
  CC      drivers/gpu/drm/i915/display/hsw_ips.o
  CC      drivers/gpu/drm/i915/display/i9xx_plane.o
  CC      drivers/gpu/drm/i915/display/i9xx_wm.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_ddi_buf_trans.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display.o
  CC      drivers/gpu/drm/i915/display/intel_alpm.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_device.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_driver.o
  CC      drivers/gpu/drm/i915/display/intel_atomic.o
  CC      drivers/gpu/drm/i915/display/intel_atomic_plane.o
  CC      drivers/gpu/drm/i915/display/intel_audio.o
  CC      drivers/gpu/drm/i915/display/intel_bios.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_irq.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_params.o
  CC      drivers/gpu/drm/i915/display/intel_bw.o
  CC      drivers/gpu/drm/i915/display/intel_cdclk.o
  CC      drivers/gpu/drm/i915/display/intel_color.o
  CC      drivers/gpu/drm/i915/display/intel_combo_phy.o
  CC      drivers/gpu/drm/i915/display/intel_connector.o
  CC      drivers/gpu/drm/i915/display/intel_crtc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_power.o
  CC      drivers/gpu/drm/i915/display/intel_crtc_state_dump.o
  CC      drivers/gpu/drm/i915/display/intel_cursor.o
  CC      drivers/gpu/drm/i915/display/intel_display.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_power_map.o
  CC      drivers/gpu/drm/i915/display/intel_display_driver.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_power_well.o
  CC      drivers/gpu/drm/i915/display/intel_display_irq.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_trace.o
  CC      drivers/gpu/drm/i915/display/intel_display_params.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_wa.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dkl_phy.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dmc.o
  CC      drivers/gpu/drm/i915/display/intel_display_power.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp_aux.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp_aux_backlight.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp_hdcp.o
  CC      drivers/gpu/drm/i915/display/intel_display_power_map.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp_link_training.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dp_mst.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dpll.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dpll_mgr.o
  CC      drivers/gpu/drm/i915/display/intel_display_power_well.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dpt_common.o
  CC      drivers/gpu/drm/i915/display/intel_display_reset.o
  CC      drivers/gpu/drm/i915/display/intel_display_rps.o
  CC      drivers/gpu/drm/i915/display/intel_display_wa.o
  CC      drivers/gpu/drm/i915/display/intel_dmc.o
  CC      drivers/gpu/drm/i915/display/intel_dmc_wl.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_drrs.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dsb.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dsi.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dsi_dcs_backlight.o
  CC      drivers/gpu/drm/i915/display/intel_dpio_phy.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dsi_vbt.o
  CC      drivers/gpu/drm/i915/display/intel_dpll.o
  CC      drivers/gpu/drm/i915/display/intel_dpll_mgr.o
  CC      drivers/gpu/drm/i915/display/intel_dpt.o
  CC      drivers/gpu/drm/i915/display/intel_dpt_common.o
  CC      drivers/gpu/drm/i915/display/intel_drrs.o
  CC      drivers/gpu/drm/i915/display/intel_dsb.o
  CC      drivers/gpu/drm/i915/display/intel_dsb_buffer.o
  CC      drivers/gpu/drm/i915/display/intel_fb.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_encoder.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_fb.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_fbc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_fdi.o
  CC      drivers/gpu/drm/i915/display/intel_fb_bo.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_fifo_underrun.o
  CC      drivers/gpu/drm/i915/display/intel_fb_pin.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_frontbuffer.o
  CC      drivers/gpu/drm/i915/display/intel_fbc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_global_state.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_gmbus.o
  CC      drivers/gpu/drm/i915/display/intel_fdi.o
  CC      drivers/gpu/drm/i915/display/intel_fifo_underrun.o
  CC      drivers/gpu/drm/i915/display/intel_frontbuffer.o
  CC      drivers/gpu/drm/i915/display/intel_global_state.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hdcp.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hdcp_gsc_message.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hdmi.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hotplug.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hotplug_irq.o
  CC      drivers/gpu/drm/i915/display/intel_hdcp.o
  CC      drivers/gpu/drm/i915/display/intel_hdcp_gsc.o
  CC      drivers/gpu/drm/i915/display/intel_hdcp_gsc_message.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_hti.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_link_bw.o
  CC      drivers/gpu/drm/i915/display/intel_hotplug.o
  CC      drivers/gpu/drm/i915/display/intel_hotplug_irq.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_lspcon.o
  CC      drivers/gpu/drm/i915/display/intel_hti.o
  CC      drivers/gpu/drm/i915/display/intel_link_bw.o
  CC      drivers/gpu/drm/i915/display/intel_load_detect.o
  CC      drivers/gpu/drm/i915/display/intel_lpe_audio.o
  CC      drivers/gpu/drm/i915/display/intel_modeset_lock.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_modeset_lock.o
  CC      drivers/gpu/drm/i915/display/intel_modeset_setup.o
  CC      drivers/gpu/drm/i915/display/intel_modeset_verify.o
  CC      drivers/gpu/drm/i915/display/intel_overlay.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_modeset_setup.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_modeset_verify.o
  CC      drivers/gpu/drm/i915/display/intel_pch_display.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_panel.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_pmdemand.o
  CC      drivers/gpu/drm/i915/display/intel_pch_refclk.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_pps.o
  CC      drivers/gpu/drm/i915/display/intel_plane_initial.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_psr.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_qp_tables.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_quirks.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_snps_phy.o
  CC      drivers/gpu/drm/i915/display/intel_pmdemand.o
  CC      drivers/gpu/drm/i915/display/intel_psr.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_tc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_vblank.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_vdsc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_vga.o
  CC      drivers/gpu/drm/i915/display/intel_quirks.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_vrr.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_dmc_wl.o
  CC      drivers/gpu/drm/i915/display/intel_sprite.o
  CC      drivers/gpu/drm/i915/display/intel_sprite_uapi.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_wm.o
  CC [M]  drivers/gpu/drm/xe/i915-display/skl_scaler.o
  CC [M]  drivers/gpu/drm/xe/i915-display/skl_universal_plane.o
  CC      drivers/gpu/drm/i915/display/intel_tc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/skl_watermark.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_acpi.o
  CC      drivers/gpu/drm/i915/display/intel_vblank.o
  CC      drivers/gpu/drm/i915/display/intel_vga.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_opregion.o
  CC      drivers/gpu/drm/i915/display/intel_wm.o
  CC      drivers/gpu/drm/i915/display/skl_scaler.o
  CC [M]  drivers/gpu/drm/xe/xe_debugfs.o
  CC      drivers/gpu/drm/i915/display/skl_universal_plane.o
  CC      drivers/gpu/drm/i915/display/skl_watermark.o
  CC      drivers/gpu/drm/i915/display/intel_acpi.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_debugfs.o
  CC      drivers/gpu/drm/i915/display/intel_opregion.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.o
  CC [M]  drivers/gpu/drm/xe/xe_gt_stats.o
  CC [M]  drivers/gpu/drm/xe/xe_guc_debugfs.o
  CC [M]  drivers/gpu/drm/xe/xe_huc_debugfs.o
  CC [M]  drivers/gpu/drm/xe/xe_uc_debugfs.o
  CC      drivers/gpu/drm/i915/display/intel_display_debugfs.o
  CC      drivers/gpu/drm/i915/display/intel_display_debugfs_params.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_debugfs.o
  CC      drivers/gpu/drm/i915/display/intel_pipe_crc.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_display_debugfs_params.o
  CC [M]  drivers/gpu/drm/xe/i915-display/intel_pipe_crc.o
  CC      drivers/gpu/drm/i915/display/dvo_ch7017.o
  CC      drivers/gpu/drm/i915/display/dvo_ch7xxx.o
  CC      drivers/gpu/drm/i915/display/dvo_ivch.o
  CC      drivers/gpu/drm/i915/display/dvo_ns2501.o
  CC      drivers/gpu/drm/i915/display/dvo_sil164.o
  CC      drivers/gpu/drm/i915/display/dvo_tfp410.o
  CC      drivers/gpu/drm/i915/display/g4x_dp.o
  CC      drivers/gpu/drm/i915/display/g4x_hdmi.o
  CC      drivers/gpu/drm/i915/display/icl_dsi.o
  CC      drivers/gpu/drm/i915/display/intel_backlight.o
  CC      drivers/gpu/drm/i915/display/intel_crt.o
  CC      drivers/gpu/drm/i915/display/intel_cx0_phy.o
  CC      drivers/gpu/drm/i915/display/intel_ddi.o
  CC      drivers/gpu/drm/i915/display/intel_ddi_buf_trans.o
  CC      drivers/gpu/drm/i915/display/intel_display_device.o
  CC      drivers/gpu/drm/i915/display/intel_display_trace.o
  CC      drivers/gpu/drm/i915/display/intel_dkl_phy.o
  CC      drivers/gpu/drm/i915/display/intel_dp.o
  CC      drivers/gpu/drm/i915/display/intel_dp_aux.o
  CC      drivers/gpu/drm/i915/display/intel_dp_aux_backlight.o
  CC      drivers/gpu/drm/i915/display/intel_dp_hdcp.o
  CC      drivers/gpu/drm/i915/display/intel_dp_link_training.o
  CC      drivers/gpu/drm/i915/display/intel_dp_mst.o
  CC      drivers/gpu/drm/i915/display/intel_dsi.o
  CC      drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.o
  CC      drivers/gpu/drm/i915/display/intel_dsi_vbt.o
  CC      drivers/gpu/drm/i915/display/intel_dvo.o
  CC      drivers/gpu/drm/i915/display/intel_encoder.o
  CC      drivers/gpu/drm/i915/display/intel_gmbus.o
  CC      drivers/gpu/drm/i915/display/intel_hdmi.o
  CC      drivers/gpu/drm/i915/display/intel_lspcon.o
  CC      drivers/gpu/drm/i915/display/intel_lvds.o
  CC      drivers/gpu/drm/i915/display/intel_panel.o
  CC      drivers/gpu/drm/i915/display/intel_pps.o
  CC      drivers/gpu/drm/i915/display/intel_qp_tables.o
  CC      drivers/gpu/drm/i915/display/intel_sdvo.o
  CC      drivers/gpu/drm/i915/display/intel_snps_phy.o
  CC      drivers/gpu/drm/i915/display/intel_tv.o
  CC      drivers/gpu/drm/i915/display/intel_vdsc.o
  CC      drivers/gpu/drm/i915/display/intel_vrr.o
  CC      drivers/gpu/drm/i915/display/vlv_dsi.o
  CC      drivers/gpu/drm/i915/display/vlv_dsi_pll.o
  CC      drivers/gpu/drm/i915/i915_perf.o
  CC      drivers/gpu/drm/i915/pxp/intel_pxp.o
  CC      drivers/gpu/drm/i915/pxp/intel_pxp_huc.o
  CC      drivers/gpu/drm/i915/pxp/intel_pxp_tee.o
  CC      drivers/gpu/drm/i915/i915_gpu_error.o
  CC      drivers/gpu/drm/i915/i915_vgpu.o
  LD [M]  drivers/gpu/drm/xe/xe.o
  AR      drivers/gpu/drm/i915/built-in.a
  AR      drivers/gpu/drm/built-in.a
  AR      drivers/gpu/built-in.a
  AR      drivers/built-in.a
  AR      built-in.a
  AR      vmlinux.a
  LD      vmlinux.o
  OBJCOPY modules.builtin.modinfo
  GEN     modules.builtin
  MODPOST Module.symvers
  CC      .vmlinux.export.o
  CC [M]  fs/efivarfs/efivarfs.mod.o
  CC [M]  drivers/gpu/drm/drm_exec.mod.o
  CC [M]  drivers/gpu/drm/drm_gpuvm.mod.o
  CC [M]  drivers/gpu/drm/drm_suballoc_helper.mod.o
  CC [M]  drivers/gpu/drm/drm_ttm_helper.mod.o
  CC [M]  drivers/gpu/drm/scheduler/gpu-sched.mod.o
  CC [M]  drivers/gpu/drm/xe/xe.mod.o
  CC [M]  drivers/thermal/intel/x86_pkg_temp_thermal.mod.o
  CC [M]  sound/core/snd-hwdep.mod.o
  CC [M]  sound/core/snd-pcm.mod.o
  CC [M]  sound/pci/hda/snd-hda-codec.mod.o
  CC [M]  sound/pci/hda/snd-hda-codec-hdmi.mod.o
  CC [M]  sound/pci/hda/snd-hda-intel.mod.o
  CC [M]  sound/hda/snd-hda-core.mod.o
  CC [M]  sound/hda/snd-intel-dspcfg.mod.o
  CC [M]  sound/hda/snd-intel-sdw-acpi.mod.o
  CC [M]  net/netfilter/nf_log_syslog.mod.o
  CC [M]  net/netfilter/xt_mark.mod.o
  CC [M]  net/netfilter/xt_nat.mod.o
  CC [M]  net/netfilter/xt_LOG.mod.o
  CC [M]  net/netfilter/xt_MASQUERADE.mod.o
  CC [M]  net/netfilter/xt_addrtype.mod.o
  CC [M]  net/ipv4/netfilter/iptable_nat.mod.o
  LD [M]  net/netfilter/nf_log_syslog.ko
  LD [M]  fs/efivarfs/efivarfs.ko
  LD [M]  drivers/thermal/intel/x86_pkg_temp_thermal.ko
  LD [M]  sound/core/snd-pcm.ko
  LD [M]  drivers/gpu/drm/drm_gpuvm.ko
  LD [M]  sound/pci/hda/snd-hda-codec.ko
  LD [M]  sound/hda/snd-intel-dspcfg.ko
  LD [M]  net/netfilter/xt_mark.ko
  LD [M]  net/netfilter/xt_LOG.ko
  LD [M]  sound/pci/hda/snd-hda-intel.ko
  LD [M]  net/netfilter/xt_addrtype.ko
  LD [M]  net/netfilter/xt_nat.ko
  LD [M]  net/ipv4/netfilter/iptable_nat.ko
  LD [M]  drivers/gpu/drm/drm_ttm_helper.ko
  LD [M]  sound/pci/hda/snd-hda-codec-hdmi.ko
  LD [M]  drivers/gpu/drm/scheduler/gpu-sched.ko
  LD [M]  drivers/gpu/drm/xe/xe.ko
  LD [M]  sound/core/snd-hwdep.ko
  LD [M]  sound/hda/snd-intel-sdw-acpi.ko
  LD [M]  drivers/gpu/drm/drm_suballoc_helper.ko
  LD [M]  drivers/gpu/drm/drm_exec.ko
  LD [M]  net/netfilter/xt_MASQUERADE.ko
  LD [M]  sound/hda/snd-hda-core.ko
  UPD     include/generated/utsversion.h
  CC      init/version-timestamp.o
  KSYMS   .tmp_vmlinux0.kallsyms.S
  AS      .tmp_vmlinux0.kallsyms.o
  LD      .tmp_vmlinux1
  NM      .tmp_vmlinux1.syms
  KSYMS   .tmp_vmlinux1.kallsyms.S
  AS      .tmp_vmlinux1.kallsyms.o
  LD      .tmp_vmlinux2
  NM      .tmp_vmlinux2.syms
  KSYMS   .tmp_vmlinux2.kallsyms.S
  AS      .tmp_vmlinux2.kallsyms.o
  LD      vmlinux
  NM      System.map
  SORTTAB vmlinux
  RELOCS  arch/x86/boot/compressed/vmlinux.relocs
  RSTRIP  vmlinux
  CC      arch/x86/boot/a20.o
  AS      arch/x86/boot/bioscall.o
  CC      arch/x86/boot/cmdline.o
  AS      arch/x86/boot/copy.o
  HOSTCC  arch/x86/boot/mkcpustr
  CC      arch/x86/boot/cpuflags.o
  CC      arch/x86/boot/cpucheck.o
  CC      arch/x86/boot/early_serial_console.o
  CC      arch/x86/boot/edd.o
  CC      arch/x86/boot/main.o
  CC      arch/x86/boot/memory.o
  CC      arch/x86/boot/pm.o
  AS      arch/x86/boot/pmjump.o
  CC      arch/x86/boot/printf.o
  CC      arch/x86/boot/regs.o
  CC      arch/x86/boot/string.o
  CC      arch/x86/boot/tty.o
  CC      arch/x86/boot/video.o
  CC      arch/x86/boot/video-mode.o
  CC      arch/x86/boot/version.o
  CC      arch/x86/boot/video-vga.o
  CC      arch/x86/boot/video-vesa.o
  CC      arch/x86/boot/video-bios.o
  HOSTCC  arch/x86/boot/tools/build
  CPUSTR  arch/x86/boot/cpustr.h
  CC      arch/x86/boot/cpu.o
  LDS     arch/x86/boot/compressed/vmlinux.lds
  AS      arch/x86/boot/compressed/kernel_info.o
  AS      arch/x86/boot/compressed/head_32.o
  VOFFSET arch/x86/boot/compressed/../voffset.h
  CC      arch/x86/boot/compressed/string.o
  CC      arch/x86/boot/compressed/cmdline.o
  CC      arch/x86/boot/compressed/error.o
  OBJCOPY arch/x86/boot/compressed/vmlinux.bin
  HOSTCC  arch/x86/boot/compressed/mkpiggy
  CC      arch/x86/boot/compressed/cpuflags.o
  CC      arch/x86/boot/compressed/early_serial_console.o
  CC      arch/x86/boot/compressed/kaslr.o
  CC      arch/x86/boot/compressed/acpi.o
  CC      arch/x86/boot/compressed/efi.o
  GZIP    arch/x86/boot/compressed/vmlinux.bin.gz
  CC      arch/x86/boot/compressed/misc.o
  MKPIGGY arch/x86/boot/compressed/piggy.S
  AS      arch/x86/boot/compressed/piggy.o
  LD      arch/x86/boot/compressed/vmlinux
  ZOFFSET arch/x86/boot/zoffset.h
  OBJCOPY arch/x86/boot/vmlinux.bin
  AS      arch/x86/boot/header.o
  LD      arch/x86/boot/setup.elf
  OBJCOPY arch/x86/boot/setup.bin
  BUILD   arch/x86/boot/bzImage
Kernel: arch/x86/boot/bzImage is ready  (#1)
run-parts: executing /workspace/ci/hooks/20-kernel-doc
+ SRC_DIR=/workspace/kernel
+ cd /workspace/kernel
+ find drivers/gpu/drm/xe/ -name '*.[ch]' -not -path 'drivers/gpu/drm/xe/display/*'
+ xargs ./scripts/kernel-doc -Werror -none include/uapi/drm/xe_drm.h
All hooks done

✓ CI.checksparse: success for Fix device private page livelock on CPU fault

[Patchwork]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : success

== Summary ==

+ trap cleanup EXIT
+ KERNEL=/kernel
+ MT=/root/linux/maintainer-tools
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools /root/linux/maintainer-tools
Cloning into '/root/linux/maintainer-tools'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ make -C /root/linux/maintainer-tools
make: Entering directory '/root/linux/maintainer-tools'
cc -O2 -g -Wextra -o remap-log remap-log.c
make: Leaving directory '/root/linux/maintainer-tools'
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ /root/linux/maintainer-tools/dim sparse --fast 43852924969a3fbe93796d71bece8e5539aa4845
Sparse version: 0.6.1 (Ubuntu: 0.6.1-2build1)
Fast mode used, each commit won't be checked separately.
Okay!

+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Alistair Popple]

Matthew Brost <matthew.brost@intel.com> writes:

> Avoid multiple CPU page faults to the same device page racing by locking
> the page in do_swap_page before taking an additional reference to the
> page. This prevents scenarios where multiple CPU page faults each take
> an extra reference to a device page, which could abort migration in
> folio_migrate_mapping. With the device page locked in do_swap_page, the
> migrate_vma_* functions need to be updated to avoid locking the
> fault_page argument.

I added the get_page() and therefore the fault_page argument to deal
with another lifetime issue. Neither Felix nor I particularly liked the
solution though (see
https://lore.kernel.org/all/cover.60659b549d8509ddecafad4f498ee7f03bb23c69.1664366292.git-series.apopple@nvidia.com/T/#m715589d57716448386ff9af41da27a952d94615a)
and this seems to make it even uglier, so I have suggested an
alternative solution below.

> Prior to this change, a livelock scenario could occur in Xe's (Intel GPU
> DRM driver) SVM implementation if enough threads faulted the same device
> page.

I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
but theoretically it seems like it should be possible. However we
serialize migrations of the same virtual address range to avoid these
kind of issues as they can happen the other way too (ie. multiple
threads trying to migrate to GPU).

So I suspect what happens in UVM is that one thread wins and installs
the migration entry while the others fail to get the driver migration
lock and bail out sufficiently early in the fault path to avoid the
live-lock.

> Cc: Philip Yang <Philip.Yang@amd.com>
> Cc: Felix Kuehling <felix.kuehling@amd.com>
> Cc: Christian König <christian.koenig@amd.com>
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> ---
>  mm/memory.c         | 13 +++++++---
>  mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>  2 files changed, 50 insertions(+), 23 deletions(-)
>
> diff --git a/mm/memory.c b/mm/memory.c
> index 3c01d68065be..bbd97d16a96a 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>  			 * Get a page reference while we know the page can't be
>  			 * freed.
>  			 */
> -			get_page(vmf->page);
> -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> -			put_page(vmf->page);
> +			if (trylock_page(vmf->page)) {
> +				get_page(vmf->page);
> +				pte_unmap_unlock(vmf->pte, vmf->ptl);

This is all beginning to look a lot like migrate_vma_collect_pmd(). So
rather than do this and then have to pass all this context
(ie. fault_page) down to the migrate_vma_* functions could we instead
just do what migrate_vma_collect_pmd() does here? Ie. we already have
the PTL and the page lock so there's no reason we couldn't just setup
the migration entry prior to calling migrate_to_ram().

Obviously calling migrate_vma_setup() would show the page as not
migrating, but drivers could easily just fill in the src_pfn info after
calling migrate_vma_setup().

This would eliminate the whole fault_page ugliness.

> +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> +				put_page(vmf->page);
> +				unlock_page(vmf->page);
> +			} else {
> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> +			}
>  		} else if (is_hwpoison_entry(entry)) {
>  			ret = VM_FAULT_HWPOISON;
>  		} else if (is_pte_marker_entry(entry)) {
> diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> index 6d66dc1c6ffa..049893a5a179 100644
> --- a/mm/migrate_device.c
> +++ b/mm/migrate_device.c
> @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>  				   struct mm_walk *walk)
>  {
>  	struct migrate_vma *migrate = walk->private;
> +	struct folio *fault_folio = migrate->fault_page ?
> +		page_folio(migrate->fault_page) : NULL;
>  	struct vm_area_struct *vma = walk->vma;
>  	struct mm_struct *mm = vma->vm_mm;
>  	unsigned long addr = start, unmapped = 0;
> @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>  
>  			folio_get(folio);
>  			spin_unlock(ptl);
> -			if (unlikely(!folio_trylock(folio)))
> +			if (unlikely(fault_folio != folio &&
> +				     !folio_trylock(folio)))
>  				return migrate_vma_collect_skip(start, end,
>  								walk);
>  			ret = split_folio(folio);
> -			folio_unlock(folio);
> +			if (fault_folio != folio)
> +				folio_unlock(folio);
>  			folio_put(folio);
>  			if (ret)
>  				return migrate_vma_collect_skip(start, end,
> @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>  		 * optimisation to avoid walking the rmap later with
>  		 * try_to_migrate().
>  		 */
> -		if (folio_trylock(folio)) {
> +		if (fault_folio == folio || folio_trylock(folio)) {
>  			bool anon_exclusive;
>  			pte_t swp_pte;
>  
> @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>  
>  				if (folio_try_share_anon_rmap_pte(folio, page)) {
>  					set_pte_at(mm, addr, ptep, pte);
> -					folio_unlock(folio);
> +					if (fault_folio != folio)
> +						folio_unlock(folio);
>  					folio_put(folio);
>  					mpfn = 0;
>  					goto next;
> @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>  					  unsigned long npages,
>  					  struct page *fault_page)
>  {
> +	struct folio *fault_folio = fault_page ?
> +		page_folio(fault_page) : NULL;
>  	unsigned long i, restore = 0;
>  	bool allow_drain = true;
>  	unsigned long unmapped = 0;
> @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>  		remove_migration_ptes(folio, folio, false);
>  
>  		src_pfns[i] = 0;
> -		folio_unlock(folio);
> +		if (fault_folio != folio)
> +			folio_unlock(folio);
>  		folio_put(folio);
>  		restore--;
>  	}
> @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>  		return -EINVAL;
>  	if (args->fault_page && !is_device_private_page(args->fault_page))
>  		return -EINVAL;
> +	if (args->fault_page && !PageLocked(args->fault_page))
> +		return -EINVAL;
>  
>  	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>  	args->cpages = 0;
> @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>  }
>  EXPORT_SYMBOL(migrate_vma_pages);
>  
> -/*
> - * migrate_device_finalize() - complete page migration
> - * @src_pfns: src_pfns returned from migrate_device_range()
> - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> - * @npages: number of pages in the range
> - *
> - * Completes migration of the page by removing special migration entries.
> - * Drivers must ensure copying of page data is complete and visible to the CPU
> - * before calling this.
> - */
> -void migrate_device_finalize(unsigned long *src_pfns,
> -			unsigned long *dst_pfns, unsigned long npages)
> +static void __migrate_device_finalize(unsigned long *src_pfns,
> +				      unsigned long *dst_pfns,
> +				      unsigned long npages,
> +				      struct page *fault_page)
>  {
> +	struct folio *fault_folio = fault_page ?
> +		page_folio(fault_page) : NULL;
>  	unsigned long i;
>  
>  	for (i = 0; i < npages; i++) {
> @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>  		src = page_folio(page);
>  		dst = page_folio(newpage);
>  		remove_migration_ptes(src, dst, false);
> -		folio_unlock(src);
> +		if (fault_folio != src)
> +			folio_unlock(src);
>  
>  		if (is_zone_device_page(page))
>  			put_page(page);
> @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>  		}
>  	}
>  }
> +
> +/*
> + * migrate_device_finalize() - complete page migration
> + * @src_pfns: src_pfns returned from migrate_device_range()
> + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> + * @npages: number of pages in the range
> + *
> + * Completes migration of the page by removing special migration entries.
> + * Drivers must ensure copying of page data is complete and visible to the CPU
> + * before calling this.
> + */
> +void migrate_device_finalize(unsigned long *src_pfns,
> +			unsigned long *dst_pfns, unsigned long npages)
> +{
> +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> +}
>  EXPORT_SYMBOL(migrate_device_finalize);
>  
>  /**
> @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>   */
>  void migrate_vma_finalize(struct migrate_vma *migrate)
>  {
> -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> +				  migrate->fault_page);
>  }
>  EXPORT_SYMBOL(migrate_vma_finalize);

✗ CI.BAT: failure for Fix device private page livelock on CPU fault

[Patchwork]

[-- Attachment #1: Type: text/plain, Size: 1389 bytes --]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : failure

== Summary ==

CI Bug Log - changes from xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3_BAT -> xe-pw-138497v1_BAT
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with xe-pw-138497v1_BAT absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in xe-pw-138497v1_BAT, please notify your bug team (I915-ci-infra@lists.freedesktop.org) to allow them
  to document this new failure mode, which will reduce false positives in CI.

  

Participating hosts (8 -> 0)
------------------------------

  ERROR: It appears as if the changes made in xe-pw-138497v1_BAT prevented too many machines from booting.

  Missing    (8): bat-bmg-1 bat-lnl-2 bat-adlp-vf bat-lnl-1 bat-dg2-oem2 bat-atsm-2 bat-bmg-2 bat-adlp-7 


Changes
-------

  No changes found


Build changes
-------------

  * Linux: xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3 -> xe-pw-138497v1

  IGT_8012: 8012
  xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3: f3eded4f8a05d73a0b94f27e05737ea3427450b3
  xe-pw-138497v1: 138497v1

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/index.html

[-- Attachment #2: Type: text/html, Size: 1961 bytes --]

✗ CI.FULL: failure for Fix device private page livelock on CPU fault

[Patchwork]

[-- Attachment #1: Type: text/plain, Size: 82031 bytes --]

== Series Details ==

Series: Fix device private page livelock on CPU fault
URL   : https://patchwork.freedesktop.org/series/138497/
State : failure

== Summary ==

CI Bug Log - changes from xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3_full -> xe-pw-138497v1_full
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with xe-pw-138497v1_full absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in xe-pw-138497v1_full, please notify your bug team (I915-ci-infra@lists.freedesktop.org) to allow them
  to document this new failure mode, which will reduce false positives in CI.

  

Participating hosts (4 -> 4)
------------------------------

  No changes in participating hosts

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in xe-pw-138497v1_full:

### IGT changes ###

#### Possible regressions ####

  * igt@kms_psr2_sf@fbc-overlay-primary-update-sf-dmg-area:
    - shard-lnl:          NOTRUN -> [FAIL][1] +1 other test fail
   [1]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_psr2_sf@fbc-overlay-primary-update-sf-dmg-area.html

  * igt@xe_wedged@wedged-at-any-timeout:
    - shard-dg2-set2:     [PASS][2] -> [FAIL][3]
   [2]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-435/igt@xe_wedged@wedged-at-any-timeout.html
   [3]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@xe_wedged@wedged-at-any-timeout.html
    - shard-lnl:          [PASS][4] -> [FAIL][5]
   [4]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-5/igt@xe_wedged@wedged-at-any-timeout.html
   [5]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-8/igt@xe_wedged@wedged-at-any-timeout.html

  
New tests
---------

  New tests have been introduced between xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3_full and xe-pw-138497v1_full:

### New IGT tests (12) ###

  * igt@kms_atomic_transition@plane-all-modeset-transition@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [19.06] s

  * igt@kms_atomic_transition@plane-all-modeset-transition@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [19.05] s

  * igt@kms_atomic_transition@plane-all-transition-fencing@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.75] s

  * igt@kms_atomic_transition@plane-all-transition-fencing@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.75] s

  * igt@kms_atomic_transition@plane-all-transition-nonblocking-fencing@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.74] s

  * igt@kms_atomic_transition@plane-all-transition-nonblocking-fencing@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.74] s

  * igt@kms_atomic_transition@plane-all-transition-nonblocking@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.75] s

  * igt@kms_atomic_transition@plane-all-transition-nonblocking@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [3.75] s

  * igt@kms_atomic_transition@plane-use-after-nonblocking-unbind-fencing@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [0.10] s

  * igt@kms_atomic_transition@plane-use-after-nonblocking-unbind-fencing@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [0.10] s

  * igt@kms_atomic_transition@plane-use-after-nonblocking-unbind@pipe-a-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [0.10] s

  * igt@kms_atomic_transition@plane-use-after-nonblocking-unbind@pipe-b-hdmi-a-3:
    - Statuses : 1 pass(s)
    - Exec time: [0.10] s

  

Known issues
------------

  Here are the changes found in xe-pw-138497v1_full that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@kms_async_flips@async-flip-with-page-flip-events@pipe-c-hdmi-a-1-x:
    - shard-adlp:         [PASS][6] -> [DMESG-WARN][7] ([Intel XE#1033])
   [6]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-2/igt@kms_async_flips@async-flip-with-page-flip-events@pipe-c-hdmi-a-1-x.html
   [7]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-6/igt@kms_async_flips@async-flip-with-page-flip-events@pipe-c-hdmi-a-1-x.html

  * igt@kms_atomic_transition@plane-toggle-modeset-transition@pipe-a-edp-1:
    - shard-lnl:          [PASS][8] -> [FAIL][9] ([Intel XE#1426]) +1 other test fail
   [8]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-3/igt@kms_atomic_transition@plane-toggle-modeset-transition@pipe-a-edp-1.html
   [9]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-6/igt@kms_atomic_transition@plane-toggle-modeset-transition@pipe-a-edp-1.html

  * igt@kms_big_fb@4-tiled-16bpp-rotate-90:
    - shard-dg2-set2:     NOTRUN -> [SKIP][10] ([Intel XE#1201] / [Intel XE#316])
   [10]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_big_fb@4-tiled-16bpp-rotate-90.html

  * igt@kms_big_fb@4-tiled-max-hw-stride-32bpp-rotate-0:
    - shard-adlp:         NOTRUN -> [SKIP][11] ([Intel XE#1124] / [Intel XE#1201]) +1 other test skip
   [11]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_big_fb@4-tiled-max-hw-stride-32bpp-rotate-0.html

  * igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-180-hflip:
    - shard-lnl:          [PASS][12] -> [FAIL][13] ([Intel XE#1659]) +1 other test fail
   [12]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-6/igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-180-hflip.html
   [13]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-7/igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-180-hflip.html

  * igt@kms_big_fb@y-tiled-16bpp-rotate-270:
    - shard-dg2-set2:     NOTRUN -> [SKIP][14] ([Intel XE#1124] / [Intel XE#1201])
   [14]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_big_fb@y-tiled-16bpp-rotate-270.html

  * igt@kms_big_fb@y-tiled-addfb:
    - shard-lnl:          NOTRUN -> [SKIP][15] ([Intel XE#1467])
   [15]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_big_fb@y-tiled-addfb.html

  * igt@kms_big_fb@yf-tiled-8bpp-rotate-270:
    - shard-lnl:          NOTRUN -> [SKIP][16] ([Intel XE#1124]) +2 other tests skip
   [16]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_big_fb@yf-tiled-8bpp-rotate-270.html

  * igt@kms_big_fb@yf-tiled-addfb:
    - shard-dg2-set2:     NOTRUN -> [SKIP][17] ([Intel XE#1201] / [Intel XE#619])
   [17]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_big_fb@yf-tiled-addfb.html

  * igt@kms_bw@connected-linear-tiling-2-displays-1920x1080p:
    - shard-adlp:         NOTRUN -> [SKIP][18] ([Intel XE#1201] / [Intel XE#2191])
   [18]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_bw@connected-linear-tiling-2-displays-1920x1080p.html

  * igt@kms_bw@connected-linear-tiling-4-displays-3840x2160p:
    - shard-dg2-set2:     NOTRUN -> [SKIP][19] ([Intel XE#1201] / [Intel XE#2191])
   [19]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_bw@connected-linear-tiling-4-displays-3840x2160p.html

  * igt@kms_bw@linear-tiling-4-displays-2560x1440p:
    - shard-lnl:          NOTRUN -> [SKIP][20] ([Intel XE#1512])
   [20]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_bw@linear-tiling-4-displays-2560x1440p.html

  * igt@kms_ccs@bad-rotation-90-4-tiled-dg2-rc-ccs:
    - shard-lnl:          NOTRUN -> [SKIP][21] ([Intel XE#1399]) +2 other tests skip
   [21]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_ccs@bad-rotation-90-4-tiled-dg2-rc-ccs.html

  * igt@kms_ccs@bad-rotation-90-4-tiled-dg2-rc-ccs-cc@pipe-b-hdmi-a-1:
    - shard-adlp:         NOTRUN -> [SKIP][22] ([Intel XE#1201] / [Intel XE#787]) +11 other tests skip
   [22]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_ccs@bad-rotation-90-4-tiled-dg2-rc-ccs-cc@pipe-b-hdmi-a-1.html

  * igt@kms_ccs@crc-primary-rotation-180-4-tiled-mtl-rc-ccs:
    - shard-dg2-set2:     NOTRUN -> [SKIP][23] ([Intel XE#1201] / [Intel XE#455] / [Intel XE#787]) +1 other test skip
   [23]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_ccs@crc-primary-rotation-180-4-tiled-mtl-rc-ccs.html

  * igt@kms_ccs@crc-primary-rotation-180-4-tiled-mtl-rc-ccs@pipe-a-dp-4:
    - shard-dg2-set2:     NOTRUN -> [SKIP][24] ([Intel XE#1201] / [Intel XE#787]) +6 other tests skip
   [24]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_ccs@crc-primary-rotation-180-4-tiled-mtl-rc-ccs@pipe-a-dp-4.html

  * igt@kms_ccs@crc-sprite-planes-basic-yf-tiled-ccs:
    - shard-adlp:         NOTRUN -> [SKIP][25] ([Intel XE#1201] / [Intel XE#455] / [Intel XE#787]) +7 other tests skip
   [25]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_ccs@crc-sprite-planes-basic-yf-tiled-ccs.html

  * igt@kms_ccs@random-ccs-data-4-tiled-lnl-ccs:
    - shard-adlp:         NOTRUN -> [SKIP][26] ([Intel XE#1201] / [Intel XE#1252])
   [26]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@kms_ccs@random-ccs-data-4-tiled-lnl-ccs.html

  * igt@kms_cdclk@mode-transition-all-outputs:
    - shard-dg2-set2:     NOTRUN -> [SKIP][27] ([Intel XE#1201] / [Intel XE#314])
   [27]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_cdclk@mode-transition-all-outputs.html

  * igt@kms_chamelium_audio@dp-audio-edid:
    - shard-lnl:          NOTRUN -> [SKIP][28] ([Intel XE#373]) +1 other test skip
   [28]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_chamelium_audio@dp-audio-edid.html

  * igt@kms_chamelium_color@degamma:
    - shard-dg2-set2:     NOTRUN -> [SKIP][29] ([Intel XE#1201] / [Intel XE#306])
   [29]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_chamelium_color@degamma.html

  * igt@kms_chamelium_hpd@dp-hpd-fast:
    - shard-dg2-set2:     NOTRUN -> [SKIP][30] ([Intel XE#1201] / [Intel XE#373])
   [30]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_chamelium_hpd@dp-hpd-fast.html

  * igt@kms_cursor_crc@cursor-onscreen-512x512:
    - shard-adlp:         NOTRUN -> [SKIP][31] ([Intel XE#1201] / [Intel XE#308])
   [31]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_cursor_crc@cursor-onscreen-512x512.html

  * igt@kms_cursor_crc@cursor-rapid-movement-256x85:
    - shard-lnl:          NOTRUN -> [SKIP][32] ([Intel XE#1424]) +1 other test skip
   [32]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_cursor_crc@cursor-rapid-movement-256x85.html

  * igt@kms_cursor_legacy@cursorb-vs-flipa-atomic-transitions:
    - shard-lnl:          NOTRUN -> [SKIP][33] ([Intel XE#309]) +2 other tests skip
   [33]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_cursor_legacy@cursorb-vs-flipa-atomic-transitions.html

  * igt@kms_cursor_legacy@cursorb-vs-flipb-toggle:
    - shard-adlp:         NOTRUN -> [SKIP][34] ([Intel XE#1201] / [Intel XE#309])
   [34]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_cursor_legacy@cursorb-vs-flipb-toggle.html

  * igt@kms_flip@2x-busy-flip:
    - shard-adlp:         NOTRUN -> [SKIP][35] ([Intel XE#1201] / [Intel XE#310])
   [35]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_flip@2x-busy-flip.html

  * igt@kms_flip@2x-plain-flip-ts-check:
    - shard-lnl:          NOTRUN -> [SKIP][36] ([Intel XE#1421]) +3 other tests skip
   [36]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_flip@2x-plain-flip-ts-check.html

  * igt@kms_flip@basic-flip-vs-wf_vblank:
    - shard-lnl:          [PASS][37] -> [FAIL][38] ([Intel XE#886]) +1 other test fail
   [37]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-4/igt@kms_flip@basic-flip-vs-wf_vblank.html
   [38]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-8/igt@kms_flip@basic-flip-vs-wf_vblank.html

  * igt@kms_flip_scaled_crc@flip-32bpp-yftileccs-to-64bpp-yftile-downscaling@pipe-a-default-mode:
    - shard-lnl:          NOTRUN -> [SKIP][39] ([Intel XE#1401]) +3 other tests skip
   [39]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_flip_scaled_crc@flip-32bpp-yftileccs-to-64bpp-yftile-downscaling@pipe-a-default-mode.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-64bpp-ytile-downscaling:
    - shard-lnl:          NOTRUN -> [SKIP][40] ([Intel XE#1401] / [Intel XE#1745]) +3 other tests skip
   [40]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-64bpp-ytile-downscaling.html

  * igt@kms_flip_scaled_crc@flip-64bpp-4tile-to-32bpp-4tile-downscaling:
    - shard-adlp:         NOTRUN -> [SKIP][41] ([Intel XE#1201] / [Intel XE#455]) +5 other tests skip
   [41]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_flip_scaled_crc@flip-64bpp-4tile-to-32bpp-4tile-downscaling.html

  * igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-y:
    - shard-adlp:         [PASS][42] -> [FAIL][43] ([Intel XE#1874])
   [42]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-2/igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-y.html
   [43]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-6/igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-y.html

  * igt@kms_frontbuffer_tracking@drrs-indfb-scaledprimary:
    - shard-dg2-set2:     NOTRUN -> [SKIP][44] ([Intel XE#1201] / [Intel XE#651]) +1 other test skip
   [44]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_frontbuffer_tracking@drrs-indfb-scaledprimary.html

  * igt@kms_frontbuffer_tracking@fbcdrrs-1p-pri-indfb-multidraw:
    - shard-lnl:          NOTRUN -> [SKIP][45] ([Intel XE#651]) +4 other tests skip
   [45]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_frontbuffer_tracking@fbcdrrs-1p-pri-indfb-multidraw.html

  * igt@kms_frontbuffer_tracking@fbcdrrs-1p-primscrn-spr-indfb-draw-blt:
    - shard-adlp:         NOTRUN -> [SKIP][46] ([Intel XE#1201] / [Intel XE#651]) +3 other tests skip
   [46]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_frontbuffer_tracking@fbcdrrs-1p-primscrn-spr-indfb-draw-blt.html

  * igt@kms_frontbuffer_tracking@fbcdrrs-2p-primscrn-pri-shrfb-draw-blt:
    - shard-lnl:          NOTRUN -> [SKIP][47] ([Intel XE#656]) +8 other tests skip
   [47]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_frontbuffer_tracking@fbcdrrs-2p-primscrn-pri-shrfb-draw-blt.html

  * igt@kms_frontbuffer_tracking@fbcdrrs-2p-scndscrn-spr-indfb-draw-render:
    - shard-adlp:         NOTRUN -> [SKIP][48] ([Intel XE#1201] / [Intel XE#656])
   [48]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@kms_frontbuffer_tracking@fbcdrrs-2p-scndscrn-spr-indfb-draw-render.html

  * igt@kms_frontbuffer_tracking@fbcpsr-1p-primscrn-spr-indfb-draw-blt:
    - shard-lnl:          NOTRUN -> [CRASH][49] ([Intel XE#2749]) +4 other tests crash
   [49]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_frontbuffer_tracking@fbcpsr-1p-primscrn-spr-indfb-draw-blt.html

  * igt@kms_frontbuffer_tracking@fbcpsr-rgb101010-draw-mmap-wc:
    - shard-adlp:         NOTRUN -> [SKIP][50] ([Intel XE#1201] / [Intel XE#653]) +1 other test skip
   [50]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_frontbuffer_tracking@fbcpsr-rgb101010-draw-mmap-wc.html

  * igt@kms_frontbuffer_tracking@psr-2p-scndscrn-indfb-msflip-blt:
    - shard-dg2-set2:     NOTRUN -> [SKIP][51] ([Intel XE#1201] / [Intel XE#653]) +1 other test skip
   [51]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_frontbuffer_tracking@psr-2p-scndscrn-indfb-msflip-blt.html

  * igt@kms_hdr@static-toggle-suspend:
    - shard-lnl:          NOTRUN -> [SKIP][52] ([Intel XE#1503] / [Intel XE#599])
   [52]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_hdr@static-toggle-suspend.html

  * igt@kms_plane@pixel-format:
    - shard-adlp:         NOTRUN -> [INCOMPLETE][53] ([Intel XE#1035] / [Intel XE#1195])
   [53]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_plane@pixel-format.html

  * igt@kms_plane@pixel-format@pipe-a-plane-3:
    - shard-adlp:         NOTRUN -> [WARN][54] ([Intel XE#2078]) +1 other test warn
   [54]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_plane@pixel-format@pipe-a-plane-3.html

  * igt@kms_plane@plane-position-hole-dpms@pipe-a-plane-3:
    - shard-lnl:          [PASS][55] -> [DMESG-WARN][56] ([Intel XE#324]) +2 other tests dmesg-warn
   [55]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-1/igt@kms_plane@plane-position-hole-dpms@pipe-a-plane-3.html
   [56]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-6/igt@kms_plane@plane-position-hole-dpms@pipe-a-plane-3.html

  * igt@kms_plane_scaling@intel-max-src-size:
    - shard-lnl:          NOTRUN -> [SKIP][57] ([Intel XE#599]) +1 other test skip
   [57]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_plane_scaling@intel-max-src-size.html

  * igt@kms_plane_scaling@intel-max-src-size@pipe-a-hdmi-a-6:
    - shard-dg2-set2:     [PASS][58] -> [FAIL][59] ([Intel XE#361])
   [58]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-436/igt@kms_plane_scaling@intel-max-src-size@pipe-a-hdmi-a-6.html
   [59]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_plane_scaling@intel-max-src-size@pipe-a-hdmi-a-6.html

  * igt@kms_plane_scaling@planes-unity-scaling-downscale-factor-0-5@pipe-b-edp-1:
    - shard-lnl:          NOTRUN -> [SKIP][60] ([Intel XE#2318]) +3 other tests skip
   [60]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_plane_scaling@planes-unity-scaling-downscale-factor-0-5@pipe-b-edp-1.html

  * igt@kms_pm_rpm@dpms-mode-unset-non-lpsp:
    - shard-lnl:          NOTRUN -> [SKIP][61] ([Intel XE#1439])
   [61]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_pm_rpm@dpms-mode-unset-non-lpsp.html

  * igt@kms_properties@connector-properties-legacy:
    - shard-dg2-set2:     NOTRUN -> [INCOMPLETE][62] ([Intel XE#1195]) +1 other test incomplete
   [62]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_properties@connector-properties-legacy.html

  * igt@kms_psr2_sf@cursor-plane-move-continuous-exceed-sf:
    - shard-dg2-set2:     NOTRUN -> [SKIP][63] ([Intel XE#1201] / [Intel XE#1489]) +1 other test skip
   [63]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_psr2_sf@cursor-plane-move-continuous-exceed-sf.html

  * igt@kms_psr2_su@page_flip-p010:
    - shard-lnl:          NOTRUN -> [SKIP][64] ([Intel XE#1128])
   [64]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_psr2_su@page_flip-p010.html

  * igt@kms_psr@fbc-psr-cursor-blt@edp-1:
    - shard-lnl:          NOTRUN -> [CRASH][65] ([Intel XE#2748]) +1 other test crash
   [65]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_psr@fbc-psr-cursor-blt@edp-1.html

  * igt@kms_psr@fbc-psr-sprite-plane-onoff:
    - shard-lnl:          NOTRUN -> [WARN][66] ([Intel XE#2748]) +1 other test warn
   [66]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_psr@fbc-psr-sprite-plane-onoff.html

  * igt@kms_psr@fbc-psr2-cursor-blt:
    - shard-adlp:         NOTRUN -> [SKIP][67] ([Intel XE#1201] / [Intel XE#929]) +4 other tests skip
   [67]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_psr@fbc-psr2-cursor-blt.html

  * igt@kms_psr@pr-no-drrs:
    - shard-lnl:          NOTRUN -> [SKIP][68] ([Intel XE#1406])
   [68]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_psr@pr-no-drrs.html

  * igt@kms_psr@psr-dpms:
    - shard-dg2-set2:     NOTRUN -> [SKIP][69] ([Intel XE#1201] / [Intel XE#929]) +2 other tests skip
   [69]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_psr@psr-dpms.html

  * igt@kms_rotation_crc@bad-pixel-format:
    - shard-adlp:         NOTRUN -> [SKIP][70] ([Intel XE#1201] / [Intel XE#327])
   [70]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_rotation_crc@bad-pixel-format.html

  * igt@kms_rotation_crc@sprite-rotation-90-pos-100-0:
    - shard-lnl:          NOTRUN -> [SKIP][71] ([Intel XE#1437])
   [71]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@kms_rotation_crc@sprite-rotation-90-pos-100-0.html

  * igt@kms_tiled_display@basic-test-pattern:
    - shard-lnl:          NOTRUN -> [SKIP][72] ([Intel XE#362])
   [72]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_tiled_display@basic-test-pattern.html

  * igt@kms_universal_plane@cursor-fb-leak:
    - shard-adlp:         [PASS][73] -> [FAIL][74] ([Intel XE#771] / [Intel XE#899])
   [73]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-1/igt@kms_universal_plane@cursor-fb-leak.html
   [74]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@kms_universal_plane@cursor-fb-leak.html

  * igt@kms_universal_plane@cursor-fb-leak@pipe-d-hdmi-a-1:
    - shard-adlp:         [PASS][75] -> [FAIL][76] ([Intel XE#899]) +1 other test fail
   [75]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-1/igt@kms_universal_plane@cursor-fb-leak@pipe-d-hdmi-a-1.html
   [76]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@kms_universal_plane@cursor-fb-leak@pipe-d-hdmi-a-1.html

  * igt@kms_vrr@flip-basic:
    - shard-lnl:          NOTRUN -> [FAIL][77] ([Intel XE#2443]) +1 other test fail
   [77]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@kms_vrr@flip-basic.html

  * igt@kms_vrr@flip-basic-fastset:
    - shard-dg2-set2:     NOTRUN -> [SKIP][78] ([Intel XE#1201] / [Intel XE#455]) +4 other tests skip
   [78]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_vrr@flip-basic-fastset.html

  * igt@kms_writeback@writeback-pixel-formats:
    - shard-adlp:         NOTRUN -> [SKIP][79] ([Intel XE#1201] / [Intel XE#756])
   [79]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@kms_writeback@writeback-pixel-formats.html

  * igt@xe_drm_fdinfo@utilization-single-full-load-destroy-queue:
    - shard-lnl:          [PASS][80] -> [FAIL][81] ([Intel XE#2667])
   [80]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-7/igt@xe_drm_fdinfo@utilization-single-full-load-destroy-queue.html
   [81]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-2/igt@xe_drm_fdinfo@utilization-single-full-load-destroy-queue.html

  * igt@xe_evict@evict-large-external:
    - shard-adlp:         NOTRUN -> [SKIP][82] ([Intel XE#1201] / [Intel XE#261])
   [82]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_evict@evict-large-external.html

  * igt@xe_evict@evict-mixed-many-threads-small:
    - shard-lnl:          NOTRUN -> [SKIP][83] ([Intel XE#688]) +1 other test skip
   [83]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@xe_evict@evict-mixed-many-threads-small.html

  * igt@xe_evict@evict-mixed-threads-small:
    - shard-adlp:         NOTRUN -> [SKIP][84] ([Intel XE#1201] / [Intel XE#261] / [Intel XE#688])
   [84]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@xe_evict@evict-mixed-threads-small.html

  * igt@xe_evict@evict-threads-large:
    - shard-dg2-set2:     [PASS][85] -> [TIMEOUT][86] ([Intel XE#1473])
   [85]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-436/igt@xe_evict@evict-threads-large.html
   [86]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_evict@evict-threads-large.html

  * igt@xe_evict_ccs@evict-overcommit-standalone-nofree-samefd:
    - shard-adlp:         NOTRUN -> [SKIP][87] ([Intel XE#1201] / [Intel XE#688])
   [87]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_evict_ccs@evict-overcommit-standalone-nofree-samefd.html

  * igt@xe_exec_basic@multigpu-many-execqueues-many-vm-userptr-invalidate:
    - shard-adlp:         NOTRUN -> [SKIP][88] ([Intel XE#1201] / [Intel XE#1392]) +1 other test skip
   [88]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_exec_basic@multigpu-many-execqueues-many-vm-userptr-invalidate.html

  * igt@xe_exec_basic@multigpu-no-exec-userptr-invalidate:
    - shard-lnl:          NOTRUN -> [SKIP][89] ([Intel XE#1392]) +3 other tests skip
   [89]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@xe_exec_basic@multigpu-no-exec-userptr-invalidate.html

  * igt@xe_exec_fault_mode@many-userptr-imm:
    - shard-adlp:         NOTRUN -> [SKIP][90] ([Intel XE#1201] / [Intel XE#288]) +3 other tests skip
   [90]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_exec_fault_mode@many-userptr-imm.html

  * igt@xe_exec_fault_mode@twice-userptr-prefetch:
    - shard-dg2-set2:     NOTRUN -> [SKIP][91] ([Intel XE#1201] / [Intel XE#288]) +1 other test skip
   [91]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_exec_fault_mode@twice-userptr-prefetch.html

  * igt@xe_oa@oa-regs-whitelisted@ccs-0:
    - shard-lnl:          NOTRUN -> [FAIL][92] ([Intel XE#2514])
   [92]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-3/igt@xe_oa@oa-regs-whitelisted@ccs-0.html

  * igt@xe_oa@polling-small-buf:
    - shard-adlp:         NOTRUN -> [SKIP][93] ([Intel XE#1201] / [Intel XE#2541])
   [93]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@xe_oa@polling-small-buf.html

  * igt@xe_peer2peer@read:
    - shard-lnl:          NOTRUN -> [SKIP][94] ([Intel XE#1061])
   [94]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@xe_peer2peer@read.html

  * igt@xe_pm@d3cold-basic:
    - shard-lnl:          NOTRUN -> [SKIP][95] ([Intel XE#2284] / [Intel XE#366])
   [95]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@xe_pm@d3cold-basic.html

  * igt@xe_pm@s3-vm-bind-prefetch:
    - shard-adlp:         [PASS][96] -> [ABORT][97] ([Intel XE#1794])
   [96]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-6/igt@xe_pm@s3-vm-bind-prefetch.html
   [97]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@xe_pm@s3-vm-bind-prefetch.html

  * igt@xe_pm@s4-vm-bind-prefetch:
    - shard-adlp:         [PASS][98] -> [ABORT][99] ([Intel XE#1607] / [Intel XE#1794])
   [98]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-4/igt@xe_pm@s4-vm-bind-prefetch.html
   [99]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-9/igt@xe_pm@s4-vm-bind-prefetch.html

  * igt@xe_pm@s4-vm-bind-unbind-all:
    - shard-lnl:          [PASS][100] -> [ABORT][101] ([Intel XE#1794])
   [100]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-1/igt@xe_pm@s4-vm-bind-unbind-all.html
   [101]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-2/igt@xe_pm@s4-vm-bind-unbind-all.html

  * igt@xe_query@multigpu-query-gt-list:
    - shard-lnl:          NOTRUN -> [SKIP][102] ([Intel XE#944])
   [102]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@xe_query@multigpu-query-gt-list.html

  * igt@xe_query@multigpu-query-invalid-query:
    - shard-adlp:         NOTRUN -> [SKIP][103] ([Intel XE#1201] / [Intel XE#944])
   [103]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_query@multigpu-query-invalid-query.html

  * igt@xe_wedged@wedged-mode-toggle:
    - shard-adlp:         [PASS][104] -> [ABORT][105] ([Intel XE#2310] / [Intel XE#2317])
   [104]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-1/igt@xe_wedged@wedged-mode-toggle.html
   [105]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-1/igt@xe_wedged@wedged-mode-toggle.html

  
#### Possible fixes ####

  * igt@kms_atomic_transition@plane-toggle-modeset-transition:
    - shard-adlp:         [FAIL][106] ([Intel XE#1426]) -> [PASS][107] +1 other test pass
   [106]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-8/igt@kms_atomic_transition@plane-toggle-modeset-transition.html
   [107]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-4/igt@kms_atomic_transition@plane-toggle-modeset-transition.html
    - {shard-bmg}:        [FAIL][108] ([Intel XE#1426]) -> [PASS][109] +1 other test pass
   [108]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-5/igt@kms_atomic_transition@plane-toggle-modeset-transition.html
   [109]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-5/igt@kms_atomic_transition@plane-toggle-modeset-transition.html

  * igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-0:
    - shard-lnl:          [FAIL][110] ([Intel XE#1659]) -> [PASS][111] +1 other test pass
   [110]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-5/igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-0.html
   [111]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-8/igt@kms_big_fb@4-tiled-max-hw-stride-64bpp-rotate-0.html

  * igt@kms_ccs@crc-sprite-planes-basic-4-tiled-bmg-ccs:
    - {shard-bmg}:        [FAIL][112] ([Intel XE#2436]) -> [PASS][113] +3 other tests pass
   [112]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-7/igt@kms_ccs@crc-sprite-planes-basic-4-tiled-bmg-ccs.html
   [113]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-2/igt@kms_ccs@crc-sprite-planes-basic-4-tiled-bmg-ccs.html

  * igt@kms_cursor_crc@cursor-suspend:
    - shard-dg2-set2:     [DMESG-WARN][114] ([Intel XE#1551]) -> [PASS][115] +1 other test pass
   [114]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_cursor_crc@cursor-suspend.html
   [115]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_cursor_crc@cursor-suspend.html

  * igt@kms_flip@2x-flip-vs-blocking-wf-vblank@ad-dp2-hdmi-a3:
    - {shard-bmg}:        [INCOMPLETE][116] -> [PASS][117] +2 other tests pass
   [116]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-1/igt@kms_flip@2x-flip-vs-blocking-wf-vblank@ad-dp2-hdmi-a3.html
   [117]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-8/igt@kms_flip@2x-flip-vs-blocking-wf-vblank@ad-dp2-hdmi-a3.html

  * igt@kms_flip@2x-flip-vs-panning-interruptible:
    - shard-dg2-set2:     [DMESG-WARN][118] ([Intel XE#877]) -> [PASS][119]
   [118]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_flip@2x-flip-vs-panning-interruptible.html
   [119]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_flip@2x-flip-vs-panning-interruptible.html

  * igt@kms_flip@2x-flip-vs-panning-interruptible@ac-hdmi-a6-dp4:
    - shard-dg2-set2:     [DMESG-WARN][120] -> [PASS][121]
   [120]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_flip@2x-flip-vs-panning-interruptible@ac-hdmi-a6-dp4.html
   [121]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_flip@2x-flip-vs-panning-interruptible@ac-hdmi-a6-dp4.html

  * igt@kms_flip@absolute-wf_vblank-interruptible:
    - shard-adlp:         [INCOMPLETE][122] ([Intel XE#1195]) -> [PASS][123] +1 other test pass
   [122]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-6/igt@kms_flip@absolute-wf_vblank-interruptible.html
   [123]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-2/igt@kms_flip@absolute-wf_vblank-interruptible.html

  * igt@kms_flip@blocking-wf_vblank:
    - shard-lnl:          [FAIL][124] ([Intel XE#886]) -> [PASS][125] +1 other test pass
   [124]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-3/igt@kms_flip@blocking-wf_vblank.html
   [125]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-6/igt@kms_flip@blocking-wf_vblank.html

  * igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-x:
    - shard-adlp:         [FAIL][126] ([Intel XE#1874]) -> [PASS][127]
   [126]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-2/igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-x.html
   [127]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-6/igt@kms_flip_tiling@flip-change-tiling@pipe-c-hdmi-a-1-x-to-x.html

  * igt@kms_plane_scaling@2x-scaler-multi-pipe:
    - {shard-bmg}:        [DMESG-WARN][128] ([Intel XE#877]) -> [PASS][129] +12 other tests pass
   [128]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-8/igt@kms_plane_scaling@2x-scaler-multi-pipe.html
   [129]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-4/igt@kms_plane_scaling@2x-scaler-multi-pipe.html

  * igt@kms_universal_plane@cursor-fb-leak:
    - shard-dg2-set2:     [FAIL][130] ([Intel XE#771] / [Intel XE#899]) -> [PASS][131]
   [130]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-436/igt@kms_universal_plane@cursor-fb-leak.html
   [131]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-435/igt@kms_universal_plane@cursor-fb-leak.html

  * igt@kms_universal_plane@cursor-fb-leak@pipe-b-dp-4:
    - shard-dg2-set2:     [FAIL][132] ([Intel XE#899]) -> [PASS][133]
   [132]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-436/igt@kms_universal_plane@cursor-fb-leak@pipe-b-dp-4.html
   [133]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-435/igt@kms_universal_plane@cursor-fb-leak@pipe-b-dp-4.html

  * igt@kms_universal_plane@cursor-fb-leak@pipe-d-dp-2:
    - {shard-bmg}:        [FAIL][134] ([Intel XE#899]) -> [PASS][135]
   [134]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-4/igt@kms_universal_plane@cursor-fb-leak@pipe-d-dp-2.html
   [135]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-7/igt@kms_universal_plane@cursor-fb-leak@pipe-d-dp-2.html

  * igt@kms_vblank@accuracy-idle:
    - shard-lnl:          [FAIL][136] ([Intel XE#1523]) -> [PASS][137] +1 other test pass
   [136]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-3/igt@kms_vblank@accuracy-idle.html
   [137]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-6/igt@kms_vblank@accuracy-idle.html

  * igt@xe_evict@evict-beng-large-multi-vm-cm:
    - {shard-bmg}:        [FAIL][138] ([Intel XE#2364]) -> [PASS][139]
   [138]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-bmg-1/igt@xe_evict@evict-beng-large-multi-vm-cm.html
   [139]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-bmg-8/igt@xe_evict@evict-beng-large-multi-vm-cm.html

  * igt@xe_evict@evict-beng-mixed-threads-large:
    - shard-dg2-set2:     [TIMEOUT][140] ([Intel XE#1473]) -> [PASS][141]
   [140]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@xe_evict@evict-beng-mixed-threads-large.html
   [141]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-463/igt@xe_evict@evict-beng-mixed-threads-large.html

  * igt@xe_evict@evict-mixed-threads-large:
    - shard-dg2-set2:     [FAIL][142] ([Intel XE#1000]) -> [PASS][143]
   [142]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@xe_evict@evict-mixed-threads-large.html
   [143]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-463/igt@xe_evict@evict-mixed-threads-large.html

  * igt@xe_exec_reset@parallel-gt-reset:
    - shard-dg2-set2:     [TIMEOUT][144] ([Intel XE#2105]) -> [PASS][145]
   [144]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_exec_reset@parallel-gt-reset.html
   [145]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_exec_reset@parallel-gt-reset.html

  * igt@xe_exec_threads@threads-hang-shared-vm-userptr-rebind:
    - shard-lnl:          [ABORT][146] -> [PASS][147]
   [146]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-7/igt@xe_exec_threads@threads-hang-shared-vm-userptr-rebind.html
   [147]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-5/igt@xe_exec_threads@threads-hang-shared-vm-userptr-rebind.html

  * igt@xe_oa@oa-exponents:
    - shard-lnl:          [FAIL][148] ([Intel XE#2723]) -> [PASS][149]
   [148]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-4/igt@xe_oa@oa-exponents.html
   [149]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-8/igt@xe_oa@oa-exponents.html

  * igt@xe_pm@s2idle-basic:
    - shard-dg2-set2:     [INCOMPLETE][150] ([Intel XE#1195] / [Intel XE#1358]) -> [PASS][151]
   [150]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-435/igt@xe_pm@s2idle-basic.html
   [151]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_pm@s2idle-basic.html

  * igt@xe_pm@s4-vm-bind-prefetch:
    - shard-lnl:          [ABORT][152] ([Intel XE#1607] / [Intel XE#1794]) -> [PASS][153]
   [152]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-lnl-2/igt@xe_pm@s4-vm-bind-prefetch.html
   [153]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-lnl-4/igt@xe_pm@s4-vm-bind-prefetch.html

  * igt@xe_pm@s4-vm-bind-userptr:
    - shard-adlp:         [ABORT][154] ([Intel XE#1794]) -> [PASS][155]
   [154]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-9/igt@xe_pm@s4-vm-bind-userptr.html
   [155]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-8/igt@xe_pm@s4-vm-bind-userptr.html

  
#### Warnings ####

  * igt@kms_big_fb@4-tiled-8bpp-rotate-270:
    - shard-dg2-set2:     [SKIP][156] ([Intel XE#1201] / [Intel XE#316]) -> [SKIP][157] ([Intel XE#316]) +3 other tests skip
   [156]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_big_fb@4-tiled-8bpp-rotate-270.html
   [157]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_big_fb@4-tiled-8bpp-rotate-270.html

  * igt@kms_big_fb@linear-16bpp-rotate-270:
    - shard-dg2-set2:     [SKIP][158] ([Intel XE#316]) -> [SKIP][159] ([Intel XE#1201] / [Intel XE#316]) +1 other test skip
   [158]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_big_fb@linear-16bpp-rotate-270.html
   [159]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_big_fb@linear-16bpp-rotate-270.html

  * igt@kms_big_fb@x-tiled-max-hw-stride-64bpp-rotate-180-hflip-async-flip:
    - shard-adlp:         [FAIL][160] ([Intel XE#1231]) -> [DMESG-FAIL][161] ([Intel XE#324])
   [160]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-8/igt@kms_big_fb@x-tiled-max-hw-stride-64bpp-rotate-180-hflip-async-flip.html
   [161]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-1/igt@kms_big_fb@x-tiled-max-hw-stride-64bpp-rotate-180-hflip-async-flip.html

  * igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip:
    - shard-dg2-set2:     [SKIP][162] ([Intel XE#1124] / [Intel XE#1201]) -> [SKIP][163] ([Intel XE#1124]) +10 other tests skip
   [162]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip.html
   [163]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip.html

  * igt@kms_big_fb@y-tiled-max-hw-stride-64bpp-rotate-0-async-flip:
    - shard-adlp:         [DMESG-FAIL][164] ([Intel XE#324]) -> [FAIL][165] ([Intel XE#1231])
   [164]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-adlp-2/igt@kms_big_fb@y-tiled-max-hw-stride-64bpp-rotate-0-async-flip.html
   [165]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-adlp-6/igt@kms_big_fb@y-tiled-max-hw-stride-64bpp-rotate-0-async-flip.html

  * igt@kms_big_fb@yf-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip:
    - shard-dg2-set2:     [SKIP][166] ([Intel XE#1124]) -> [SKIP][167] ([Intel XE#1124] / [Intel XE#1201]) +8 other tests skip
   [166]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_big_fb@yf-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip.html
   [167]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_big_fb@yf-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip.html

  * igt@kms_bw@connected-linear-tiling-3-displays-2560x1440p:
    - shard-dg2-set2:     [SKIP][168] ([Intel XE#1201] / [Intel XE#2191]) -> [SKIP][169] ([Intel XE#2191]) +1 other test skip
   [168]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_bw@connected-linear-tiling-3-displays-2560x1440p.html
   [169]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_bw@connected-linear-tiling-3-displays-2560x1440p.html

  * igt@kms_bw@connected-linear-tiling-4-displays-1920x1080p:
    - shard-dg2-set2:     [SKIP][170] ([Intel XE#2191]) -> [SKIP][171] ([Intel XE#1201] / [Intel XE#2191]) +1 other test skip
   [170]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_bw@connected-linear-tiling-4-displays-1920x1080p.html
   [171]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_bw@connected-linear-tiling-4-displays-1920x1080p.html

  * igt@kms_bw@linear-tiling-3-displays-2560x1440p:
    - shard-dg2-set2:     [SKIP][172] ([Intel XE#1201] / [Intel XE#367]) -> [SKIP][173] ([Intel XE#367]) +2 other tests skip
   [172]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_bw@linear-tiling-3-displays-2560x1440p.html
   [173]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_bw@linear-tiling-3-displays-2560x1440p.html

  * igt@kms_bw@linear-tiling-3-displays-3840x2160p:
    - shard-dg2-set2:     [SKIP][174] ([Intel XE#367]) -> [SKIP][175] ([Intel XE#1201] / [Intel XE#367]) +2 other tests skip
   [174]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_bw@linear-tiling-3-displays-3840x2160p.html
   [175]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_bw@linear-tiling-3-displays-3840x2160p.html

  * igt@kms_ccs@bad-aux-stride-y-tiled-gen12-rc-ccs-cc@pipe-d-dp-4:
    - shard-dg2-set2:     [SKIP][176] ([Intel XE#1201] / [Intel XE#455] / [Intel XE#787]) -> [SKIP][177] ([Intel XE#455] / [Intel XE#787]) +25 other tests skip
   [176]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_ccs@bad-aux-stride-y-tiled-gen12-rc-ccs-cc@pipe-d-dp-4.html
   [177]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_ccs@bad-aux-stride-y-tiled-gen12-rc-ccs-cc@pipe-d-dp-4.html

  * igt@kms_ccs@crc-primary-basic-4-tiled-bmg-ccs:
    - shard-dg2-set2:     [SKIP][178] ([Intel XE#1201] / [Intel XE#1252]) -> [SKIP][179] ([Intel XE#1252])
   [178]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_ccs@crc-primary-basic-4-tiled-bmg-ccs.html
   [179]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_ccs@crc-primary-basic-4-tiled-bmg-ccs.html

  * igt@kms_ccs@crc-primary-basic-4-tiled-mtl-rc-ccs@pipe-b-hdmi-a-6:
    - shard-dg2-set2:     [SKIP][180] ([Intel XE#787]) -> [SKIP][181] ([Intel XE#1201] / [Intel XE#787]) +76 other tests skip
   [180]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_ccs@crc-primary-basic-4-tiled-mtl-rc-ccs@pipe-b-hdmi-a-6.html
   [181]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_ccs@crc-primary-basic-4-tiled-mtl-rc-ccs@pipe-b-hdmi-a-6.html

  * igt@kms_ccs@crc-primary-basic-y-tiled-gen12-rc-ccs-cc@pipe-a-dp-4:
    - shard-dg2-set2:     [SKIP][182] ([Intel XE#1201] / [Intel XE#787]) -> [SKIP][183] ([Intel XE#787]) +90 other tests skip
   [182]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_ccs@crc-primary-basic-y-tiled-gen12-rc-ccs-cc@pipe-a-dp-4.html
   [183]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_ccs@crc-primary-basic-y-tiled-gen12-rc-ccs-cc@pipe-a-dp-4.html

  * igt@kms_ccs@missing-ccs-buffer-4-tiled-mtl-mc-ccs@pipe-d-dp-4:
    - shard-dg2-set2:     [SKIP][184] ([Intel XE#455] / [Intel XE#787]) -> [SKIP][185] ([Intel XE#1201] / [Intel XE#455] / [Intel XE#787]) +21 other tests skip
   [184]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_ccs@missing-ccs-buffer-4-tiled-mtl-mc-ccs@pipe-d-dp-4.html
   [185]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_ccs@missing-ccs-buffer-4-tiled-mtl-mc-ccs@pipe-d-dp-4.html

  * igt@kms_cdclk@plane-scaling@pipe-b-dp-4:
    - shard-dg2-set2:     [SKIP][186] ([Intel XE#1152] / [Intel XE#1201]) -> [SKIP][187] ([Intel XE#1152]) +3 other tests skip
   [186]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_cdclk@plane-scaling@pipe-b-dp-4.html
   [187]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_cdclk@plane-scaling@pipe-b-dp-4.html

  * igt@kms_chamelium_color@ctm-blue-to-red:
    - shard-dg2-set2:     [SKIP][188] ([Intel XE#306]) -> [SKIP][189] ([Intel XE#1201] / [Intel XE#306])
   [188]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_chamelium_color@ctm-blue-to-red.html
   [189]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_chamelium_color@ctm-blue-to-red.html

  * igt@kms_chamelium_color@gamma:
    - shard-dg2-set2:     [SKIP][190] ([Intel XE#1201] / [Intel XE#306]) -> [SKIP][191] ([Intel XE#306]) +1 other test skip
   [190]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_chamelium_color@gamma.html
   [191]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_chamelium_color@gamma.html

  * igt@kms_chamelium_frames@hdmi-aspect-ratio:
    - shard-dg2-set2:     [SKIP][192] ([Intel XE#373]) -> [SKIP][193] ([Intel XE#1201] / [Intel XE#373]) +5 other tests skip
   [192]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_chamelium_frames@hdmi-aspect-ratio.html
   [193]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_chamelium_frames@hdmi-aspect-ratio.html

  * igt@kms_chamelium_hpd@vga-hpd:
    - shard-dg2-set2:     [SKIP][194] ([Intel XE#1201] / [Intel XE#373]) -> [SKIP][195] ([Intel XE#373]) +10 other tests skip
   [194]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_chamelium_hpd@vga-hpd.html
   [195]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_chamelium_hpd@vga-hpd.html

  * igt@kms_content_protection@dp-mst-type-0:
    - shard-dg2-set2:     [SKIP][196] ([Intel XE#1201] / [Intel XE#307]) -> [SKIP][197] ([Intel XE#307])
   [196]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_content_protection@dp-mst-type-0.html
   [197]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_content_protection@dp-mst-type-0.html

  * igt@kms_cursor_crc@cursor-offscreen-512x512:
    - shard-dg2-set2:     [SKIP][198] ([Intel XE#1201] / [Intel XE#308]) -> [SKIP][199] ([Intel XE#308])
   [198]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_cursor_crc@cursor-offscreen-512x512.html
   [199]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_cursor_crc@cursor-offscreen-512x512.html

  * igt@kms_cursor_crc@cursor-sliding-512x170:
    - shard-dg2-set2:     [SKIP][200] ([Intel XE#308]) -> [SKIP][201] ([Intel XE#1201] / [Intel XE#308]) +2 other tests skip
   [200]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_cursor_crc@cursor-sliding-512x170.html
   [201]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_cursor_crc@cursor-sliding-512x170.html

  * igt@kms_cursor_legacy@short-busy-flip-before-cursor-toggle:
    - shard-dg2-set2:     [SKIP][202] ([Intel XE#1201] / [Intel XE#323]) -> [SKIP][203] ([Intel XE#323]) +2 other tests skip
   [202]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_cursor_legacy@short-busy-flip-before-cursor-toggle.html
   [203]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_cursor_legacy@short-busy-flip-before-cursor-toggle.html

  * igt@kms_dither@fb-8bpc-vs-panel-6bpc@pipe-a-hdmi-a-6:
    - shard-dg2-set2:     [SKIP][204] ([Intel XE#1201] / [i915#3804]) -> [SKIP][205] ([i915#3804])
   [204]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_dither@fb-8bpc-vs-panel-6bpc@pipe-a-hdmi-a-6.html
   [205]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_dither@fb-8bpc-vs-panel-6bpc@pipe-a-hdmi-a-6.html

  * igt@kms_feature_discovery@display-3x:
    - shard-dg2-set2:     [SKIP][206] ([Intel XE#703]) -> [SKIP][207] ([Intel XE#1201] / [Intel XE#703])
   [206]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_feature_discovery@display-3x.html
   [207]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_feature_discovery@display-3x.html

  * igt@kms_feature_discovery@dp-mst:
    - shard-dg2-set2:     [SKIP][208] ([Intel XE#1137] / [Intel XE#1201]) -> [SKIP][209] ([Intel XE#1137])
   [208]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_feature_discovery@dp-mst.html
   [209]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_feature_discovery@dp-mst.html

  * igt@kms_feature_discovery@psr2:
    - shard-dg2-set2:     [SKIP][210] ([Intel XE#1135]) -> [SKIP][211] ([Intel XE#1135] / [Intel XE#1201])
   [210]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_feature_discovery@psr2.html
   [211]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_feature_discovery@psr2.html

  * igt@kms_flip_scaled_crc@flip-64bpp-ytile-to-32bpp-ytile-downscaling:
    - shard-dg2-set2:     [SKIP][212] ([Intel XE#455]) -> [SKIP][213] ([Intel XE#1201] / [Intel XE#455]) +11 other tests skip
   [212]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_flip_scaled_crc@flip-64bpp-ytile-to-32bpp-ytile-downscaling.html
   [213]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_flip_scaled_crc@flip-64bpp-ytile-to-32bpp-ytile-downscaling.html

  * igt@kms_force_connector_basic@prune-stale-modes:
    - shard-dg2-set2:     [SKIP][214] ([Intel XE#1201] / [i915#5274]) -> [SKIP][215] ([i915#5274])
   [214]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_force_connector_basic@prune-stale-modes.html
   [215]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_force_connector_basic@prune-stale-modes.html

  * igt@kms_frontbuffer_tracking@drrs-2p-scndscrn-pri-shrfb-draw-render:
    - shard-dg2-set2:     [SKIP][216] ([Intel XE#651]) -> [SKIP][217] ([Intel XE#1201] / [Intel XE#651]) +21 other tests skip
   [216]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_frontbuffer_tracking@drrs-2p-scndscrn-pri-shrfb-draw-render.html
   [217]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_frontbuffer_tracking@drrs-2p-scndscrn-pri-shrfb-draw-render.html

  * igt@kms_frontbuffer_tracking@fbcdrrs-rgb101010-draw-mmap-wc:
    - shard-dg2-set2:     [SKIP][218] ([Intel XE#1201] / [Intel XE#651]) -> [SKIP][219] ([Intel XE#651]) +29 other tests skip
   [218]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_frontbuffer_tracking@fbcdrrs-rgb101010-draw-mmap-wc.html
   [219]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_frontbuffer_tracking@fbcdrrs-rgb101010-draw-mmap-wc.html

  * igt@kms_frontbuffer_tracking@fbcpsr-tiling-y:
    - shard-dg2-set2:     [SKIP][220] ([Intel XE#1201] / [Intel XE#658]) -> [SKIP][221] ([Intel XE#658])
   [220]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_frontbuffer_tracking@fbcpsr-tiling-y.html
   [221]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_frontbuffer_tracking@fbcpsr-tiling-y.html

  * igt@kms_frontbuffer_tracking@plane-fbc-rte:
    - shard-dg2-set2:     [SKIP][222] ([Intel XE#1158] / [Intel XE#1201]) -> [SKIP][223] ([Intel XE#1158])
   [222]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_frontbuffer_tracking@plane-fbc-rte.html
   [223]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_frontbuffer_tracking@plane-fbc-rte.html

  * igt@kms_frontbuffer_tracking@psr-2p-primscrn-shrfb-plflip-blt:
    - shard-dg2-set2:     [SKIP][224] ([Intel XE#1201] / [Intel XE#653]) -> [SKIP][225] ([Intel XE#653]) +28 other tests skip
   [224]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_frontbuffer_tracking@psr-2p-primscrn-shrfb-plflip-blt.html
   [225]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_frontbuffer_tracking@psr-2p-primscrn-shrfb-plflip-blt.html

  * igt@kms_frontbuffer_tracking@psr-slowdraw:
    - shard-dg2-set2:     [SKIP][226] ([Intel XE#653]) -> [SKIP][227] ([Intel XE#1201] / [Intel XE#653]) +22 other tests skip
   [226]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_frontbuffer_tracking@psr-slowdraw.html
   [227]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_frontbuffer_tracking@psr-slowdraw.html

  * igt@kms_hdr@invalid-hdr:
    - shard-dg2-set2:     [SKIP][228] ([Intel XE#1201] / [Intel XE#455]) -> [SKIP][229] ([Intel XE#455]) +14 other tests skip
   [228]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@kms_hdr@invalid-hdr.html
   [229]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_hdr@invalid-hdr.html

  * igt@kms_multipipe_modeset@basic-max-pipe-crc-check:
    - shard-dg2-set2:     [SKIP][230] ([Intel XE#1201] / [Intel XE#356]) -> [SKIP][231] ([Intel XE#356])
   [230]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_multipipe_modeset@basic-max-pipe-crc-check.html
   [231]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_multipipe_modeset@basic-max-pipe-crc-check.html

  * igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers:
    - shard-dg2-set2:     [SKIP][232] ([Intel XE#455] / [Intel XE#498]) -> [SKIP][233] ([Intel XE#1201] / [Intel XE#455] / [Intel XE#498]) +1 other test skip
   [232]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers.html
   [233]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers.html

  * igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers@pipe-c-hdmi-a-6:
    - shard-dg2-set2:     [SKIP][234] ([Intel XE#498]) -> [SKIP][235] ([Intel XE#1201] / [Intel XE#498]) +2 other tests skip
   [234]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers@pipe-c-hdmi-a-6.html
   [235]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@kms_plane_scaling@plane-downscale-factor-0-25-with-modifiers@pipe-c-hdmi-a-6.html

  * igt@kms_plane_scaling@planes-downscale-factor-0-25-unity-scaling:
    - shard-dg2-set2:     [SKIP][236] ([Intel XE#2318] / [Intel XE#455]) -> [SKIP][237] ([Intel XE#1201] / [Intel XE#2318] / [Intel XE#455]) +5 other tests skip
   [236]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_plane_scaling@planes-downscale-factor-0-25-unity-scaling.html
   [237]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_plane_scaling@planes-downscale-factor-0-25-unity-scaling.html

  * igt@kms_plane_scaling@planes-upscale-factor-0-25-downscale-factor-0-25@pipe-c-hdmi-a-6:
    - shard-dg2-set2:     [SKIP][238] ([Intel XE#2318]) -> [SKIP][239] ([Intel XE#1201] / [Intel XE#2318]) +8 other tests skip
   [238]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_plane_scaling@planes-upscale-factor-0-25-downscale-factor-0-25@pipe-c-hdmi-a-6.html
   [239]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_plane_scaling@planes-upscale-factor-0-25-downscale-factor-0-25@pipe-c-hdmi-a-6.html

  * igt@kms_pm_backlight@fade-with-dpms:
    - shard-dg2-set2:     [SKIP][240] ([Intel XE#1201] / [Intel XE#870]) -> [SKIP][241] ([Intel XE#870])
   [240]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_pm_backlight@fade-with-dpms.html
   [241]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_pm_backlight@fade-with-dpms.html

  * igt@kms_pm_dc@dc3co-vpb-simulation:
    - shard-dg2-set2:     [SKIP][242] ([Intel XE#1122] / [Intel XE#1201]) -> [SKIP][243] ([Intel XE#1122])
   [242]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_pm_dc@dc3co-vpb-simulation.html
   [243]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_pm_dc@dc3co-vpb-simulation.html

  * igt@kms_psr2_sf@fbc-cursor-plane-move-continuous-exceed-fully-sf:
    - shard-dg2-set2:     [SKIP][244] ([Intel XE#1489]) -> [SKIP][245] ([Intel XE#1201] / [Intel XE#1489]) +2 other tests skip
   [244]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_psr2_sf@fbc-cursor-plane-move-continuous-exceed-fully-sf.html
   [245]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@kms_psr2_sf@fbc-cursor-plane-move-continuous-exceed-fully-sf.html

  * igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area:
    - shard-dg2-set2:     [SKIP][246] ([Intel XE#1201] / [Intel XE#1489]) -> [SKIP][247] ([Intel XE#1489]) +4 other tests skip
   [246]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area.html
   [247]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area.html

  * igt@kms_psr@fbc-pr-dpms:
    - shard-dg2-set2:     [SKIP][248] ([Intel XE#929]) -> [SKIP][249] ([Intel XE#1201] / [Intel XE#929]) +11 other tests skip
   [248]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_psr@fbc-pr-dpms.html
   [249]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_psr@fbc-pr-dpms.html

  * igt@kms_psr@fbc-psr2-sprite-plane-move:
    - shard-dg2-set2:     [SKIP][250] ([Intel XE#1201] / [Intel XE#929]) -> [SKIP][251] ([Intel XE#929]) +14 other tests skip
   [250]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_psr@fbc-psr2-sprite-plane-move.html
   [251]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_psr@fbc-psr2-sprite-plane-move.html

  * igt@kms_rotation_crc@primary-rotation-270:
    - shard-dg2-set2:     [SKIP][252] ([Intel XE#327]) -> [SKIP][253] ([Intel XE#1201] / [Intel XE#327]) +1 other test skip
   [252]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_rotation_crc@primary-rotation-270.html
   [253]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_rotation_crc@primary-rotation-270.html

  * igt@kms_rotation_crc@primary-yf-tiled-reflect-x-0:
    - shard-dg2-set2:     [SKIP][254] ([Intel XE#1127]) -> [SKIP][255] ([Intel XE#1127] / [Intel XE#1201])
   [254]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_rotation_crc@primary-yf-tiled-reflect-x-0.html
   [255]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_rotation_crc@primary-yf-tiled-reflect-x-0.html

  * igt@kms_rotation_crc@sprite-rotation-90:
    - shard-dg2-set2:     [SKIP][256] ([Intel XE#1201] / [Intel XE#327]) -> [SKIP][257] ([Intel XE#327])
   [256]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@kms_rotation_crc@sprite-rotation-90.html
   [257]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_rotation_crc@sprite-rotation-90.html

  * igt@kms_tiled_display@basic-test-pattern-with-chamelium:
    - shard-dg2-set2:     [SKIP][258] ([Intel XE#1201] / [Intel XE#362]) -> [SKIP][259] ([Intel XE#1201] / [Intel XE#1500])
   [258]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_tiled_display@basic-test-pattern-with-chamelium.html
   [259]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-436/igt@kms_tiled_display@basic-test-pattern-with-chamelium.html

  * igt@kms_writeback@writeback-check-output:
    - shard-dg2-set2:     [SKIP][260] ([Intel XE#1201] / [Intel XE#756]) -> [SKIP][261] ([Intel XE#756])
   [260]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@kms_writeback@writeback-check-output.html
   [261]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@kms_writeback@writeback-check-output.html

  * igt@kms_writeback@writeback-fb-id-xrgb2101010:
    - shard-dg2-set2:     [SKIP][262] ([Intel XE#756]) -> [SKIP][263] ([Intel XE#1201] / [Intel XE#756])
   [262]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@kms_writeback@writeback-fb-id-xrgb2101010.html
   [263]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@kms_writeback@writeback-fb-id-xrgb2101010.html

  * igt@xe_compute_preempt@compute-preempt:
    - shard-dg2-set2:     [SKIP][264] ([Intel XE#1280] / [Intel XE#455]) -> [SKIP][265] ([Intel XE#1201] / [Intel XE#1280] / [Intel XE#455]) +1 other test skip
   [264]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_compute_preempt@compute-preempt.html
   [265]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@xe_compute_preempt@compute-preempt.html

  * igt@xe_compute_preempt@compute-threadgroup-preempt@engine-drm_xe_engine_class_compute:
    - shard-dg2-set2:     [SKIP][266] ([Intel XE#1201] / [Intel XE#1280] / [Intel XE#455]) -> [SKIP][267] ([Intel XE#1280] / [Intel XE#455]) +1 other test skip
   [266]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@xe_compute_preempt@compute-threadgroup-preempt@engine-drm_xe_engine_class_compute.html
   [267]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_compute_preempt@compute-threadgroup-preempt@engine-drm_xe_engine_class_compute.html

  * igt@xe_copy_basic@mem-copy-linear-0xfd:
    - shard-dg2-set2:     [SKIP][268] ([Intel XE#1123] / [Intel XE#1201]) -> [SKIP][269] ([Intel XE#1123])
   [268]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-466/igt@xe_copy_basic@mem-copy-linear-0xfd.html
   [269]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_copy_basic@mem-copy-linear-0xfd.html

  * igt@xe_copy_basic@mem-copy-linear-0xfffe:
    - shard-dg2-set2:     [SKIP][270] ([Intel XE#1123]) -> [SKIP][271] ([Intel XE#1123] / [Intel XE#1201])
   [270]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_copy_basic@mem-copy-linear-0xfffe.html
   [271]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_copy_basic@mem-copy-linear-0xfffe.html

  * igt@xe_copy_basic@mem-set-linear-0xfd:
    - shard-dg2-set2:     [SKIP][272] ([Intel XE#1126] / [Intel XE#1201]) -> [SKIP][273] ([Intel XE#1126]) +1 other test skip
   [272]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_copy_basic@mem-set-linear-0xfd.html
   [273]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_copy_basic@mem-set-linear-0xfd.html

  * igt@xe_exec_fault_mode@once-bindexecqueue-imm:
    - shard-dg2-set2:     [SKIP][274] ([Intel XE#1201] / [Intel XE#288]) -> [SKIP][275] ([Intel XE#288]) +24 other tests skip
   [274]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-433/igt@xe_exec_fault_mode@once-bindexecqueue-imm.html
   [275]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_exec_fault_mode@once-bindexecqueue-imm.html

  * igt@xe_exec_fault_mode@twice-invalid-fault:
    - shard-dg2-set2:     [SKIP][276] ([Intel XE#288]) -> [SKIP][277] ([Intel XE#1201] / [Intel XE#288]) +18 other tests skip
   [276]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_exec_fault_mode@twice-invalid-fault.html
   [277]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_exec_fault_mode@twice-invalid-fault.html

  * igt@xe_exec_mix_modes@exec-spinner-interrupted-dma-fence:
    - shard-dg2-set2:     [SKIP][278] ([Intel XE#1201] / [Intel XE#2360]) -> [SKIP][279] ([Intel XE#2360])
   [278]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_exec_mix_modes@exec-spinner-interrupted-dma-fence.html
   [279]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_exec_mix_modes@exec-spinner-interrupted-dma-fence.html

  * igt@xe_live_ktest@xe_migrate@xe_validate_ccs_kunit:
    - shard-dg2-set2:     [SKIP][280] ([Intel XE#1201] / [Intel XE#2229]) -> [SKIP][281] ([Intel XE#2229])
   [280]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_live_ktest@xe_migrate@xe_validate_ccs_kunit.html
   [281]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_live_ktest@xe_migrate@xe_validate_ccs_kunit.html

  * igt@xe_oa@mmio-triggered-reports:
    - shard-dg2-set2:     [SKIP][282] ([Intel XE#2541]) -> [SKIP][283] ([Intel XE#1201] / [Intel XE#2541]) +4 other tests skip
   [282]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_oa@mmio-triggered-reports.html
   [283]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-466/igt@xe_oa@mmio-triggered-reports.html

  * igt@xe_oa@oa-unit-exclusive-stream-sample-oa:
    - shard-dg2-set2:     [SKIP][284] ([Intel XE#1201] / [Intel XE#2541]) -> [SKIP][285] ([Intel XE#2541]) +6 other tests skip
   [284]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_oa@oa-unit-exclusive-stream-sample-oa.html
   [285]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_oa@oa-unit-exclusive-stream-sample-oa.html

  * igt@xe_pat@pat-index-xehpc:
    - shard-dg2-set2:     [SKIP][286] ([Intel XE#979]) -> [SKIP][287] ([Intel XE#1201] / [Intel XE#979])
   [286]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_pat@pat-index-xehpc.html
   [287]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@xe_pat@pat-index-xehpc.html

  * igt@xe_pm@d3cold-mocs:
    - shard-dg2-set2:     [SKIP][288] ([Intel XE#2284]) -> [SKIP][289] ([Intel XE#1201] / [Intel XE#2284])
   [288]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_pm@d3cold-mocs.html
   [289]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-433/igt@xe_pm@d3cold-mocs.html

  * igt@xe_query@multigpu-query-mem-usage:
    - shard-dg2-set2:     [SKIP][290] ([Intel XE#1201] / [Intel XE#944]) -> [SKIP][291] ([Intel XE#944]) +3 other tests skip
   [290]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-463/igt@xe_query@multigpu-query-mem-usage.html
   [291]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-432/igt@xe_query@multigpu-query-mem-usage.html

  * igt@xe_query@multigpu-query-uc-fw-version-huc:
    - shard-dg2-set2:     [SKIP][292] ([Intel XE#944]) -> [SKIP][293] ([Intel XE#1201] / [Intel XE#944]) +1 other test skip
   [292]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3/shard-dg2-432/igt@xe_query@multigpu-query-uc-fw-version-huc.html
   [293]: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/shard-dg2-434/igt@xe_query@multigpu-query-uc-fw-version-huc.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [Intel XE#1000]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1000
  [Intel XE#1033]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1033
  [Intel XE#1035]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1035
  [Intel XE#1061]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1061
  [Intel XE#1091]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1091
  [Intel XE#1122]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1122
  [Intel XE#1123]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1123
  [Intel XE#1124]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1124
  [Intel XE#1126]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1126
  [Intel XE#1127]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1127
  [Intel XE#1128]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1128
  [Intel XE#1135]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1135
  [Intel XE#1137]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1137
  [Intel XE#1152]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1152
  [Intel XE#1158]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1158
  [Intel XE#1195]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1195
  [Intel XE#1201]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1201
  [Intel XE#1231]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1231
  [Intel XE#1252]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1252
  [Intel XE#1280]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1280
  [Intel XE#1358]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1358
  [Intel XE#1392]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1392
  [Intel XE#1399]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1399
  [Intel XE#1401]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1401
  [Intel XE#1406]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1406
  [Intel XE#1421]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1421
  [Intel XE#1424]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1424
  [Intel XE#1426]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1426
  [Intel XE#1437]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1437
  [Intel XE#1439]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1439
  [Intel XE#1467]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1467
  [Intel XE#1473]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1473
  [Intel XE#1489]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1489
  [Intel XE#1500]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1500
  [Intel XE#1503]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1503
  [Intel XE#1512]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1512
  [Intel XE#1523]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1523
  [Intel XE#1551]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1551
  [Intel XE#1607]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1607
  [Intel XE#1659]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1659
  [Intel XE#1745]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1745
  [Intel XE#1794]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1794
  [Intel XE#1874]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/1874
  [Intel XE#2026]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2026
  [Intel XE#2078]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2078
  [Intel XE#2105]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2105
  [Intel XE#2191]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2191
  [Intel XE#2229]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2229
  [Intel XE#2234]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2234
  [Intel XE#2244]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2244
  [Intel XE#2251]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2251
  [Intel XE#2252]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2252
  [Intel XE#2284]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2284
  [Intel XE#2310]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2310
  [Intel XE#2311]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2311
  [Intel XE#2313]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2313
  [Intel XE#2317]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2317
  [Intel XE#2318]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2318
  [Intel XE#2320]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2320
  [Intel XE#2321]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2321
  [Intel XE#2322]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2322
  [Intel XE#2327]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2327
  [Intel XE#2333]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2333
  [Intel XE#2357]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2357
  [Intel XE#2360]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2360
  [Intel XE#2364]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2364
  [Intel XE#2393]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2393
  [Intel XE#2426]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2426
  [Intel XE#2436]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2436
  [Intel XE#2443]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2443
  [Intel XE#2509]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2509
  [Intel XE#2514]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2514
  [Intel XE#2541]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2541
  [Intel XE#261]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/261
  [Intel XE#2667]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2667
  [Intel XE#2723]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2723
  [Intel XE#2748]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2748
  [Intel XE#2749]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/2749
  [Intel XE#288]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/288
  [Intel XE#306]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/306
  [Intel XE#307]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/307
  [Intel XE#308]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/308
  [Intel XE#309]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/309
  [Intel XE#310]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/310
  [Intel XE#314]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/314
  [Intel XE#316]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/316
  [Intel XE#323]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/323
  [Intel XE#324]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/324
  [Intel XE#327]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/327
  [Intel XE#356]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/356
  [Intel XE#361]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/361
  [Intel XE#362]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/362
  [Intel XE#366]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/366
  [Intel XE#367]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/367
  [Intel XE#373]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/373
  [Intel XE#455]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/455
  [Intel XE#498]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/498
  [Intel XE#599]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/599
  [Intel XE#619]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/619
  [Intel XE#651]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/651
  [Intel XE#653]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/653
  [Intel XE#656]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/656
  [Intel XE#658]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/658
  [Intel XE#688]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/688
  [Intel XE#703]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/703
  [Intel XE#756]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/756
  [Intel XE#771]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/771
  [Intel XE#787]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/787
  [Intel XE#870]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/870
  [Intel XE#877]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/877
  [Intel XE#886]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/886
  [Intel XE#899]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/899
  [Intel XE#929]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/929
  [Intel XE#944]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/944
  [Intel XE#979]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/979
  [i915#3804]: https://gitlab.freedesktop.org/drm/i915/kernel/-/issues/3804
  [i915#5274]: https://gitlab.freedesktop.org/drm/i915/kernel/-/issues/5274


Build changes
-------------

  * Linux: xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3 -> xe-pw-138497v1

  IGT_8012: 8012
  xe-1924-f3eded4f8a05d73a0b94f27e05737ea3427450b3: f3eded4f8a05d73a0b94f27e05737ea3427450b3
  xe-pw-138497v1: 138497v1

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/intel-xe/xe-pw-138497v1/index.html

[-- Attachment #2: Type: text/html, Size: 104665 bytes --]

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> 
> Matthew Brost <matthew.brost@intel.com> writes:
> 
> > Avoid multiple CPU page faults to the same device page racing by locking
> > the page in do_swap_page before taking an additional reference to the
> > page. This prevents scenarios where multiple CPU page faults each take
> > an extra reference to a device page, which could abort migration in
> > folio_migrate_mapping. With the device page locked in do_swap_page, the
> > migrate_vma_* functions need to be updated to avoid locking the
> > fault_page argument.
> 
> I added the get_page() and therefore the fault_page argument to deal
> with another lifetime issue. Neither Felix nor I particularly liked the
> solution though (see
> https://lore.kernel.org/all/cover.60659b549d8509ddecafad4f498ee7f03bb23c69.1664366292.git-series.apopple@nvidia.com/T/#m715589d57716448386ff9af41da27a952d94615a)
> and this seems to make it even uglier, so I have suggested an
> alternative solution below.
> 

Thanks for the ref, will read this through.

> > Prior to this change, a livelock scenario could occur in Xe's (Intel GPU
> > DRM driver) SVM implementation if enough threads faulted the same device
> > page.
> 
> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)

Still a driver.

> but theoretically it seems like it should be possible. However we
> serialize migrations of the same virtual address range to avoid these
> kind of issues as they can happen the other way too (ie. multiple
> threads trying to migrate to GPU).
> 
> So I suspect what happens in UVM is that one thread wins and installs
> the migration entry while the others fail to get the driver migration
> lock and bail out sufficiently early in the fault path to avoid the
> live-lock.
> 

I had to try hard to show this, doubt an actual user could trigger this.

I wrote a test which kicked 8 threads, each thread did a pthread join,
and then tried to read the same page. This repeats in loop for like 512
pages or something. I needed an exclusive lock in migrate_to_ram vfunc
for it to livelock. Without an exclusive lock I think on average I saw
about 32k retries (i.e. migrate_to_ram calls on the same page) before a
thread won this race.

From reading UVM, pretty sure if you tried hard enough you could trigger
a livelock given it appears you take excluvise locks in migrate_to_ram.

> > Cc: Philip Yang <Philip.Yang@amd.com>
> > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > Cc: Christian König <christian.koenig@amd.com>
> > Cc: Andrew Morton <akpm@linux-foundation.org>
> > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> >  mm/memory.c         | 13 +++++++---
> >  mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> >  2 files changed, 50 insertions(+), 23 deletions(-)
> >
> > diff --git a/mm/memory.c b/mm/memory.c
> > index 3c01d68065be..bbd97d16a96a 100644
> > --- a/mm/memory.c
> > +++ b/mm/memory.c
> > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> >  			 * Get a page reference while we know the page can't be
> >  			 * freed.
> >  			 */
> > -			get_page(vmf->page);
> > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > -			put_page(vmf->page);
> > +			if (trylock_page(vmf->page)) {
> > +				get_page(vmf->page);
> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> 
> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> rather than do this and then have to pass all this context
> (ie. fault_page) down to the migrate_vma_* functions could we instead
> just do what migrate_vma_collect_pmd() does here? Ie. we already have
> the PTL and the page lock so there's no reason we couldn't just setup
> the migration entry prior to calling migrate_to_ram().
> 
> Obviously calling migrate_vma_setup() would show the page as not
> migrating, but drivers could easily just fill in the src_pfn info after
> calling migrate_vma_setup().
> 
> This would eliminate the whole fault_page ugliness.
>

This seems like it would work and agree it likely be cleaner. Let me
play around with this and see what I come up with. Multi-tasking a bit
so expect a bit of delay here.

Thanks for the input,
Matt

> > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > +				put_page(vmf->page);
> > +				unlock_page(vmf->page);
> > +			} else {
> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > +			}
> >  		} else if (is_hwpoison_entry(entry)) {
> >  			ret = VM_FAULT_HWPOISON;
> >  		} else if (is_pte_marker_entry(entry)) {
> > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > index 6d66dc1c6ffa..049893a5a179 100644
> > --- a/mm/migrate_device.c
> > +++ b/mm/migrate_device.c
> > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  				   struct mm_walk *walk)
> >  {
> >  	struct migrate_vma *migrate = walk->private;
> > +	struct folio *fault_folio = migrate->fault_page ?
> > +		page_folio(migrate->fault_page) : NULL;
> >  	struct vm_area_struct *vma = walk->vma;
> >  	struct mm_struct *mm = vma->vm_mm;
> >  	unsigned long addr = start, unmapped = 0;
> > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  
> >  			folio_get(folio);
> >  			spin_unlock(ptl);
> > -			if (unlikely(!folio_trylock(folio)))
> > +			if (unlikely(fault_folio != folio &&
> > +				     !folio_trylock(folio)))
> >  				return migrate_vma_collect_skip(start, end,
> >  								walk);
> >  			ret = split_folio(folio);
> > -			folio_unlock(folio);
> > +			if (fault_folio != folio)
> > +				folio_unlock(folio);
> >  			folio_put(folio);
> >  			if (ret)
> >  				return migrate_vma_collect_skip(start, end,
> > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  		 * optimisation to avoid walking the rmap later with
> >  		 * try_to_migrate().
> >  		 */
> > -		if (folio_trylock(folio)) {
> > +		if (fault_folio == folio || folio_trylock(folio)) {
> >  			bool anon_exclusive;
> >  			pte_t swp_pte;
> >  
> > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  
> >  				if (folio_try_share_anon_rmap_pte(folio, page)) {
> >  					set_pte_at(mm, addr, ptep, pte);
> > -					folio_unlock(folio);
> > +					if (fault_folio != folio)
> > +						folio_unlock(folio);
> >  					folio_put(folio);
> >  					mpfn = 0;
> >  					goto next;
> > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >  					  unsigned long npages,
> >  					  struct page *fault_page)
> >  {
> > +	struct folio *fault_folio = fault_page ?
> > +		page_folio(fault_page) : NULL;
> >  	unsigned long i, restore = 0;
> >  	bool allow_drain = true;
> >  	unsigned long unmapped = 0;
> > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >  		remove_migration_ptes(folio, folio, false);
> >  
> >  		src_pfns[i] = 0;
> > -		folio_unlock(folio);
> > +		if (fault_folio != folio)
> > +			folio_unlock(folio);
> >  		folio_put(folio);
> >  		restore--;
> >  	}
> > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> >  		return -EINVAL;
> >  	if (args->fault_page && !is_device_private_page(args->fault_page))
> >  		return -EINVAL;
> > +	if (args->fault_page && !PageLocked(args->fault_page))
> > +		return -EINVAL;
> >  
> >  	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> >  	args->cpages = 0;
> > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> >  }
> >  EXPORT_SYMBOL(migrate_vma_pages);
> >  
> > -/*
> > - * migrate_device_finalize() - complete page migration
> > - * @src_pfns: src_pfns returned from migrate_device_range()
> > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > - * @npages: number of pages in the range
> > - *
> > - * Completes migration of the page by removing special migration entries.
> > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > - * before calling this.
> > - */
> > -void migrate_device_finalize(unsigned long *src_pfns,
> > -			unsigned long *dst_pfns, unsigned long npages)
> > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > +				      unsigned long *dst_pfns,
> > +				      unsigned long npages,
> > +				      struct page *fault_page)
> >  {
> > +	struct folio *fault_folio = fault_page ?
> > +		page_folio(fault_page) : NULL;
> >  	unsigned long i;
> >  
> >  	for (i = 0; i < npages; i++) {
> > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >  		src = page_folio(page);
> >  		dst = page_folio(newpage);
> >  		remove_migration_ptes(src, dst, false);
> > -		folio_unlock(src);
> > +		if (fault_folio != src)
> > +			folio_unlock(src);
> >  
> >  		if (is_zone_device_page(page))
> >  			put_page(page);
> > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >  		}
> >  	}
> >  }
> > +
> > +/*
> > + * migrate_device_finalize() - complete page migration
> > + * @src_pfns: src_pfns returned from migrate_device_range()
> > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > + * @npages: number of pages in the range
> > + *
> > + * Completes migration of the page by removing special migration entries.
> > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > + * before calling this.
> > + */
> > +void migrate_device_finalize(unsigned long *src_pfns,
> > +			unsigned long *dst_pfns, unsigned long npages)
> > +{
> > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > +}
> >  EXPORT_SYMBOL(migrate_device_finalize);
> >  
> >  /**
> > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> >   */
> >  void migrate_vma_finalize(struct migrate_vma *migrate)
> >  {
> > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > +				  migrate->fault_page);
> >  }
> >  EXPORT_SYMBOL(migrate_vma_finalize);
> 

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Alistair Popple]

Matthew Brost <matthew.brost@intel.com> writes:

> On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
>> 
>> Matthew Brost <matthew.brost@intel.com> writes:
>> 
>> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
>
> Still a driver.

Indeed, and I'm happy to answer any questions about our implementation.

>> but theoretically it seems like it should be possible. However we
>> serialize migrations of the same virtual address range to avoid these
>> kind of issues as they can happen the other way too (ie. multiple
>> threads trying to migrate to GPU).
>> 
>> So I suspect what happens in UVM is that one thread wins and installs
>> the migration entry while the others fail to get the driver migration
>> lock and bail out sufficiently early in the fault path to avoid the
>> live-lock.
>> 
>
> I had to try hard to show this, doubt an actual user could trigger this.
>
> I wrote a test which kicked 8 threads, each thread did a pthread join,
> and then tried to read the same page. This repeats in loop for like 512
> pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> for it to livelock. Without an exclusive lock I think on average I saw
> about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> thread won this race.
>
> From reading UVM, pretty sure if you tried hard enough you could trigger
> a livelock given it appears you take excluvise locks in migrate_to_ram.

Yes, I suspect you're correct here and that we just haven't tried hard
enough to trigger it.

>> > Cc: Philip Yang <Philip.Yang@amd.com>
>> > Cc: Felix Kuehling <felix.kuehling@amd.com>
>> > Cc: Christian König <christian.koenig@amd.com>
>> > Cc: Andrew Morton <akpm@linux-foundation.org>
>> > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
>> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>> > ---
>> >  mm/memory.c         | 13 +++++++---
>> >  mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>> >  2 files changed, 50 insertions(+), 23 deletions(-)
>> >
>> > diff --git a/mm/memory.c b/mm/memory.c
>> > index 3c01d68065be..bbd97d16a96a 100644
>> > --- a/mm/memory.c
>> > +++ b/mm/memory.c
>> > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>> >  			 * Get a page reference while we know the page can't be
>> >  			 * freed.
>> >  			 */
>> > -			get_page(vmf->page);
>> > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
>> > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>> > -			put_page(vmf->page);
>> > +			if (trylock_page(vmf->page)) {
>> > +				get_page(vmf->page);
>> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>> 
>> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
>> rather than do this and then have to pass all this context
>> (ie. fault_page) down to the migrate_vma_* functions could we instead
>> just do what migrate_vma_collect_pmd() does here? Ie. we already have
>> the PTL and the page lock so there's no reason we couldn't just setup
>> the migration entry prior to calling migrate_to_ram().
>> 
>> Obviously calling migrate_vma_setup() would show the page as not
>> migrating, but drivers could easily just fill in the src_pfn info after
>> calling migrate_vma_setup().
>> 
>> This would eliminate the whole fault_page ugliness.
>>
>
> This seems like it would work and agree it likely be cleaner. Let me
> play around with this and see what I come up with. Multi-tasking a bit
> so expect a bit of delay here.
>
> Thanks for the input,
> Matt
>
>> > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>> > +				put_page(vmf->page);
>> > +				unlock_page(vmf->page);
>> > +			} else {
>> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>> > +			}
>> >  		} else if (is_hwpoison_entry(entry)) {
>> >  			ret = VM_FAULT_HWPOISON;
>> >  		} else if (is_pte_marker_entry(entry)) {
>> > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
>> > index 6d66dc1c6ffa..049893a5a179 100644
>> > --- a/mm/migrate_device.c
>> > +++ b/mm/migrate_device.c
>> > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> >  				   struct mm_walk *walk)
>> >  {
>> >  	struct migrate_vma *migrate = walk->private;
>> > +	struct folio *fault_folio = migrate->fault_page ?
>> > +		page_folio(migrate->fault_page) : NULL;
>> >  	struct vm_area_struct *vma = walk->vma;
>> >  	struct mm_struct *mm = vma->vm_mm;
>> >  	unsigned long addr = start, unmapped = 0;
>> > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> >  
>> >  			folio_get(folio);
>> >  			spin_unlock(ptl);
>> > -			if (unlikely(!folio_trylock(folio)))
>> > +			if (unlikely(fault_folio != folio &&
>> > +				     !folio_trylock(folio)))
>> >  				return migrate_vma_collect_skip(start, end,
>> >  								walk);
>> >  			ret = split_folio(folio);
>> > -			folio_unlock(folio);
>> > +			if (fault_folio != folio)
>> > +				folio_unlock(folio);
>> >  			folio_put(folio);
>> >  			if (ret)
>> >  				return migrate_vma_collect_skip(start, end,
>> > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> >  		 * optimisation to avoid walking the rmap later with
>> >  		 * try_to_migrate().
>> >  		 */
>> > -		if (folio_trylock(folio)) {
>> > +		if (fault_folio == folio || folio_trylock(folio)) {
>> >  			bool anon_exclusive;
>> >  			pte_t swp_pte;
>> >  
>> > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> >  
>> >  				if (folio_try_share_anon_rmap_pte(folio, page)) {
>> >  					set_pte_at(mm, addr, ptep, pte);
>> > -					folio_unlock(folio);
>> > +					if (fault_folio != folio)
>> > +						folio_unlock(folio);
>> >  					folio_put(folio);
>> >  					mpfn = 0;
>> >  					goto next;
>> > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>> >  					  unsigned long npages,
>> >  					  struct page *fault_page)
>> >  {
>> > +	struct folio *fault_folio = fault_page ?
>> > +		page_folio(fault_page) : NULL;
>> >  	unsigned long i, restore = 0;
>> >  	bool allow_drain = true;
>> >  	unsigned long unmapped = 0;
>> > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>> >  		remove_migration_ptes(folio, folio, false);
>> >  
>> >  		src_pfns[i] = 0;
>> > -		folio_unlock(folio);
>> > +		if (fault_folio != folio)
>> > +			folio_unlock(folio);
>> >  		folio_put(folio);
>> >  		restore--;
>> >  	}
>> > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>> >  		return -EINVAL;
>> >  	if (args->fault_page && !is_device_private_page(args->fault_page))
>> >  		return -EINVAL;
>> > +	if (args->fault_page && !PageLocked(args->fault_page))
>> > +		return -EINVAL;
>> >  
>> >  	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>> >  	args->cpages = 0;
>> > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>> >  }
>> >  EXPORT_SYMBOL(migrate_vma_pages);
>> >  
>> > -/*
>> > - * migrate_device_finalize() - complete page migration
>> > - * @src_pfns: src_pfns returned from migrate_device_range()
>> > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>> > - * @npages: number of pages in the range
>> > - *
>> > - * Completes migration of the page by removing special migration entries.
>> > - * Drivers must ensure copying of page data is complete and visible to the CPU
>> > - * before calling this.
>> > - */
>> > -void migrate_device_finalize(unsigned long *src_pfns,
>> > -			unsigned long *dst_pfns, unsigned long npages)
>> > +static void __migrate_device_finalize(unsigned long *src_pfns,
>> > +				      unsigned long *dst_pfns,
>> > +				      unsigned long npages,
>> > +				      struct page *fault_page)
>> >  {
>> > +	struct folio *fault_folio = fault_page ?
>> > +		page_folio(fault_page) : NULL;
>> >  	unsigned long i;
>> >  
>> >  	for (i = 0; i < npages; i++) {
>> > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>> >  		src = page_folio(page);
>> >  		dst = page_folio(newpage);
>> >  		remove_migration_ptes(src, dst, false);
>> > -		folio_unlock(src);
>> > +		if (fault_folio != src)
>> > +			folio_unlock(src);
>> >  
>> >  		if (is_zone_device_page(page))
>> >  			put_page(page);
>> > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>> >  		}
>> >  	}
>> >  }
>> > +
>> > +/*
>> > + * migrate_device_finalize() - complete page migration
>> > + * @src_pfns: src_pfns returned from migrate_device_range()
>> > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>> > + * @npages: number of pages in the range
>> > + *
>> > + * Completes migration of the page by removing special migration entries.
>> > + * Drivers must ensure copying of page data is complete and visible to the CPU
>> > + * before calling this.
>> > + */
>> > +void migrate_device_finalize(unsigned long *src_pfns,
>> > +			unsigned long *dst_pfns, unsigned long npages)
>> > +{
>> > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
>> > +}
>> >  EXPORT_SYMBOL(migrate_device_finalize);
>> >  
>> >  /**
>> > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>> >   */
>> >  void migrate_vma_finalize(struct migrate_vma *migrate)
>> >  {
>> > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
>> > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
>> > +				  migrate->fault_page);
>> >  }
>> >  EXPORT_SYMBOL(migrate_vma_finalize);
>> 

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Felix Kuehling]

On 2024-09-18 11:10, Alistair Popple wrote:
> Matthew Brost <matthew.brost@intel.com> writes:
>
>> On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
>>> Matthew Brost <matthew.brost@intel.com> writes:
>>>
>>> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
>> Still a driver.
> Indeed, and I'm happy to answer any questions about our implementation.
>
>>> but theoretically it seems like it should be possible. However we
>>> serialize migrations of the same virtual address range to avoid these
>>> kind of issues as they can happen the other way too (ie. multiple
>>> threads trying to migrate to GPU).
>>>
>>> So I suspect what happens in UVM is that one thread wins and installs
>>> the migration entry while the others fail to get the driver migration
>>> lock and bail out sufficiently early in the fault path to avoid the
>>> live-lock.
>>>
>> I had to try hard to show this, doubt an actual user could trigger this.
>>
>> I wrote a test which kicked 8 threads, each thread did a pthread join,
>> and then tried to read the same page. This repeats in loop for like 512
>> pages or something. I needed an exclusive lock in migrate_to_ram vfunc
>> for it to livelock. Without an exclusive lock I think on average I saw
>> about 32k retries (i.e. migrate_to_ram calls on the same page) before a
>> thread won this race.
>>
>>  From reading UVM, pretty sure if you tried hard enough you could trigger
>> a livelock given it appears you take excluvise locks in migrate_to_ram.
> Yes, I suspect you're correct here and that we just haven't tried hard
> enough to trigger it.
>
>>>> Cc: Philip Yang <Philip.Yang@amd.com>
>>>> Cc: Felix Kuehling <felix.kuehling@amd.com>
>>>> Cc: Christian König <christian.koenig@amd.com>
>>>> Cc: Andrew Morton <akpm@linux-foundation.org>
>>>> Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
>>>> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>>>> ---
>>>>   mm/memory.c         | 13 +++++++---
>>>>   mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>>>>   2 files changed, 50 insertions(+), 23 deletions(-)
>>>>
>>>> diff --git a/mm/memory.c b/mm/memory.c
>>>> index 3c01d68065be..bbd97d16a96a 100644
>>>> --- a/mm/memory.c
>>>> +++ b/mm/memory.c
>>>> @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>>>>   			 * Get a page reference while we know the page can't be
>>>>   			 * freed.
>>>>   			 */
>>>> -			get_page(vmf->page);
>>>> -			pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>> -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>> -			put_page(vmf->page);
>>>> +			if (trylock_page(vmf->page)) {
>>>> +				get_page(vmf->page);
>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
>>> rather than do this and then have to pass all this context
>>> (ie. fault_page) down to the migrate_vma_* functions could we instead
>>> just do what migrate_vma_collect_pmd() does here? Ie. we already have
>>> the PTL and the page lock so there's no reason we couldn't just setup
>>> the migration entry prior to calling migrate_to_ram().
>>>
>>> Obviously calling migrate_vma_setup() would show the page as not
>>> migrating, but drivers could easily just fill in the src_pfn info after
>>> calling migrate_vma_setup().
>>>
>>> This would eliminate the whole fault_page ugliness.
>>>
>> This seems like it would work and agree it likely be cleaner. Let me
>> play around with this and see what I come up with. Multi-tasking a bit
>> so expect a bit of delay here.
>>
>> Thanks for the input,
>> Matt

Thanks! Sorry, I'm late catching up after a vacation. Please keep 
Philip, Christian and myself in the loop with future patches in this area.

Regards,
   Felix


>>
>>>> +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>> +				put_page(vmf->page);
>>>> +				unlock_page(vmf->page);
>>>> +			} else {
>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>> +			}
>>>>   		} else if (is_hwpoison_entry(entry)) {
>>>>   			ret = VM_FAULT_HWPOISON;
>>>>   		} else if (is_pte_marker_entry(entry)) {
>>>> diff --git a/mm/migrate_device.c b/mm/migrate_device.c
>>>> index 6d66dc1c6ffa..049893a5a179 100644
>>>> --- a/mm/migrate_device.c
>>>> +++ b/mm/migrate_device.c
>>>> @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>   				   struct mm_walk *walk)
>>>>   {
>>>>   	struct migrate_vma *migrate = walk->private;
>>>> +	struct folio *fault_folio = migrate->fault_page ?
>>>> +		page_folio(migrate->fault_page) : NULL;
>>>>   	struct vm_area_struct *vma = walk->vma;
>>>>   	struct mm_struct *mm = vma->vm_mm;
>>>>   	unsigned long addr = start, unmapped = 0;
>>>> @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>   
>>>>   			folio_get(folio);
>>>>   			spin_unlock(ptl);
>>>> -			if (unlikely(!folio_trylock(folio)))
>>>> +			if (unlikely(fault_folio != folio &&
>>>> +				     !folio_trylock(folio)))
>>>>   				return migrate_vma_collect_skip(start, end,
>>>>   								walk);
>>>>   			ret = split_folio(folio);
>>>> -			folio_unlock(folio);
>>>> +			if (fault_folio != folio)
>>>> +				folio_unlock(folio);
>>>>   			folio_put(folio);
>>>>   			if (ret)
>>>>   				return migrate_vma_collect_skip(start, end,
>>>> @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>   		 * optimisation to avoid walking the rmap later with
>>>>   		 * try_to_migrate().
>>>>   		 */
>>>> -		if (folio_trylock(folio)) {
>>>> +		if (fault_folio == folio || folio_trylock(folio)) {
>>>>   			bool anon_exclusive;
>>>>   			pte_t swp_pte;
>>>>   
>>>> @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>   
>>>>   				if (folio_try_share_anon_rmap_pte(folio, page)) {
>>>>   					set_pte_at(mm, addr, ptep, pte);
>>>> -					folio_unlock(folio);
>>>> +					if (fault_folio != folio)
>>>> +						folio_unlock(folio);
>>>>   					folio_put(folio);
>>>>   					mpfn = 0;
>>>>   					goto next;
>>>> @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>   					  unsigned long npages,
>>>>   					  struct page *fault_page)
>>>>   {
>>>> +	struct folio *fault_folio = fault_page ?
>>>> +		page_folio(fault_page) : NULL;
>>>>   	unsigned long i, restore = 0;
>>>>   	bool allow_drain = true;
>>>>   	unsigned long unmapped = 0;
>>>> @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>   		remove_migration_ptes(folio, folio, false);
>>>>   
>>>>   		src_pfns[i] = 0;
>>>> -		folio_unlock(folio);
>>>> +		if (fault_folio != folio)
>>>> +			folio_unlock(folio);
>>>>   		folio_put(folio);
>>>>   		restore--;
>>>>   	}
>>>> @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>>>>   		return -EINVAL;
>>>>   	if (args->fault_page && !is_device_private_page(args->fault_page))
>>>>   		return -EINVAL;
>>>> +	if (args->fault_page && !PageLocked(args->fault_page))
>>>> +		return -EINVAL;
>>>>   
>>>>   	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>>>>   	args->cpages = 0;
>>>> @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>>>>   }
>>>>   EXPORT_SYMBOL(migrate_vma_pages);
>>>>   
>>>> -/*
>>>> - * migrate_device_finalize() - complete page migration
>>>> - * @src_pfns: src_pfns returned from migrate_device_range()
>>>> - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>> - * @npages: number of pages in the range
>>>> - *
>>>> - * Completes migration of the page by removing special migration entries.
>>>> - * Drivers must ensure copying of page data is complete and visible to the CPU
>>>> - * before calling this.
>>>> - */
>>>> -void migrate_device_finalize(unsigned long *src_pfns,
>>>> -			unsigned long *dst_pfns, unsigned long npages)
>>>> +static void __migrate_device_finalize(unsigned long *src_pfns,
>>>> +				      unsigned long *dst_pfns,
>>>> +				      unsigned long npages,
>>>> +				      struct page *fault_page)
>>>>   {
>>>> +	struct folio *fault_folio = fault_page ?
>>>> +		page_folio(fault_page) : NULL;
>>>>   	unsigned long i;
>>>>   
>>>>   	for (i = 0; i < npages; i++) {
>>>> @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>   		src = page_folio(page);
>>>>   		dst = page_folio(newpage);
>>>>   		remove_migration_ptes(src, dst, false);
>>>> -		folio_unlock(src);
>>>> +		if (fault_folio != src)
>>>> +			folio_unlock(src);
>>>>   
>>>>   		if (is_zone_device_page(page))
>>>>   			put_page(page);
>>>> @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>   		}
>>>>   	}
>>>>   }
>>>> +
>>>> +/*
>>>> + * migrate_device_finalize() - complete page migration
>>>> + * @src_pfns: src_pfns returned from migrate_device_range()
>>>> + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>> + * @npages: number of pages in the range
>>>> + *
>>>> + * Completes migration of the page by removing special migration entries.
>>>> + * Drivers must ensure copying of page data is complete and visible to the CPU
>>>> + * before calling this.
>>>> + */
>>>> +void migrate_device_finalize(unsigned long *src_pfns,
>>>> +			unsigned long *dst_pfns, unsigned long npages)
>>>> +{
>>>> +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
>>>> +}
>>>>   EXPORT_SYMBOL(migrate_device_finalize);
>>>>   
>>>>   /**
>>>> @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>>>>    */
>>>>   void migrate_vma_finalize(struct migrate_vma *migrate)
>>>>   {
>>>> -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
>>>> +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
>>>> +				  migrate->fault_page);
>>>>   }
>>>>   EXPORT_SYMBOL(migrate_vma_finalize);

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> On 2024-09-18 11:10, Alistair Popple wrote:
> > Matthew Brost <matthew.brost@intel.com> writes:
> > 
> > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > 
> > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > Still a driver.
> > Indeed, and I'm happy to answer any questions about our implementation.
> > 
> > > > but theoretically it seems like it should be possible. However we
> > > > serialize migrations of the same virtual address range to avoid these
> > > > kind of issues as they can happen the other way too (ie. multiple
> > > > threads trying to migrate to GPU).
> > > > 
> > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > the migration entry while the others fail to get the driver migration
> > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > live-lock.
> > > > 
> > > I had to try hard to show this, doubt an actual user could trigger this.
> > > 
> > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > and then tried to read the same page. This repeats in loop for like 512
> > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > for it to livelock. Without an exclusive lock I think on average I saw
> > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > thread won this race.
> > > 
> > >  From reading UVM, pretty sure if you tried hard enough you could trigger
> > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > Yes, I suspect you're correct here and that we just haven't tried hard
> > enough to trigger it.
> > 
> > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > ---
> > > > >   mm/memory.c         | 13 +++++++---
> > > > >   mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > >   2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > 
> > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > --- a/mm/memory.c
> > > > > +++ b/mm/memory.c
> > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > >   			 * Get a page reference while we know the page can't be
> > > > >   			 * freed.
> > > > >   			 */
> > > > > -			get_page(vmf->page);
> > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > -			put_page(vmf->page);
> > > > > +			if (trylock_page(vmf->page)) {
> > > > > +				get_page(vmf->page);
> > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > rather than do this and then have to pass all this context
> > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > the migration entry prior to calling migrate_to_ram().
> > > > 
> > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > calling migrate_vma_setup().
> > > > 
> > > > This would eliminate the whole fault_page ugliness.
> > > > 
> > > This seems like it would work and agree it likely be cleaner. Let me
> > > play around with this and see what I come up with. Multi-tasking a bit
> > > so expect a bit of delay here.
> > > 
> > > Thanks for the input,
> > > Matt
> 
> Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> Christian and myself in the loop with future patches in this area.
> 

Will do. Already have another local patch set which helps drivers dma
map 2M pages for migrations if SRAM is physically contiguous. Seems
helpful for performance on Intel hardware. Probably post that soon for
early feedack.

Longer term I thinking 2M migration entries, 2M device pages, and being
able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
finding migrate_vma_* functions take up like 80-90% of the time in the
CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
ideal. Seems like 2M entries for everything would really help here. No
idea how feasible this is as the core MM stuff gets confusing fast. Any
input on this idea?

Matt

> Regards,
>   Felix
> 
> 
> > > 
> > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > +				put_page(vmf->page);
> > > > > +				unlock_page(vmf->page);
> > > > > +			} else {
> > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > +			}
> > > > >   		} else if (is_hwpoison_entry(entry)) {
> > > > >   			ret = VM_FAULT_HWPOISON;
> > > > >   		} else if (is_pte_marker_entry(entry)) {
> > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > --- a/mm/migrate_device.c
> > > > > +++ b/mm/migrate_device.c
> > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > >   				   struct mm_walk *walk)
> > > > >   {
> > > > >   	struct migrate_vma *migrate = walk->private;
> > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > >   	struct vm_area_struct *vma = walk->vma;
> > > > >   	struct mm_struct *mm = vma->vm_mm;
> > > > >   	unsigned long addr = start, unmapped = 0;
> > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > >   			folio_get(folio);
> > > > >   			spin_unlock(ptl);
> > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > +			if (unlikely(fault_folio != folio &&
> > > > > +				     !folio_trylock(folio)))
> > > > >   				return migrate_vma_collect_skip(start, end,
> > > > >   								walk);
> > > > >   			ret = split_folio(folio);
> > > > > -			folio_unlock(folio);
> > > > > +			if (fault_folio != folio)
> > > > > +				folio_unlock(folio);
> > > > >   			folio_put(folio);
> > > > >   			if (ret)
> > > > >   				return migrate_vma_collect_skip(start, end,
> > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > >   		 * optimisation to avoid walking the rmap later with
> > > > >   		 * try_to_migrate().
> > > > >   		 */
> > > > > -		if (folio_trylock(folio)) {
> > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > >   			bool anon_exclusive;
> > > > >   			pte_t swp_pte;
> > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > >   				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > >   					set_pte_at(mm, addr, ptep, pte);
> > > > > -					folio_unlock(folio);
> > > > > +					if (fault_folio != folio)
> > > > > +						folio_unlock(folio);
> > > > >   					folio_put(folio);
> > > > >   					mpfn = 0;
> > > > >   					goto next;
> > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > >   					  unsigned long npages,
> > > > >   					  struct page *fault_page)
> > > > >   {
> > > > > +	struct folio *fault_folio = fault_page ?
> > > > > +		page_folio(fault_page) : NULL;
> > > > >   	unsigned long i, restore = 0;
> > > > >   	bool allow_drain = true;
> > > > >   	unsigned long unmapped = 0;
> > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > >   		remove_migration_ptes(folio, folio, false);
> > > > >   		src_pfns[i] = 0;
> > > > > -		folio_unlock(folio);
> > > > > +		if (fault_folio != folio)
> > > > > +			folio_unlock(folio);
> > > > >   		folio_put(folio);
> > > > >   		restore--;
> > > > >   	}
> > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > >   		return -EINVAL;
> > > > >   	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > >   		return -EINVAL;
> > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > +		return -EINVAL;
> > > > >   	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > >   	args->cpages = 0;
> > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > >   }
> > > > >   EXPORT_SYMBOL(migrate_vma_pages);
> > > > > -/*
> > > > > - * migrate_device_finalize() - complete page migration
> > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > - * @npages: number of pages in the range
> > > > > - *
> > > > > - * Completes migration of the page by removing special migration entries.
> > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > - * before calling this.
> > > > > - */
> > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > +				      unsigned long *dst_pfns,
> > > > > +				      unsigned long npages,
> > > > > +				      struct page *fault_page)
> > > > >   {
> > > > > +	struct folio *fault_folio = fault_page ?
> > > > > +		page_folio(fault_page) : NULL;
> > > > >   	unsigned long i;
> > > > >   	for (i = 0; i < npages; i++) {
> > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > >   		src = page_folio(page);
> > > > >   		dst = page_folio(newpage);
> > > > >   		remove_migration_ptes(src, dst, false);
> > > > > -		folio_unlock(src);
> > > > > +		if (fault_folio != src)
> > > > > +			folio_unlock(src);
> > > > >   		if (is_zone_device_page(page))
> > > > >   			put_page(page);
> > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > >   		}
> > > > >   	}
> > > > >   }
> > > > > +
> > > > > +/*
> > > > > + * migrate_device_finalize() - complete page migration
> > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > + * @npages: number of pages in the range
> > > > > + *
> > > > > + * Completes migration of the page by removing special migration entries.
> > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > + * before calling this.
> > > > > + */
> > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > +{
> > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > +}
> > > > >   EXPORT_SYMBOL(migrate_device_finalize);
> > > > >   /**
> > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > >    */
> > > > >   void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > >   {
> > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > +				  migrate->fault_page);
> > > > >   }
> > > > >   EXPORT_SYMBOL(migrate_vma_finalize);

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Felix Kuehling]

On 2024-09-20 17:23, Matthew Brost wrote:
> On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
>> On 2024-09-18 11:10, Alistair Popple wrote:
>>> Matthew Brost <matthew.brost@intel.com> writes:
>>>
>>>> On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
>>>>> Matthew Brost <matthew.brost@intel.com> writes:
>>>>>
>>>>> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
>>>> Still a driver.
>>> Indeed, and I'm happy to answer any questions about our implementation.
>>>
>>>>> but theoretically it seems like it should be possible. However we
>>>>> serialize migrations of the same virtual address range to avoid these
>>>>> kind of issues as they can happen the other way too (ie. multiple
>>>>> threads trying to migrate to GPU).
>>>>>
>>>>> So I suspect what happens in UVM is that one thread wins and installs
>>>>> the migration entry while the others fail to get the driver migration
>>>>> lock and bail out sufficiently early in the fault path to avoid the
>>>>> live-lock.
>>>>>
>>>> I had to try hard to show this, doubt an actual user could trigger this.
>>>>
>>>> I wrote a test which kicked 8 threads, each thread did a pthread join,
>>>> and then tried to read the same page. This repeats in loop for like 512
>>>> pages or something. I needed an exclusive lock in migrate_to_ram vfunc
>>>> for it to livelock. Without an exclusive lock I think on average I saw
>>>> about 32k retries (i.e. migrate_to_ram calls on the same page) before a
>>>> thread won this race.
>>>>
>>>>   From reading UVM, pretty sure if you tried hard enough you could trigger
>>>> a livelock given it appears you take excluvise locks in migrate_to_ram.
>>> Yes, I suspect you're correct here and that we just haven't tried hard
>>> enough to trigger it.
>>>
>>>>>> Cc: Philip Yang <Philip.Yang@amd.com>
>>>>>> Cc: Felix Kuehling <felix.kuehling@amd.com>
>>>>>> Cc: Christian König <christian.koenig@amd.com>
>>>>>> Cc: Andrew Morton <akpm@linux-foundation.org>
>>>>>> Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
>>>>>> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>>>>>> ---
>>>>>>    mm/memory.c         | 13 +++++++---
>>>>>>    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>>>>>>    2 files changed, 50 insertions(+), 23 deletions(-)
>>>>>>
>>>>>> diff --git a/mm/memory.c b/mm/memory.c
>>>>>> index 3c01d68065be..bbd97d16a96a 100644
>>>>>> --- a/mm/memory.c
>>>>>> +++ b/mm/memory.c
>>>>>> @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>>>>>>    			 * Get a page reference while we know the page can't be
>>>>>>    			 * freed.
>>>>>>    			 */
>>>>>> -			get_page(vmf->page);
>>>>>> -			pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>>> -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>>>> -			put_page(vmf->page);
>>>>>> +			if (trylock_page(vmf->page)) {
>>>>>> +				get_page(vmf->page);
>>>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
>>>>> rather than do this and then have to pass all this context
>>>>> (ie. fault_page) down to the migrate_vma_* functions could we instead
>>>>> just do what migrate_vma_collect_pmd() does here? Ie. we already have
>>>>> the PTL and the page lock so there's no reason we couldn't just setup
>>>>> the migration entry prior to calling migrate_to_ram().
>>>>>
>>>>> Obviously calling migrate_vma_setup() would show the page as not
>>>>> migrating, but drivers could easily just fill in the src_pfn info after
>>>>> calling migrate_vma_setup().
>>>>>
>>>>> This would eliminate the whole fault_page ugliness.
>>>>>
>>>> This seems like it would work and agree it likely be cleaner. Let me
>>>> play around with this and see what I come up with. Multi-tasking a bit
>>>> so expect a bit of delay here.
>>>>
>>>> Thanks for the input,
>>>> Matt
>> Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
>> Christian and myself in the loop with future patches in this area.
>>
> Will do. Already have another local patch set which helps drivers dma
> map 2M pages for migrations if SRAM is physically contiguous. Seems
> helpful for performance on Intel hardware. Probably post that soon for
> early feedack.

OK.


>
> Longer term I thinking 2M migration entries, 2M device pages, and being
> able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> finding migrate_vma_* functions take up like 80-90% of the time in the
> CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> ideal. Seems like 2M entries for everything would really help here. No
> idea how feasible this is as the core MM stuff gets confusing fast. Any
> input on this idea?

I agree with your observations. We found that the migrate_vma_* code was 
the bottle neck for migration performance as well, and not breaking 2M 
pages could reduce that overhead a lot. I don't have any specific ideas. 
I'm not familiar with the details of that code myself. Philip has looked 
at this (and some old NVidia patches from a few years ago) in the past 
but never had enough uninterrupted time to make it past prototyping.

Regards,
   Felix


>
> Matt
>
>> Regards,
>>    Felix
>>
>>
>>>>>> +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>>>> +				put_page(vmf->page);
>>>>>> +				unlock_page(vmf->page);
>>>>>> +			} else {
>>>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>>> +			}
>>>>>>    		} else if (is_hwpoison_entry(entry)) {
>>>>>>    			ret = VM_FAULT_HWPOISON;
>>>>>>    		} else if (is_pte_marker_entry(entry)) {
>>>>>> diff --git a/mm/migrate_device.c b/mm/migrate_device.c
>>>>>> index 6d66dc1c6ffa..049893a5a179 100644
>>>>>> --- a/mm/migrate_device.c
>>>>>> +++ b/mm/migrate_device.c
>>>>>> @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>    				   struct mm_walk *walk)
>>>>>>    {
>>>>>>    	struct migrate_vma *migrate = walk->private;
>>>>>> +	struct folio *fault_folio = migrate->fault_page ?
>>>>>> +		page_folio(migrate->fault_page) : NULL;
>>>>>>    	struct vm_area_struct *vma = walk->vma;
>>>>>>    	struct mm_struct *mm = vma->vm_mm;
>>>>>>    	unsigned long addr = start, unmapped = 0;
>>>>>> @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>    			folio_get(folio);
>>>>>>    			spin_unlock(ptl);
>>>>>> -			if (unlikely(!folio_trylock(folio)))
>>>>>> +			if (unlikely(fault_folio != folio &&
>>>>>> +				     !folio_trylock(folio)))
>>>>>>    				return migrate_vma_collect_skip(start, end,
>>>>>>    								walk);
>>>>>>    			ret = split_folio(folio);
>>>>>> -			folio_unlock(folio);
>>>>>> +			if (fault_folio != folio)
>>>>>> +				folio_unlock(folio);
>>>>>>    			folio_put(folio);
>>>>>>    			if (ret)
>>>>>>    				return migrate_vma_collect_skip(start, end,
>>>>>> @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>    		 * optimisation to avoid walking the rmap later with
>>>>>>    		 * try_to_migrate().
>>>>>>    		 */
>>>>>> -		if (folio_trylock(folio)) {
>>>>>> +		if (fault_folio == folio || folio_trylock(folio)) {
>>>>>>    			bool anon_exclusive;
>>>>>>    			pte_t swp_pte;
>>>>>> @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>    				if (folio_try_share_anon_rmap_pte(folio, page)) {
>>>>>>    					set_pte_at(mm, addr, ptep, pte);
>>>>>> -					folio_unlock(folio);
>>>>>> +					if (fault_folio != folio)
>>>>>> +						folio_unlock(folio);
>>>>>>    					folio_put(folio);
>>>>>>    					mpfn = 0;
>>>>>>    					goto next;
>>>>>> @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>>>    					  unsigned long npages,
>>>>>>    					  struct page *fault_page)
>>>>>>    {
>>>>>> +	struct folio *fault_folio = fault_page ?
>>>>>> +		page_folio(fault_page) : NULL;
>>>>>>    	unsigned long i, restore = 0;
>>>>>>    	bool allow_drain = true;
>>>>>>    	unsigned long unmapped = 0;
>>>>>> @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>>>    		remove_migration_ptes(folio, folio, false);
>>>>>>    		src_pfns[i] = 0;
>>>>>> -		folio_unlock(folio);
>>>>>> +		if (fault_folio != folio)
>>>>>> +			folio_unlock(folio);
>>>>>>    		folio_put(folio);
>>>>>>    		restore--;
>>>>>>    	}
>>>>>> @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>>>>>>    		return -EINVAL;
>>>>>>    	if (args->fault_page && !is_device_private_page(args->fault_page))
>>>>>>    		return -EINVAL;
>>>>>> +	if (args->fault_page && !PageLocked(args->fault_page))
>>>>>> +		return -EINVAL;
>>>>>>    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>>>>>>    	args->cpages = 0;
>>>>>> @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>>>>>>    }
>>>>>>    EXPORT_SYMBOL(migrate_vma_pages);
>>>>>> -/*
>>>>>> - * migrate_device_finalize() - complete page migration
>>>>>> - * @src_pfns: src_pfns returned from migrate_device_range()
>>>>>> - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>>>> - * @npages: number of pages in the range
>>>>>> - *
>>>>>> - * Completes migration of the page by removing special migration entries.
>>>>>> - * Drivers must ensure copying of page data is complete and visible to the CPU
>>>>>> - * before calling this.
>>>>>> - */
>>>>>> -void migrate_device_finalize(unsigned long *src_pfns,
>>>>>> -			unsigned long *dst_pfns, unsigned long npages)
>>>>>> +static void __migrate_device_finalize(unsigned long *src_pfns,
>>>>>> +				      unsigned long *dst_pfns,
>>>>>> +				      unsigned long npages,
>>>>>> +				      struct page *fault_page)
>>>>>>    {
>>>>>> +	struct folio *fault_folio = fault_page ?
>>>>>> +		page_folio(fault_page) : NULL;
>>>>>>    	unsigned long i;
>>>>>>    	for (i = 0; i < npages; i++) {
>>>>>> @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>    		src = page_folio(page);
>>>>>>    		dst = page_folio(newpage);
>>>>>>    		remove_migration_ptes(src, dst, false);
>>>>>> -		folio_unlock(src);
>>>>>> +		if (fault_folio != src)
>>>>>> +			folio_unlock(src);
>>>>>>    		if (is_zone_device_page(page))
>>>>>>    			put_page(page);
>>>>>> @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>    		}
>>>>>>    	}
>>>>>>    }
>>>>>> +
>>>>>> +/*
>>>>>> + * migrate_device_finalize() - complete page migration
>>>>>> + * @src_pfns: src_pfns returned from migrate_device_range()
>>>>>> + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>>>> + * @npages: number of pages in the range
>>>>>> + *
>>>>>> + * Completes migration of the page by removing special migration entries.
>>>>>> + * Drivers must ensure copying of page data is complete and visible to the CPU
>>>>>> + * before calling this.
>>>>>> + */
>>>>>> +void migrate_device_finalize(unsigned long *src_pfns,
>>>>>> +			unsigned long *dst_pfns, unsigned long npages)
>>>>>> +{
>>>>>> +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
>>>>>> +}
>>>>>>    EXPORT_SYMBOL(migrate_device_finalize);
>>>>>>    /**
>>>>>> @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>>>>>>     */
>>>>>>    void migrate_vma_finalize(struct migrate_vma *migrate)
>>>>>>    {
>>>>>> -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
>>>>>> +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
>>>>>> +				  migrate->fault_page);
>>>>>>    }
>>>>>>    EXPORT_SYMBOL(migrate_vma_finalize);

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> 
> On 2024-09-20 17:23, Matthew Brost wrote:
> > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> > > On 2024-09-18 11:10, Alistair Popple wrote:
> > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > 
> > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > 
> > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > > > Still a driver.
> > > > Indeed, and I'm happy to answer any questions about our implementation.
> > > > 
> > > > > > but theoretically it seems like it should be possible. However we
> > > > > > serialize migrations of the same virtual address range to avoid these
> > > > > > kind of issues as they can happen the other way too (ie. multiple
> > > > > > threads trying to migrate to GPU).
> > > > > > 
> > > > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > > > the migration entry while the others fail to get the driver migration
> > > > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > > > live-lock.
> > > > > > 
> > > > > I had to try hard to show this, doubt an actual user could trigger this.
> > > > > 
> > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > > > and then tried to read the same page. This repeats in loop for like 512
> > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > > > for it to livelock. Without an exclusive lock I think on average I saw
> > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > > > thread won this race.
> > > > > 
> > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > > > Yes, I suspect you're correct here and that we just haven't tried hard
> > > > enough to trigger it.
> > > > 
> > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > > > ---
> > > > > > >    mm/memory.c         | 13 +++++++---
> > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > > > 
> > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > > > --- a/mm/memory.c
> > > > > > > +++ b/mm/memory.c
> > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > > > >    			 * Get a page reference while we know the page can't be
> > > > > > >    			 * freed.
> > > > > > >    			 */
> > > > > > > -			get_page(vmf->page);
> > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > -			put_page(vmf->page);
> > > > > > > +			if (trylock_page(vmf->page)) {
> > > > > > > +				get_page(vmf->page);
> > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > > > rather than do this and then have to pass all this context
> > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > > > the migration entry prior to calling migrate_to_ram().
> > > > > > 
> > > > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > > > calling migrate_vma_setup().
> > > > > > 
> > > > > > This would eliminate the whole fault_page ugliness.
> > > > > > 
> > > > > This seems like it would work and agree it likely be cleaner. Let me
> > > > > play around with this and see what I come up with. Multi-tasking a bit
> > > > > so expect a bit of delay here.
> > > > > 
> > > > > Thanks for the input,
> > > > > Matt
> > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> > > Christian and myself in the loop with future patches in this area.
> > > 
> > Will do. Already have another local patch set which helps drivers dma
> > map 2M pages for migrations if SRAM is physically contiguous. Seems
> > helpful for performance on Intel hardware. Probably post that soon for
> > early feedack.
> 
> OK.
> 
> 
> > 
> > Longer term I thinking 2M migration entries, 2M device pages, and being
> > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> > finding migrate_vma_* functions take up like 80-90% of the time in the
> > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> > ideal. Seems like 2M entries for everything would really help here. No
> > idea how feasible this is as the core MM stuff gets confusing fast. Any
> > input on this idea?
> 
> I agree with your observations. We found that the migrate_vma_* code was the
> bottle neck for migration performance as well, and not breaking 2M pages
> could reduce that overhead a lot. I don't have any specific ideas. I'm not
> familiar with the details of that code myself. Philip has looked at this
> (and some old NVidia patches from a few years ago) in the past but never had
> enough uninterrupted time to make it past prototyping.
> 

Cool good to know this isn't some crazy idea. Doubt it happen anytime
soon as I need to get a working baseline in before anything then start
applying optimizations and help in get other features to get the design
complete. But eventually will probably try to look at this. May ping
Philip and Nvidia when I have time to dig in here.

Matt

> Regards,
>   Felix
> 
> 
> > 
> > Matt
> > 
> > > Regards,
> > >    Felix
> > > 
> > > 
> > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > +				put_page(vmf->page);
> > > > > > > +				unlock_page(vmf->page);
> > > > > > > +			} else {
> > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > +			}
> > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> > > > > > >    			ret = VM_FAULT_HWPOISON;
> > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > > > --- a/mm/migrate_device.c
> > > > > > > +++ b/mm/migrate_device.c
> > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > >    				   struct mm_walk *walk)
> > > > > > >    {
> > > > > > >    	struct migrate_vma *migrate = walk->private;
> > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > > > >    	struct vm_area_struct *vma = walk->vma;
> > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> > > > > > >    	unsigned long addr = start, unmapped = 0;
> > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > >    			folio_get(folio);
> > > > > > >    			spin_unlock(ptl);
> > > > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > > > +			if (unlikely(fault_folio != folio &&
> > > > > > > +				     !folio_trylock(folio)))
> > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > >    								walk);
> > > > > > >    			ret = split_folio(folio);
> > > > > > > -			folio_unlock(folio);
> > > > > > > +			if (fault_folio != folio)
> > > > > > > +				folio_unlock(folio);
> > > > > > >    			folio_put(folio);
> > > > > > >    			if (ret)
> > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > >    		 * optimisation to avoid walking the rmap later with
> > > > > > >    		 * try_to_migrate().
> > > > > > >    		 */
> > > > > > > -		if (folio_trylock(folio)) {
> > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > > > >    			bool anon_exclusive;
> > > > > > >    			pte_t swp_pte;
> > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> > > > > > > -					folio_unlock(folio);
> > > > > > > +					if (fault_folio != folio)
> > > > > > > +						folio_unlock(folio);
> > > > > > >    					folio_put(folio);
> > > > > > >    					mpfn = 0;
> > > > > > >    					goto next;
> > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > >    					  unsigned long npages,
> > > > > > >    					  struct page *fault_page)
> > > > > > >    {
> > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > >    	unsigned long i, restore = 0;
> > > > > > >    	bool allow_drain = true;
> > > > > > >    	unsigned long unmapped = 0;
> > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > >    		remove_migration_ptes(folio, folio, false);
> > > > > > >    		src_pfns[i] = 0;
> > > > > > > -		folio_unlock(folio);
> > > > > > > +		if (fault_folio != folio)
> > > > > > > +			folio_unlock(folio);
> > > > > > >    		folio_put(folio);
> > > > > > >    		restore--;
> > > > > > >    	}
> > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > > > >    		return -EINVAL;
> > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > > > >    		return -EINVAL;
> > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > > > +		return -EINVAL;
> > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > > > >    	args->cpages = 0;
> > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > > > >    }
> > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> > > > > > > -/*
> > > > > > > - * migrate_device_finalize() - complete page migration
> > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > - * @npages: number of pages in the range
> > > > > > > - *
> > > > > > > - * Completes migration of the page by removing special migration entries.
> > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > - * before calling this.
> > > > > > > - */
> > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > +				      unsigned long *dst_pfns,
> > > > > > > +				      unsigned long npages,
> > > > > > > +				      struct page *fault_page)
> > > > > > >    {
> > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > >    	unsigned long i;
> > > > > > >    	for (i = 0; i < npages; i++) {
> > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > >    		src = page_folio(page);
> > > > > > >    		dst = page_folio(newpage);
> > > > > > >    		remove_migration_ptes(src, dst, false);
> > > > > > > -		folio_unlock(src);
> > > > > > > +		if (fault_folio != src)
> > > > > > > +			folio_unlock(src);
> > > > > > >    		if (is_zone_device_page(page))
> > > > > > >    			put_page(page);
> > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > >    		}
> > > > > > >    	}
> > > > > > >    }
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * migrate_device_finalize() - complete page migration
> > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > + * @npages: number of pages in the range
> > > > > > > + *
> > > > > > > + * Completes migration of the page by removing special migration entries.
> > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > + * before calling this.
> > > > > > > + */
> > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > +{
> > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > > > +}
> > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > >    /**
> > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > >     */
> > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > > > >    {
> > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > > > +				  migrate->fault_page);
> > > > > > >    }
> > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Simona Vetter]

On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
> On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> > 
> > On 2024-09-20 17:23, Matthew Brost wrote:
> > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> > > > On 2024-09-18 11:10, Alistair Popple wrote:
> > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > 
> > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > 
> > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > > > > Still a driver.
> > > > > Indeed, and I'm happy to answer any questions about our implementation.
> > > > > 
> > > > > > > but theoretically it seems like it should be possible. However we
> > > > > > > serialize migrations of the same virtual address range to avoid these
> > > > > > > kind of issues as they can happen the other way too (ie. multiple
> > > > > > > threads trying to migrate to GPU).
> > > > > > > 
> > > > > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > > > > the migration entry while the others fail to get the driver migration
> > > > > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > > > > live-lock.
> > > > > > > 
> > > > > > I had to try hard to show this, doubt an actual user could trigger this.
> > > > > > 
> > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > > > > and then tried to read the same page. This repeats in loop for like 512
> > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > > > > for it to livelock. Without an exclusive lock I think on average I saw
> > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > > > > thread won this race.
> > > > > > 
> > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > > > > Yes, I suspect you're correct here and that we just haven't tried hard
> > > > > enough to trigger it.
> > > > > 
> > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > > > > ---
> > > > > > > >    mm/memory.c         | 13 +++++++---
> > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > > > > 
> > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > > > > --- a/mm/memory.c
> > > > > > > > +++ b/mm/memory.c
> > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > > > > >    			 * Get a page reference while we know the page can't be
> > > > > > > >    			 * freed.
> > > > > > > >    			 */
> > > > > > > > -			get_page(vmf->page);
> > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > -			put_page(vmf->page);
> > > > > > > > +			if (trylock_page(vmf->page)) {
> > > > > > > > +				get_page(vmf->page);
> > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > > > > rather than do this and then have to pass all this context
> > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > > > > the migration entry prior to calling migrate_to_ram().
> > > > > > > 
> > > > > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > > > > calling migrate_vma_setup().
> > > > > > > 
> > > > > > > This would eliminate the whole fault_page ugliness.
> > > > > > > 
> > > > > > This seems like it would work and agree it likely be cleaner. Let me
> > > > > > play around with this and see what I come up with. Multi-tasking a bit
> > > > > > so expect a bit of delay here.
> > > > > > 
> > > > > > Thanks for the input,
> > > > > > Matt
> > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> > > > Christian and myself in the loop with future patches in this area.
> > > > 
> > > Will do. Already have another local patch set which helps drivers dma
> > > map 2M pages for migrations if SRAM is physically contiguous. Seems
> > > helpful for performance on Intel hardware. Probably post that soon for
> > > early feedack.
> > 
> > OK.
> > 
> > 
> > > 
> > > Longer term I thinking 2M migration entries, 2M device pages, and being
> > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> > > finding migrate_vma_* functions take up like 80-90% of the time in the
> > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> > > ideal. Seems like 2M entries for everything would really help here. No
> > > idea how feasible this is as the core MM stuff gets confusing fast. Any
> > > input on this idea?
> > 
> > I agree with your observations. We found that the migrate_vma_* code was the
> > bottle neck for migration performance as well, and not breaking 2M pages
> > could reduce that overhead a lot. I don't have any specific ideas. I'm not
> > familiar with the details of that code myself. Philip has looked at this
> > (and some old NVidia patches from a few years ago) in the past but never had
> > enough uninterrupted time to make it past prototyping.
> > 
> 
> Cool good to know this isn't some crazy idea. Doubt it happen anytime
> soon as I need to get a working baseline in before anything then start
> applying optimizations and help in get other features to get the design
> complete. But eventually will probably try to look at this. May ping
> Philip and Nvidia when I have time to dig in here.

I think the big step will be moving hmm.c and migrate.c apis over from
struct page to folios. That should also give us some nice benefits on the
gpu side, since instead of 4k pages to track we could allocate 2m gpu
pages.

Once we have folios at the driver/core mm api level doing all the fancy
thp stuff should be at least a well-contained problem. But I might be
dellusionally optimistic here :-)
-Sima

> 
> Matt
> 
> > Regards,
> >   Felix
> > 
> > 
> > > 
> > > Matt
> > > 
> > > > Regards,
> > > >    Felix
> > > > 
> > > > 
> > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > +				put_page(vmf->page);
> > > > > > > > +				unlock_page(vmf->page);
> > > > > > > > +			} else {
> > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > +			}
> > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> > > > > > > >    			ret = VM_FAULT_HWPOISON;
> > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > > > > --- a/mm/migrate_device.c
> > > > > > > > +++ b/mm/migrate_device.c
> > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > >    				   struct mm_walk *walk)
> > > > > > > >    {
> > > > > > > >    	struct migrate_vma *migrate = walk->private;
> > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > > > > >    	struct vm_area_struct *vma = walk->vma;
> > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> > > > > > > >    	unsigned long addr = start, unmapped = 0;
> > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > >    			folio_get(folio);
> > > > > > > >    			spin_unlock(ptl);
> > > > > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > > > > +			if (unlikely(fault_folio != folio &&
> > > > > > > > +				     !folio_trylock(folio)))
> > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > >    								walk);
> > > > > > > >    			ret = split_folio(folio);
> > > > > > > > -			folio_unlock(folio);
> > > > > > > > +			if (fault_folio != folio)
> > > > > > > > +				folio_unlock(folio);
> > > > > > > >    			folio_put(folio);
> > > > > > > >    			if (ret)
> > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > >    		 * optimisation to avoid walking the rmap later with
> > > > > > > >    		 * try_to_migrate().
> > > > > > > >    		 */
> > > > > > > > -		if (folio_trylock(folio)) {
> > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > > > > >    			bool anon_exclusive;
> > > > > > > >    			pte_t swp_pte;
> > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> > > > > > > > -					folio_unlock(folio);
> > > > > > > > +					if (fault_folio != folio)
> > > > > > > > +						folio_unlock(folio);
> > > > > > > >    					folio_put(folio);
> > > > > > > >    					mpfn = 0;
> > > > > > > >    					goto next;
> > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > >    					  unsigned long npages,
> > > > > > > >    					  struct page *fault_page)
> > > > > > > >    {
> > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > >    	unsigned long i, restore = 0;
> > > > > > > >    	bool allow_drain = true;
> > > > > > > >    	unsigned long unmapped = 0;
> > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > >    		remove_migration_ptes(folio, folio, false);
> > > > > > > >    		src_pfns[i] = 0;
> > > > > > > > -		folio_unlock(folio);
> > > > > > > > +		if (fault_folio != folio)
> > > > > > > > +			folio_unlock(folio);
> > > > > > > >    		folio_put(folio);
> > > > > > > >    		restore--;
> > > > > > > >    	}
> > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > > > > >    		return -EINVAL;
> > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > > > > >    		return -EINVAL;
> > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > > > > +		return -EINVAL;
> > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > > > > >    	args->cpages = 0;
> > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > > > > >    }
> > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> > > > > > > > -/*
> > > > > > > > - * migrate_device_finalize() - complete page migration
> > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > - * @npages: number of pages in the range
> > > > > > > > - *
> > > > > > > > - * Completes migration of the page by removing special migration entries.
> > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > - * before calling this.
> > > > > > > > - */
> > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > +				      unsigned long *dst_pfns,
> > > > > > > > +				      unsigned long npages,
> > > > > > > > +				      struct page *fault_page)
> > > > > > > >    {
> > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > >    	unsigned long i;
> > > > > > > >    	for (i = 0; i < npages; i++) {
> > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > >    		src = page_folio(page);
> > > > > > > >    		dst = page_folio(newpage);
> > > > > > > >    		remove_migration_ptes(src, dst, false);
> > > > > > > > -		folio_unlock(src);
> > > > > > > > +		if (fault_folio != src)
> > > > > > > > +			folio_unlock(src);
> > > > > > > >    		if (is_zone_device_page(page))
> > > > > > > >    			put_page(page);
> > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > >    		}
> > > > > > > >    	}
> > > > > > > >    }
> > > > > > > > +
> > > > > > > > +/*
> > > > > > > > + * migrate_device_finalize() - complete page migration
> > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > + * @npages: number of pages in the range
> > > > > > > > + *
> > > > > > > > + * Completes migration of the page by removing special migration entries.
> > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > + * before calling this.
> > > > > > > > + */
> > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > +{
> > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > > > > +}
> > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > >    /**
> > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > >     */
> > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > > > > >    {
> > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > > > > +				  migrate->fault_page);
> > > > > > > >    }
> > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);

-- 
Simona Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
> On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
> > On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> > > 
> > > On 2024-09-20 17:23, Matthew Brost wrote:
> > > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> > > > > On 2024-09-18 11:10, Alistair Popple wrote:
> > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > 
> > > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > > 
> > > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > > > > > Still a driver.
> > > > > > Indeed, and I'm happy to answer any questions about our implementation.
> > > > > > 
> > > > > > > > but theoretically it seems like it should be possible. However we
> > > > > > > > serialize migrations of the same virtual address range to avoid these
> > > > > > > > kind of issues as they can happen the other way too (ie. multiple
> > > > > > > > threads trying to migrate to GPU).
> > > > > > > > 
> > > > > > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > > > > > the migration entry while the others fail to get the driver migration
> > > > > > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > > > > > live-lock.
> > > > > > > > 
> > > > > > > I had to try hard to show this, doubt an actual user could trigger this.
> > > > > > > 
> > > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > > > > > and then tried to read the same page. This repeats in loop for like 512
> > > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > > > > > for it to livelock. Without an exclusive lock I think on average I saw
> > > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > > > > > thread won this race.
> > > > > > > 
> > > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> > > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > > > > > Yes, I suspect you're correct here and that we just haven't tried hard
> > > > > > enough to trigger it.
> > > > > > 
> > > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > > > > > ---
> > > > > > > > >    mm/memory.c         | 13 +++++++---
> > > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > > > > > --- a/mm/memory.c
> > > > > > > > > +++ b/mm/memory.c
> > > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > > > > > >    			 * Get a page reference while we know the page can't be
> > > > > > > > >    			 * freed.
> > > > > > > > >    			 */
> > > > > > > > > -			get_page(vmf->page);
> > > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > -			put_page(vmf->page);
> > > > > > > > > +			if (trylock_page(vmf->page)) {
> > > > > > > > > +				get_page(vmf->page);
> > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > > > > > rather than do this and then have to pass all this context
> > > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > > > > > the migration entry prior to calling migrate_to_ram().
> > > > > > > > 
> > > > > > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > > > > > calling migrate_vma_setup().
> > > > > > > > 
> > > > > > > > This would eliminate the whole fault_page ugliness.
> > > > > > > > 
> > > > > > > This seems like it would work and agree it likely be cleaner. Let me
> > > > > > > play around with this and see what I come up with. Multi-tasking a bit
> > > > > > > so expect a bit of delay here.
> > > > > > > 
> > > > > > > Thanks for the input,
> > > > > > > Matt
> > > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> > > > > Christian and myself in the loop with future patches in this area.
> > > > > 
> > > > Will do. Already have another local patch set which helps drivers dma
> > > > map 2M pages for migrations if SRAM is physically contiguous. Seems
> > > > helpful for performance on Intel hardware. Probably post that soon for
> > > > early feedack.
> > > 
> > > OK.
> > > 
> > > 
> > > > 
> > > > Longer term I thinking 2M migration entries, 2M device pages, and being
> > > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> > > > finding migrate_vma_* functions take up like 80-90% of the time in the
> > > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> > > > ideal. Seems like 2M entries for everything would really help here. No
> > > > idea how feasible this is as the core MM stuff gets confusing fast. Any
> > > > input on this idea?
> > > 
> > > I agree with your observations. We found that the migrate_vma_* code was the
> > > bottle neck for migration performance as well, and not breaking 2M pages
> > > could reduce that overhead a lot. I don't have any specific ideas. I'm not
> > > familiar with the details of that code myself. Philip has looked at this
> > > (and some old NVidia patches from a few years ago) in the past but never had
> > > enough uninterrupted time to make it past prototyping.
> > > 
> > 
> > Cool good to know this isn't some crazy idea. Doubt it happen anytime
> > soon as I need to get a working baseline in before anything then start
> > applying optimizations and help in get other features to get the design
> > complete. But eventually will probably try to look at this. May ping
> > Philip and Nvidia when I have time to dig in here.
> 
> I think the big step will be moving hmm.c and migrate.c apis over from
> struct page to folios. That should also give us some nice benefits on the
> gpu side, since instead of 4k pages to track we could allocate 2m gpu
> pages.
> 

I think was thinking just encode the order in the migration PFN like HMM
does. Really only Nth order entry in the page array needs to be
populated then - HMM populates every entry though which doesn't seem
like that is required. Maybe having a folio API makes more sense?

> Once we have folios at the driver/core mm api level doing all the fancy
> thp stuff should be at least a well-contained problem. But I might be
> dellusionally optimistic here :-)

I think it contained in the sense is the DRM SVM layer just allocates a
THP or large continous device memory and hands it off to migrate layer
and that layer does the right thing. The 'right thing' here I believe is
a decent amount of core MM work though.

Matt

> -Sima
> 
> > 
> > Matt
> > 
> > > Regards,
> > >   Felix
> > > 
> > > 
> > > > 
> > > > Matt
> > > > 
> > > > > Regards,
> > > > >    Felix
> > > > > 
> > > > > 
> > > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > +				put_page(vmf->page);
> > > > > > > > > +				unlock_page(vmf->page);
> > > > > > > > > +			} else {
> > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > +			}
> > > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> > > > > > > > >    			ret = VM_FAULT_HWPOISON;
> > > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> > > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > > > > > --- a/mm/migrate_device.c
> > > > > > > > > +++ b/mm/migrate_device.c
> > > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > >    				   struct mm_walk *walk)
> > > > > > > > >    {
> > > > > > > > >    	struct migrate_vma *migrate = walk->private;
> > > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > > > > > >    	struct vm_area_struct *vma = walk->vma;
> > > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> > > > > > > > >    	unsigned long addr = start, unmapped = 0;
> > > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > >    			folio_get(folio);
> > > > > > > > >    			spin_unlock(ptl);
> > > > > > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > > > > > +			if (unlikely(fault_folio != folio &&
> > > > > > > > > +				     !folio_trylock(folio)))
> > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > >    								walk);
> > > > > > > > >    			ret = split_folio(folio);
> > > > > > > > > -			folio_unlock(folio);
> > > > > > > > > +			if (fault_folio != folio)
> > > > > > > > > +				folio_unlock(folio);
> > > > > > > > >    			folio_put(folio);
> > > > > > > > >    			if (ret)
> > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > >    		 * optimisation to avoid walking the rmap later with
> > > > > > > > >    		 * try_to_migrate().
> > > > > > > > >    		 */
> > > > > > > > > -		if (folio_trylock(folio)) {
> > > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > > > > > >    			bool anon_exclusive;
> > > > > > > > >    			pte_t swp_pte;
> > > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> > > > > > > > > -					folio_unlock(folio);
> > > > > > > > > +					if (fault_folio != folio)
> > > > > > > > > +						folio_unlock(folio);
> > > > > > > > >    					folio_put(folio);
> > > > > > > > >    					mpfn = 0;
> > > > > > > > >    					goto next;
> > > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > >    					  unsigned long npages,
> > > > > > > > >    					  struct page *fault_page)
> > > > > > > > >    {
> > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > >    	unsigned long i, restore = 0;
> > > > > > > > >    	bool allow_drain = true;
> > > > > > > > >    	unsigned long unmapped = 0;
> > > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > >    		remove_migration_ptes(folio, folio, false);
> > > > > > > > >    		src_pfns[i] = 0;
> > > > > > > > > -		folio_unlock(folio);
> > > > > > > > > +		if (fault_folio != folio)
> > > > > > > > > +			folio_unlock(folio);
> > > > > > > > >    		folio_put(folio);
> > > > > > > > >    		restore--;
> > > > > > > > >    	}
> > > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > > > > > >    		return -EINVAL;
> > > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > > > > > >    		return -EINVAL;
> > > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > > > > > +		return -EINVAL;
> > > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > > > > > >    	args->cpages = 0;
> > > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > > > > > >    }
> > > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> > > > > > > > > -/*
> > > > > > > > > - * migrate_device_finalize() - complete page migration
> > > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > - * @npages: number of pages in the range
> > > > > > > > > - *
> > > > > > > > > - * Completes migration of the page by removing special migration entries.
> > > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > - * before calling this.
> > > > > > > > > - */
> > > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > +				      unsigned long *dst_pfns,
> > > > > > > > > +				      unsigned long npages,
> > > > > > > > > +				      struct page *fault_page)
> > > > > > > > >    {
> > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > >    	unsigned long i;
> > > > > > > > >    	for (i = 0; i < npages; i++) {
> > > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > >    		src = page_folio(page);
> > > > > > > > >    		dst = page_folio(newpage);
> > > > > > > > >    		remove_migration_ptes(src, dst, false);
> > > > > > > > > -		folio_unlock(src);
> > > > > > > > > +		if (fault_folio != src)
> > > > > > > > > +			folio_unlock(src);
> > > > > > > > >    		if (is_zone_device_page(page))
> > > > > > > > >    			put_page(page);
> > > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > >    		}
> > > > > > > > >    	}
> > > > > > > > >    }
> > > > > > > > > +
> > > > > > > > > +/*
> > > > > > > > > + * migrate_device_finalize() - complete page migration
> > > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > + * @npages: number of pages in the range
> > > > > > > > > + *
> > > > > > > > > + * Completes migration of the page by removing special migration entries.
> > > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > + * before calling this.
> > > > > > > > > + */
> > > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > +{
> > > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > > > > > +}
> > > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > >    /**
> > > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > >     */
> > > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > > > > > >    {
> > > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > > > > > +				  migrate->fault_page);
> > > > > > > > >    }
> > > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);
> 
> -- 
> Simona Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Felix Kuehling]

On 2024-09-24 12:42, Matthew Brost wrote:
> On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
>> On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
>>> On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
>>>>
>>>> On 2024-09-20 17:23, Matthew Brost wrote:
>>>>> On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
>>>>>> On 2024-09-18 11:10, Alistair Popple wrote:
>>>>>>> Matthew Brost <matthew.brost@intel.com> writes:
>>>>>>>
>>>>>>>> On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
>>>>>>>>> Matthew Brost <matthew.brost@intel.com> writes:
>>>>>>>>>
>>>>>>>>> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
>>>>>>>> Still a driver.
>>>>>>> Indeed, and I'm happy to answer any questions about our implementation.
>>>>>>>
>>>>>>>>> but theoretically it seems like it should be possible. However we
>>>>>>>>> serialize migrations of the same virtual address range to avoid these
>>>>>>>>> kind of issues as they can happen the other way too (ie. multiple
>>>>>>>>> threads trying to migrate to GPU).
>>>>>>>>>
>>>>>>>>> So I suspect what happens in UVM is that one thread wins and installs
>>>>>>>>> the migration entry while the others fail to get the driver migration
>>>>>>>>> lock and bail out sufficiently early in the fault path to avoid the
>>>>>>>>> live-lock.
>>>>>>>>>
>>>>>>>> I had to try hard to show this, doubt an actual user could trigger this.
>>>>>>>>
>>>>>>>> I wrote a test which kicked 8 threads, each thread did a pthread join,
>>>>>>>> and then tried to read the same page. This repeats in loop for like 512
>>>>>>>> pages or something. I needed an exclusive lock in migrate_to_ram vfunc
>>>>>>>> for it to livelock. Without an exclusive lock I think on average I saw
>>>>>>>> about 32k retries (i.e. migrate_to_ram calls on the same page) before a
>>>>>>>> thread won this race.
>>>>>>>>
>>>>>>>>   From reading UVM, pretty sure if you tried hard enough you could trigger
>>>>>>>> a livelock given it appears you take excluvise locks in migrate_to_ram.
>>>>>>> Yes, I suspect you're correct here and that we just haven't tried hard
>>>>>>> enough to trigger it.
>>>>>>>
>>>>>>>>>> Cc: Philip Yang <Philip.Yang@amd.com>
>>>>>>>>>> Cc: Felix Kuehling <felix.kuehling@amd.com>
>>>>>>>>>> Cc: Christian König <christian.koenig@amd.com>
>>>>>>>>>> Cc: Andrew Morton <akpm@linux-foundation.org>
>>>>>>>>>> Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
>>>>>>>>>> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>>>>>>>>>> ---
>>>>>>>>>>    mm/memory.c         | 13 +++++++---
>>>>>>>>>>    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>>>>>>>>>>    2 files changed, 50 insertions(+), 23 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/mm/memory.c b/mm/memory.c
>>>>>>>>>> index 3c01d68065be..bbd97d16a96a 100644
>>>>>>>>>> --- a/mm/memory.c
>>>>>>>>>> +++ b/mm/memory.c
>>>>>>>>>> @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>>>>>>>>>>    			 * Get a page reference while we know the page can't be
>>>>>>>>>>    			 * freed.
>>>>>>>>>>    			 */
>>>>>>>>>> -			get_page(vmf->page);
>>>>>>>>>> -			pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>>>>>>> -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>>>>>>>> -			put_page(vmf->page);
>>>>>>>>>> +			if (trylock_page(vmf->page)) {
>>>>>>>>>> +				get_page(vmf->page);
>>>>>>>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>>>>>> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
>>>>>>>>> rather than do this and then have to pass all this context
>>>>>>>>> (ie. fault_page) down to the migrate_vma_* functions could we instead
>>>>>>>>> just do what migrate_vma_collect_pmd() does here? Ie. we already have
>>>>>>>>> the PTL and the page lock so there's no reason we couldn't just setup
>>>>>>>>> the migration entry prior to calling migrate_to_ram().
>>>>>>>>>
>>>>>>>>> Obviously calling migrate_vma_setup() would show the page as not
>>>>>>>>> migrating, but drivers could easily just fill in the src_pfn info after
>>>>>>>>> calling migrate_vma_setup().
>>>>>>>>>
>>>>>>>>> This would eliminate the whole fault_page ugliness.
>>>>>>>>>
>>>>>>>> This seems like it would work and agree it likely be cleaner. Let me
>>>>>>>> play around with this and see what I come up with. Multi-tasking a bit
>>>>>>>> so expect a bit of delay here.
>>>>>>>>
>>>>>>>> Thanks for the input,
>>>>>>>> Matt
>>>>>> Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
>>>>>> Christian and myself in the loop with future patches in this area.
>>>>>>
>>>>> Will do. Already have another local patch set which helps drivers dma
>>>>> map 2M pages for migrations if SRAM is physically contiguous. Seems
>>>>> helpful for performance on Intel hardware. Probably post that soon for
>>>>> early feedack.
>>>>
>>>> OK.
>>>>
>>>>
>>>>>
>>>>> Longer term I thinking 2M migration entries, 2M device pages, and being
>>>>> able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
>>>>> finding migrate_vma_* functions take up like 80-90% of the time in the
>>>>> CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
>>>>> ideal. Seems like 2M entries for everything would really help here. No
>>>>> idea how feasible this is as the core MM stuff gets confusing fast. Any
>>>>> input on this idea?
>>>>
>>>> I agree with your observations. We found that the migrate_vma_* code was the
>>>> bottle neck for migration performance as well, and not breaking 2M pages
>>>> could reduce that overhead a lot. I don't have any specific ideas. I'm not
>>>> familiar with the details of that code myself. Philip has looked at this
>>>> (and some old NVidia patches from a few years ago) in the past but never had
>>>> enough uninterrupted time to make it past prototyping.
>>>>
>>>
>>> Cool good to know this isn't some crazy idea. Doubt it happen anytime
>>> soon as I need to get a working baseline in before anything then start
>>> applying optimizations and help in get other features to get the design
>>> complete. But eventually will probably try to look at this. May ping
>>> Philip and Nvidia when I have time to dig in here.
>>
>> I think the big step will be moving hmm.c and migrate.c apis over from
>> struct page to folios. That should also give us some nice benefits on the
>> gpu side, since instead of 4k pages to track we could allocate 2m gpu
>> pages.
>>
> 
> I think was thinking just encode the order in the migration PFN like HMM
> does. Really only Nth order entry in the page array needs to be
> populated then - HMM populates every entry though which doesn't seem
> like that is required. Maybe having a folio API makes more sense?
> 
>> Once we have folios at the driver/core mm api level doing all the fancy
>> thp stuff should be at least a well-contained problem. But I might be
>> dellusionally optimistic here :-)
> 
> I think it contained in the sense is the DRM SVM layer just allocates a
> THP or large continous device memory and hands it off to migrate layer
> and that layer does the right thing. The 'right thing' here I believe is
> a decent amount of core MM work though.

Whether or not you allocate contiguous device memory doesn't need to depend on the page size in system memory. You can migrate 2MB of scattered system memory into a contiguous 2MB block of VRAM. The AMD driver does this already. The problem (as I see it) is all the extra book-keeping with 4KB granularity in the migrate_vma_* helpers when the source data is actually contiguous.

Regards,
  Felix

> 
> Matt
> 
>> -Sima
>>
>>>
>>> Matt
>>>
>>>> Regards,
>>>>   Felix
>>>>
>>>>
>>>>>
>>>>> Matt
>>>>>
>>>>>> Regards,
>>>>>>    Felix
>>>>>>
>>>>>>
>>>>>>>>>> +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>>>>>>>>>> +				put_page(vmf->page);
>>>>>>>>>> +				unlock_page(vmf->page);
>>>>>>>>>> +			} else {
>>>>>>>>>> +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>>>>>>>>>> +			}
>>>>>>>>>>    		} else if (is_hwpoison_entry(entry)) {
>>>>>>>>>>    			ret = VM_FAULT_HWPOISON;
>>>>>>>>>>    		} else if (is_pte_marker_entry(entry)) {
>>>>>>>>>> diff --git a/mm/migrate_device.c b/mm/migrate_device.c
>>>>>>>>>> index 6d66dc1c6ffa..049893a5a179 100644
>>>>>>>>>> --- a/mm/migrate_device.c
>>>>>>>>>> +++ b/mm/migrate_device.c
>>>>>>>>>> @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>>>>>    				   struct mm_walk *walk)
>>>>>>>>>>    {
>>>>>>>>>>    	struct migrate_vma *migrate = walk->private;
>>>>>>>>>> +	struct folio *fault_folio = migrate->fault_page ?
>>>>>>>>>> +		page_folio(migrate->fault_page) : NULL;
>>>>>>>>>>    	struct vm_area_struct *vma = walk->vma;
>>>>>>>>>>    	struct mm_struct *mm = vma->vm_mm;
>>>>>>>>>>    	unsigned long addr = start, unmapped = 0;
>>>>>>>>>> @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>>>>>    			folio_get(folio);
>>>>>>>>>>    			spin_unlock(ptl);
>>>>>>>>>> -			if (unlikely(!folio_trylock(folio)))
>>>>>>>>>> +			if (unlikely(fault_folio != folio &&
>>>>>>>>>> +				     !folio_trylock(folio)))
>>>>>>>>>>    				return migrate_vma_collect_skip(start, end,
>>>>>>>>>>    								walk);
>>>>>>>>>>    			ret = split_folio(folio);
>>>>>>>>>> -			folio_unlock(folio);
>>>>>>>>>> +			if (fault_folio != folio)
>>>>>>>>>> +				folio_unlock(folio);
>>>>>>>>>>    			folio_put(folio);
>>>>>>>>>>    			if (ret)
>>>>>>>>>>    				return migrate_vma_collect_skip(start, end,
>>>>>>>>>> @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>>>>>    		 * optimisation to avoid walking the rmap later with
>>>>>>>>>>    		 * try_to_migrate().
>>>>>>>>>>    		 */
>>>>>>>>>> -		if (folio_trylock(folio)) {
>>>>>>>>>> +		if (fault_folio == folio || folio_trylock(folio)) {
>>>>>>>>>>    			bool anon_exclusive;
>>>>>>>>>>    			pte_t swp_pte;
>>>>>>>>>> @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>>>>>>>>>>    				if (folio_try_share_anon_rmap_pte(folio, page)) {
>>>>>>>>>>    					set_pte_at(mm, addr, ptep, pte);
>>>>>>>>>> -					folio_unlock(folio);
>>>>>>>>>> +					if (fault_folio != folio)
>>>>>>>>>> +						folio_unlock(folio);
>>>>>>>>>>    					folio_put(folio);
>>>>>>>>>>    					mpfn = 0;
>>>>>>>>>>    					goto next;
>>>>>>>>>> @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>>>>>>>    					  unsigned long npages,
>>>>>>>>>>    					  struct page *fault_page)
>>>>>>>>>>    {
>>>>>>>>>> +	struct folio *fault_folio = fault_page ?
>>>>>>>>>> +		page_folio(fault_page) : NULL;
>>>>>>>>>>    	unsigned long i, restore = 0;
>>>>>>>>>>    	bool allow_drain = true;
>>>>>>>>>>    	unsigned long unmapped = 0;
>>>>>>>>>> @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>>>>>>>>>>    		remove_migration_ptes(folio, folio, false);
>>>>>>>>>>    		src_pfns[i] = 0;
>>>>>>>>>> -		folio_unlock(folio);
>>>>>>>>>> +		if (fault_folio != folio)
>>>>>>>>>> +			folio_unlock(folio);
>>>>>>>>>>    		folio_put(folio);
>>>>>>>>>>    		restore--;
>>>>>>>>>>    	}
>>>>>>>>>> @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>>>>>>>>>>    		return -EINVAL;
>>>>>>>>>>    	if (args->fault_page && !is_device_private_page(args->fault_page))
>>>>>>>>>>    		return -EINVAL;
>>>>>>>>>> +	if (args->fault_page && !PageLocked(args->fault_page))
>>>>>>>>>> +		return -EINVAL;
>>>>>>>>>>    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>>>>>>>>>>    	args->cpages = 0;
>>>>>>>>>> @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>>>>>>>>>>    }
>>>>>>>>>>    EXPORT_SYMBOL(migrate_vma_pages);
>>>>>>>>>> -/*
>>>>>>>>>> - * migrate_device_finalize() - complete page migration
>>>>>>>>>> - * @src_pfns: src_pfns returned from migrate_device_range()
>>>>>>>>>> - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>>>>>>>> - * @npages: number of pages in the range
>>>>>>>>>> - *
>>>>>>>>>> - * Completes migration of the page by removing special migration entries.
>>>>>>>>>> - * Drivers must ensure copying of page data is complete and visible to the CPU
>>>>>>>>>> - * before calling this.
>>>>>>>>>> - */
>>>>>>>>>> -void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>>>>> -			unsigned long *dst_pfns, unsigned long npages)
>>>>>>>>>> +static void __migrate_device_finalize(unsigned long *src_pfns,
>>>>>>>>>> +				      unsigned long *dst_pfns,
>>>>>>>>>> +				      unsigned long npages,
>>>>>>>>>> +				      struct page *fault_page)
>>>>>>>>>>    {
>>>>>>>>>> +	struct folio *fault_folio = fault_page ?
>>>>>>>>>> +		page_folio(fault_page) : NULL;
>>>>>>>>>>    	unsigned long i;
>>>>>>>>>>    	for (i = 0; i < npages; i++) {
>>>>>>>>>> @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>>>>>    		src = page_folio(page);
>>>>>>>>>>    		dst = page_folio(newpage);
>>>>>>>>>>    		remove_migration_ptes(src, dst, false);
>>>>>>>>>> -		folio_unlock(src);
>>>>>>>>>> +		if (fault_folio != src)
>>>>>>>>>> +			folio_unlock(src);
>>>>>>>>>>    		if (is_zone_device_page(page))
>>>>>>>>>>    			put_page(page);
>>>>>>>>>> @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>>>>>    		}
>>>>>>>>>>    	}
>>>>>>>>>>    }
>>>>>>>>>> +
>>>>>>>>>> +/*
>>>>>>>>>> + * migrate_device_finalize() - complete page migration
>>>>>>>>>> + * @src_pfns: src_pfns returned from migrate_device_range()
>>>>>>>>>> + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>>>>>>>>>> + * @npages: number of pages in the range
>>>>>>>>>> + *
>>>>>>>>>> + * Completes migration of the page by removing special migration entries.
>>>>>>>>>> + * Drivers must ensure copying of page data is complete and visible to the CPU
>>>>>>>>>> + * before calling this.
>>>>>>>>>> + */
>>>>>>>>>> +void migrate_device_finalize(unsigned long *src_pfns,
>>>>>>>>>> +			unsigned long *dst_pfns, unsigned long npages)
>>>>>>>>>> +{
>>>>>>>>>> +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
>>>>>>>>>> +}
>>>>>>>>>>    EXPORT_SYMBOL(migrate_device_finalize);
>>>>>>>>>>    /**
>>>>>>>>>> @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>>>>>>>>>>     */
>>>>>>>>>>    void migrate_vma_finalize(struct migrate_vma *migrate)
>>>>>>>>>>    {
>>>>>>>>>> -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
>>>>>>>>>> +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
>>>>>>>>>> +				  migrate->fault_page);
>>>>>>>>>>    }
>>>>>>>>>>    EXPORT_SYMBOL(migrate_vma_finalize);
>>
>> -- 
>> Simona Vetter
>> Software Engineer, Intel Corporation
>> http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Simona Vetter]

On Tue, Sep 24, 2024 at 04:42:19PM +0000, Matthew Brost wrote:
> On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
> > On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
> > > On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> > > > 
> > > > On 2024-09-20 17:23, Matthew Brost wrote:
> > > > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> > > > > > On 2024-09-18 11:10, Alistair Popple wrote:
> > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > 
> > > > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > > > 
> > > > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > > > > > > Still a driver.
> > > > > > > Indeed, and I'm happy to answer any questions about our implementation.
> > > > > > > 
> > > > > > > > > but theoretically it seems like it should be possible. However we
> > > > > > > > > serialize migrations of the same virtual address range to avoid these
> > > > > > > > > kind of issues as they can happen the other way too (ie. multiple
> > > > > > > > > threads trying to migrate to GPU).
> > > > > > > > > 
> > > > > > > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > > > > > > the migration entry while the others fail to get the driver migration
> > > > > > > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > > > > > > live-lock.
> > > > > > > > > 
> > > > > > > > I had to try hard to show this, doubt an actual user could trigger this.
> > > > > > > > 
> > > > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > > > > > > and then tried to read the same page. This repeats in loop for like 512
> > > > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > > > > > > for it to livelock. Without an exclusive lock I think on average I saw
> > > > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > > > > > > thread won this race.
> > > > > > > > 
> > > > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> > > > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > > > > > > Yes, I suspect you're correct here and that we just haven't tried hard
> > > > > > > enough to trigger it.
> > > > > > > 
> > > > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > > > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > > > > > > ---
> > > > > > > > > >    mm/memory.c         | 13 +++++++---
> > > > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > > > > > > 
> > > > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > > > > > > --- a/mm/memory.c
> > > > > > > > > > +++ b/mm/memory.c
> > > > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > > > > > > >    			 * Get a page reference while we know the page can't be
> > > > > > > > > >    			 * freed.
> > > > > > > > > >    			 */
> > > > > > > > > > -			get_page(vmf->page);
> > > > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > > -			put_page(vmf->page);
> > > > > > > > > > +			if (trylock_page(vmf->page)) {
> > > > > > > > > > +				get_page(vmf->page);
> > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > > > > > > rather than do this and then have to pass all this context
> > > > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > > > > > > the migration entry prior to calling migrate_to_ram().
> > > > > > > > > 
> > > > > > > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > > > > > > calling migrate_vma_setup().
> > > > > > > > > 
> > > > > > > > > This would eliminate the whole fault_page ugliness.
> > > > > > > > > 
> > > > > > > > This seems like it would work and agree it likely be cleaner. Let me
> > > > > > > > play around with this and see what I come up with. Multi-tasking a bit
> > > > > > > > so expect a bit of delay here.
> > > > > > > > 
> > > > > > > > Thanks for the input,
> > > > > > > > Matt
> > > > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> > > > > > Christian and myself in the loop with future patches in this area.
> > > > > > 
> > > > > Will do. Already have another local patch set which helps drivers dma
> > > > > map 2M pages for migrations if SRAM is physically contiguous. Seems
> > > > > helpful for performance on Intel hardware. Probably post that soon for
> > > > > early feedack.
> > > > 
> > > > OK.
> > > > 
> > > > 
> > > > > 
> > > > > Longer term I thinking 2M migration entries, 2M device pages, and being
> > > > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> > > > > finding migrate_vma_* functions take up like 80-90% of the time in the
> > > > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> > > > > ideal. Seems like 2M entries for everything would really help here. No
> > > > > idea how feasible this is as the core MM stuff gets confusing fast. Any
> > > > > input on this idea?
> > > > 
> > > > I agree with your observations. We found that the migrate_vma_* code was the
> > > > bottle neck for migration performance as well, and not breaking 2M pages
> > > > could reduce that overhead a lot. I don't have any specific ideas. I'm not
> > > > familiar with the details of that code myself. Philip has looked at this
> > > > (and some old NVidia patches from a few years ago) in the past but never had
> > > > enough uninterrupted time to make it past prototyping.
> > > > 
> > > 
> > > Cool good to know this isn't some crazy idea. Doubt it happen anytime
> > > soon as I need to get a working baseline in before anything then start
> > > applying optimizations and help in get other features to get the design
> > > complete. But eventually will probably try to look at this. May ping
> > > Philip and Nvidia when I have time to dig in here.
> > 
> > I think the big step will be moving hmm.c and migrate.c apis over from
> > struct page to folios. That should also give us some nice benefits on the
> > gpu side, since instead of 4k pages to track we could allocate 2m gpu
> > pages.
> > 
> 
> I think was thinking just encode the order in the migration PFN like HMM
> does. Really only Nth order entry in the page array needs to be
> populated then - HMM populates every entry though which doesn't seem
> like that is required. Maybe having a folio API makes more sense?

Both I'd say, as a first attempt at least. An array of folios, but only
populate the ones we need and jump over empty entries. A bit wasteful, but
eh it's just allocations.

> > Once we have folios at the driver/core mm api level doing all the fancy
> > thp stuff should be at least a well-contained problem. But I might be
> > dellusionally optimistic here :-)
> 
> I think it contained in the sense is the DRM SVM layer just allocates a
> THP or large continous device memory and hands it off to migrate layer
> and that layer does the right thing. The 'right thing' here I believe is
> a decent amount of core MM work though.

Yeah that's what I meant, once we have switched the interfaces to be
arrays of folios, where for larger folios we leave the entries in between
NULL and have some appropriate iterators, then the driver side is done
mostly.  The core mm side to actually make use of that will be fairly
gnarly though.
-Sima

> 
> Matt
> 
> > -Sima
> > 
> > > 
> > > Matt
> > > 
> > > > Regards,
> > > >   Felix
> > > > 
> > > > 
> > > > > 
> > > > > Matt
> > > > > 
> > > > > > Regards,
> > > > > >    Felix
> > > > > > 
> > > > > > 
> > > > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > > +				put_page(vmf->page);
> > > > > > > > > > +				unlock_page(vmf->page);
> > > > > > > > > > +			} else {
> > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > > +			}
> > > > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> > > > > > > > > >    			ret = VM_FAULT_HWPOISON;
> > > > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> > > > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > > > > > > --- a/mm/migrate_device.c
> > > > > > > > > > +++ b/mm/migrate_device.c
> > > > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > >    				   struct mm_walk *walk)
> > > > > > > > > >    {
> > > > > > > > > >    	struct migrate_vma *migrate = walk->private;
> > > > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > > > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > > > > > > >    	struct vm_area_struct *vma = walk->vma;
> > > > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> > > > > > > > > >    	unsigned long addr = start, unmapped = 0;
> > > > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > >    			folio_get(folio);
> > > > > > > > > >    			spin_unlock(ptl);
> > > > > > > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > > > > > > +			if (unlikely(fault_folio != folio &&
> > > > > > > > > > +				     !folio_trylock(folio)))
> > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > > >    								walk);
> > > > > > > > > >    			ret = split_folio(folio);
> > > > > > > > > > -			folio_unlock(folio);
> > > > > > > > > > +			if (fault_folio != folio)
> > > > > > > > > > +				folio_unlock(folio);
> > > > > > > > > >    			folio_put(folio);
> > > > > > > > > >    			if (ret)
> > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > >    		 * optimisation to avoid walking the rmap later with
> > > > > > > > > >    		 * try_to_migrate().
> > > > > > > > > >    		 */
> > > > > > > > > > -		if (folio_trylock(folio)) {
> > > > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > > > > > > >    			bool anon_exclusive;
> > > > > > > > > >    			pte_t swp_pte;
> > > > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> > > > > > > > > > -					folio_unlock(folio);
> > > > > > > > > > +					if (fault_folio != folio)
> > > > > > > > > > +						folio_unlock(folio);
> > > > > > > > > >    					folio_put(folio);
> > > > > > > > > >    					mpfn = 0;
> > > > > > > > > >    					goto next;
> > > > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > > >    					  unsigned long npages,
> > > > > > > > > >    					  struct page *fault_page)
> > > > > > > > > >    {
> > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > > >    	unsigned long i, restore = 0;
> > > > > > > > > >    	bool allow_drain = true;
> > > > > > > > > >    	unsigned long unmapped = 0;
> > > > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > > >    		remove_migration_ptes(folio, folio, false);
> > > > > > > > > >    		src_pfns[i] = 0;
> > > > > > > > > > -		folio_unlock(folio);
> > > > > > > > > > +		if (fault_folio != folio)
> > > > > > > > > > +			folio_unlock(folio);
> > > > > > > > > >    		folio_put(folio);
> > > > > > > > > >    		restore--;
> > > > > > > > > >    	}
> > > > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > > > > > > >    		return -EINVAL;
> > > > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > > > > > > >    		return -EINVAL;
> > > > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > > > > > > +		return -EINVAL;
> > > > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > > > > > > >    	args->cpages = 0;
> > > > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > > > > > > >    }
> > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> > > > > > > > > > -/*
> > > > > > > > > > - * migrate_device_finalize() - complete page migration
> > > > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > > - * @npages: number of pages in the range
> > > > > > > > > > - *
> > > > > > > > > > - * Completes migration of the page by removing special migration entries.
> > > > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > > - * before calling this.
> > > > > > > > > > - */
> > > > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > +				      unsigned long *dst_pfns,
> > > > > > > > > > +				      unsigned long npages,
> > > > > > > > > > +				      struct page *fault_page)
> > > > > > > > > >    {
> > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > > >    	unsigned long i;
> > > > > > > > > >    	for (i = 0; i < npages; i++) {
> > > > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > >    		src = page_folio(page);
> > > > > > > > > >    		dst = page_folio(newpage);
> > > > > > > > > >    		remove_migration_ptes(src, dst, false);
> > > > > > > > > > -		folio_unlock(src);
> > > > > > > > > > +		if (fault_folio != src)
> > > > > > > > > > +			folio_unlock(src);
> > > > > > > > > >    		if (is_zone_device_page(page))
> > > > > > > > > >    			put_page(page);
> > > > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > >    		}
> > > > > > > > > >    	}
> > > > > > > > > >    }
> > > > > > > > > > +
> > > > > > > > > > +/*
> > > > > > > > > > + * migrate_device_finalize() - complete page migration
> > > > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > > + * @npages: number of pages in the range
> > > > > > > > > > + *
> > > > > > > > > > + * Completes migration of the page by removing special migration entries.
> > > > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > > + * before calling this.
> > > > > > > > > > + */
> > > > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > > +{
> > > > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > > > > > > +}
> > > > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > > >    /**
> > > > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > > >     */
> > > > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > > > > > > >    {
> > > > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > > > > > > +				  migrate->fault_page);
> > > > > > > > > >    }
> > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);
> > 
> > -- 
> > Simona Vetter
> > Software Engineer, Intel Corporation
> > http://blog.ffwll.ch

-- 
Simona Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Simona Vetter]

On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> 
> Matthew Brost <matthew.brost@intel.com> writes:
> 
> > Avoid multiple CPU page faults to the same device page racing by locking
> > the page in do_swap_page before taking an additional reference to the
> > page. This prevents scenarios where multiple CPU page faults each take
> > an extra reference to a device page, which could abort migration in
> > folio_migrate_mapping. With the device page locked in do_swap_page, the
> > migrate_vma_* functions need to be updated to avoid locking the
> > fault_page argument.
> 
> I added the get_page() and therefore the fault_page argument to deal
> with another lifetime issue. Neither Felix nor I particularly liked the
> solution though (see
> https://lore.kernel.org/all/cover.60659b549d8509ddecafad4f498ee7f03bb23c69.1664366292.git-series.apopple@nvidia.com/T/#m715589d57716448386ff9af41da27a952d94615a)
> and this seems to make it even uglier, so I have suggested an
> alternative solution below.

Just an aside, I don't fine the fault_page special handling all that
offensive, because fundamentally fault_page _is_ special: It's the one
page we _really_ have to migrate or userspace blows up because the kernel
sucks.

But maybe we shold stuff it into the vmf struct instead and pass that
around? I think that would be conceptually a notch cleaner.
-Sima

> 
> > Prior to this change, a livelock scenario could occur in Xe's (Intel GPU
> > DRM driver) SVM implementation if enough threads faulted the same device
> > page.
> 
> I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> but theoretically it seems like it should be possible. However we
> serialize migrations of the same virtual address range to avoid these
> kind of issues as they can happen the other way too (ie. multiple
> threads trying to migrate to GPU).
> 
> So I suspect what happens in UVM is that one thread wins and installs
> the migration entry while the others fail to get the driver migration
> lock and bail out sufficiently early in the fault path to avoid the
> live-lock.
> 
> > Cc: Philip Yang <Philip.Yang@amd.com>
> > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > Cc: Christian König <christian.koenig@amd.com>
> > Cc: Andrew Morton <akpm@linux-foundation.org>
> > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > ---
> >  mm/memory.c         | 13 +++++++---
> >  mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> >  2 files changed, 50 insertions(+), 23 deletions(-)
> >
> > diff --git a/mm/memory.c b/mm/memory.c
> > index 3c01d68065be..bbd97d16a96a 100644
> > --- a/mm/memory.c
> > +++ b/mm/memory.c
> > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> >  			 * Get a page reference while we know the page can't be
> >  			 * freed.
> >  			 */
> > -			get_page(vmf->page);
> > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > -			put_page(vmf->page);
> > +			if (trylock_page(vmf->page)) {
> > +				get_page(vmf->page);
> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> 
> This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> rather than do this and then have to pass all this context
> (ie. fault_page) down to the migrate_vma_* functions could we instead
> just do what migrate_vma_collect_pmd() does here? Ie. we already have
> the PTL and the page lock so there's no reason we couldn't just setup
> the migration entry prior to calling migrate_to_ram().
> 
> Obviously calling migrate_vma_setup() would show the page as not
> migrating, but drivers could easily just fill in the src_pfn info after
> calling migrate_vma_setup().
> 
> This would eliminate the whole fault_page ugliness.
> 
> > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > +				put_page(vmf->page);
> > +				unlock_page(vmf->page);
> > +			} else {
> > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > +			}
> >  		} else if (is_hwpoison_entry(entry)) {
> >  			ret = VM_FAULT_HWPOISON;
> >  		} else if (is_pte_marker_entry(entry)) {
> > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > index 6d66dc1c6ffa..049893a5a179 100644
> > --- a/mm/migrate_device.c
> > +++ b/mm/migrate_device.c
> > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  				   struct mm_walk *walk)
> >  {
> >  	struct migrate_vma *migrate = walk->private;
> > +	struct folio *fault_folio = migrate->fault_page ?
> > +		page_folio(migrate->fault_page) : NULL;
> >  	struct vm_area_struct *vma = walk->vma;
> >  	struct mm_struct *mm = vma->vm_mm;
> >  	unsigned long addr = start, unmapped = 0;
> > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  
> >  			folio_get(folio);
> >  			spin_unlock(ptl);
> > -			if (unlikely(!folio_trylock(folio)))
> > +			if (unlikely(fault_folio != folio &&
> > +				     !folio_trylock(folio)))
> >  				return migrate_vma_collect_skip(start, end,
> >  								walk);
> >  			ret = split_folio(folio);
> > -			folio_unlock(folio);
> > +			if (fault_folio != folio)
> > +				folio_unlock(folio);
> >  			folio_put(folio);
> >  			if (ret)
> >  				return migrate_vma_collect_skip(start, end,
> > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  		 * optimisation to avoid walking the rmap later with
> >  		 * try_to_migrate().
> >  		 */
> > -		if (folio_trylock(folio)) {
> > +		if (fault_folio == folio || folio_trylock(folio)) {
> >  			bool anon_exclusive;
> >  			pte_t swp_pte;
> >  
> > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >  
> >  				if (folio_try_share_anon_rmap_pte(folio, page)) {
> >  					set_pte_at(mm, addr, ptep, pte);
> > -					folio_unlock(folio);
> > +					if (fault_folio != folio)
> > +						folio_unlock(folio);
> >  					folio_put(folio);
> >  					mpfn = 0;
> >  					goto next;
> > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >  					  unsigned long npages,
> >  					  struct page *fault_page)
> >  {
> > +	struct folio *fault_folio = fault_page ?
> > +		page_folio(fault_page) : NULL;
> >  	unsigned long i, restore = 0;
> >  	bool allow_drain = true;
> >  	unsigned long unmapped = 0;
> > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >  		remove_migration_ptes(folio, folio, false);
> >  
> >  		src_pfns[i] = 0;
> > -		folio_unlock(folio);
> > +		if (fault_folio != folio)
> > +			folio_unlock(folio);
> >  		folio_put(folio);
> >  		restore--;
> >  	}
> > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> >  		return -EINVAL;
> >  	if (args->fault_page && !is_device_private_page(args->fault_page))
> >  		return -EINVAL;
> > +	if (args->fault_page && !PageLocked(args->fault_page))
> > +		return -EINVAL;
> >  
> >  	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> >  	args->cpages = 0;
> > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> >  }
> >  EXPORT_SYMBOL(migrate_vma_pages);
> >  
> > -/*
> > - * migrate_device_finalize() - complete page migration
> > - * @src_pfns: src_pfns returned from migrate_device_range()
> > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > - * @npages: number of pages in the range
> > - *
> > - * Completes migration of the page by removing special migration entries.
> > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > - * before calling this.
> > - */
> > -void migrate_device_finalize(unsigned long *src_pfns,
> > -			unsigned long *dst_pfns, unsigned long npages)
> > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > +				      unsigned long *dst_pfns,
> > +				      unsigned long npages,
> > +				      struct page *fault_page)
> >  {
> > +	struct folio *fault_folio = fault_page ?
> > +		page_folio(fault_page) : NULL;
> >  	unsigned long i;
> >  
> >  	for (i = 0; i < npages; i++) {
> > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >  		src = page_folio(page);
> >  		dst = page_folio(newpage);
> >  		remove_migration_ptes(src, dst, false);
> > -		folio_unlock(src);
> > +		if (fault_folio != src)
> > +			folio_unlock(src);
> >  
> >  		if (is_zone_device_page(page))
> >  			put_page(page);
> > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >  		}
> >  	}
> >  }
> > +
> > +/*
> > + * migrate_device_finalize() - complete page migration
> > + * @src_pfns: src_pfns returned from migrate_device_range()
> > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > + * @npages: number of pages in the range
> > + *
> > + * Completes migration of the page by removing special migration entries.
> > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > + * before calling this.
> > + */
> > +void migrate_device_finalize(unsigned long *src_pfns,
> > +			unsigned long *dst_pfns, unsigned long npages)
> > +{
> > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > +}
> >  EXPORT_SYMBOL(migrate_device_finalize);
> >  
> >  /**
> > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> >   */
> >  void migrate_vma_finalize(struct migrate_vma *migrate)
> >  {
> > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > +				  migrate->fault_page);
> >  }
> >  EXPORT_SYMBOL(migrate_vma_finalize);
> 

-- 
Simona Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Simona Vetter]

On Wed, Sep 25, 2024 at 01:44:37PM +0200, Simona Vetter wrote:
> On Tue, Sep 24, 2024 at 04:42:19PM +0000, Matthew Brost wrote:
> > On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
> > > On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
> > > > On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> > > > > 
> > > > > On 2024-09-20 17:23, Matthew Brost wrote:
> > > > > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> > > > > > > On 2024-09-18 11:10, Alistair Popple wrote:
> > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > > 
> > > > > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> > > > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> > > > > > > > > > 
> > > > > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> > > > > > > > > Still a driver.
> > > > > > > > Indeed, and I'm happy to answer any questions about our implementation.
> > > > > > > > 
> > > > > > > > > > but theoretically it seems like it should be possible. However we
> > > > > > > > > > serialize migrations of the same virtual address range to avoid these
> > > > > > > > > > kind of issues as they can happen the other way too (ie. multiple
> > > > > > > > > > threads trying to migrate to GPU).
> > > > > > > > > > 
> > > > > > > > > > So I suspect what happens in UVM is that one thread wins and installs
> > > > > > > > > > the migration entry while the others fail to get the driver migration
> > > > > > > > > > lock and bail out sufficiently early in the fault path to avoid the
> > > > > > > > > > live-lock.
> > > > > > > > > > 
> > > > > > > > > I had to try hard to show this, doubt an actual user could trigger this.
> > > > > > > > > 
> > > > > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> > > > > > > > > and then tried to read the same page. This repeats in loop for like 512
> > > > > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> > > > > > > > > for it to livelock. Without an exclusive lock I think on average I saw
> > > > > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> > > > > > > > > thread won this race.
> > > > > > > > > 
> > > > > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> > > > > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> > > > > > > > Yes, I suspect you're correct here and that we just haven't tried hard
> > > > > > > > enough to trigger it.
> > > > > > > > 
> > > > > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> > > > > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> > > > > > > > > > > Cc: Christian König <christian.koenig@amd.com>
> > > > > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> > > > > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> > > > > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> > > > > > > > > > > ---
> > > > > > > > > > >    mm/memory.c         | 13 +++++++---
> > > > > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> > > > > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> > > > > > > > > > > 
> > > > > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> > > > > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> > > > > > > > > > > --- a/mm/memory.c
> > > > > > > > > > > +++ b/mm/memory.c
> > > > > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> > > > > > > > > > >    			 * Get a page reference while we know the page can't be
> > > > > > > > > > >    			 * freed.
> > > > > > > > > > >    			 */
> > > > > > > > > > > -			get_page(vmf->page);
> > > > > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > > > -			put_page(vmf->page);
> > > > > > > > > > > +			if (trylock_page(vmf->page)) {
> > > > > > > > > > > +				get_page(vmf->page);
> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> > > > > > > > > > rather than do this and then have to pass all this context
> > > > > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> > > > > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> > > > > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> > > > > > > > > > the migration entry prior to calling migrate_to_ram().
> > > > > > > > > > 
> > > > > > > > > > Obviously calling migrate_vma_setup() would show the page as not
> > > > > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> > > > > > > > > > calling migrate_vma_setup().
> > > > > > > > > > 
> > > > > > > > > > This would eliminate the whole fault_page ugliness.
> > > > > > > > > > 
> > > > > > > > > This seems like it would work and agree it likely be cleaner. Let me
> > > > > > > > > play around with this and see what I come up with. Multi-tasking a bit
> > > > > > > > > so expect a bit of delay here.
> > > > > > > > > 
> > > > > > > > > Thanks for the input,
> > > > > > > > > Matt
> > > > > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> > > > > > > Christian and myself in the loop with future patches in this area.
> > > > > > > 
> > > > > > Will do. Already have another local patch set which helps drivers dma
> > > > > > map 2M pages for migrations if SRAM is physically contiguous. Seems
> > > > > > helpful for performance on Intel hardware. Probably post that soon for
> > > > > > early feedack.
> > > > > 
> > > > > OK.
> > > > > 
> > > > > 
> > > > > > 
> > > > > > Longer term I thinking 2M migration entries, 2M device pages, and being
> > > > > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> > > > > > finding migrate_vma_* functions take up like 80-90% of the time in the
> > > > > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> > > > > > ideal. Seems like 2M entries for everything would really help here. No
> > > > > > idea how feasible this is as the core MM stuff gets confusing fast. Any
> > > > > > input on this idea?
> > > > > 
> > > > > I agree with your observations. We found that the migrate_vma_* code was the
> > > > > bottle neck for migration performance as well, and not breaking 2M pages
> > > > > could reduce that overhead a lot. I don't have any specific ideas. I'm not
> > > > > familiar with the details of that code myself. Philip has looked at this
> > > > > (and some old NVidia patches from a few years ago) in the past but never had
> > > > > enough uninterrupted time to make it past prototyping.
> > > > > 
> > > > 
> > > > Cool good to know this isn't some crazy idea. Doubt it happen anytime
> > > > soon as I need to get a working baseline in before anything then start
> > > > applying optimizations and help in get other features to get the design
> > > > complete. But eventually will probably try to look at this. May ping
> > > > Philip and Nvidia when I have time to dig in here.
> > > 
> > > I think the big step will be moving hmm.c and migrate.c apis over from
> > > struct page to folios. That should also give us some nice benefits on the
> > > gpu side, since instead of 4k pages to track we could allocate 2m gpu
> > > pages.
> > > 
> > 
> > I think was thinking just encode the order in the migration PFN like HMM
> > does. Really only Nth order entry in the page array needs to be
> > populated then - HMM populates every entry though which doesn't seem
> > like that is required. Maybe having a folio API makes more sense?
> 
> Both I'd say, as a first attempt at least. An array of folios, but only
> populate the ones we need and jump over empty entries. A bit wasteful, but
> eh it's just allocations.

Ok thought some more, I think there's two things going on:

- spot contig memory sections so that the gpu side is more efficient and
  can user bigger pagetables and stuff like that. this is what
  hmm_range_fault does.

- optimize the core mm book-keeping by working on folios instead of
  individual pages. hmm_range_fault does not care because it doesn't grab
  references or lock pages or do anything else interesting with them, the
  entire synchronization is provided by mmu notifier retry loops. But the
  migration code does do a lot of expensive stuff, so it would need that.
  For hmm_range_fault it's probably not so important, so maybe we could
  leave the folio conversion of that to later.

I think we need both, meaning:
- switch to folio, leave out entries as NULL for compound folios
- on top of compoung folios still track contig ranges so that the gpu side
  can additionally benefit when e.g. 2M pages are split into smaller
  folios but happen to be contig

Cheers, Sima

> 
> > > Once we have folios at the driver/core mm api level doing all the fancy
> > > thp stuff should be at least a well-contained problem. But I might be
> > > dellusionally optimistic here :-)
> > 
> > I think it contained in the sense is the DRM SVM layer just allocates a
> > THP or large continous device memory and hands it off to migrate layer
> > and that layer does the right thing. The 'right thing' here I believe is
> > a decent amount of core MM work though.
> 
> Yeah that's what I meant, once we have switched the interfaces to be
> arrays of folios, where for larger folios we leave the entries in between
> NULL and have some appropriate iterators, then the driver side is done
> mostly.  The core mm side to actually make use of that will be fairly
> gnarly though.
> -Sima
> 
> > 
> > Matt
> > 
> > > -Sima
> > > 
> > > > 
> > > > Matt
> > > > 
> > > > > Regards,
> > > > >   Felix
> > > > > 
> > > > > 
> > > > > > 
> > > > > > Matt
> > > > > > 
> > > > > > > Regards,
> > > > > > >    Felix
> > > > > > > 
> > > > > > > 
> > > > > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> > > > > > > > > > > +				put_page(vmf->page);
> > > > > > > > > > > +				unlock_page(vmf->page);
> > > > > > > > > > > +			} else {
> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> > > > > > > > > > > +			}
> > > > > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> > > > > > > > > > >    			ret = VM_FAULT_HWPOISON;
> > > > > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> > > > > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> > > > > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> > > > > > > > > > > --- a/mm/migrate_device.c
> > > > > > > > > > > +++ b/mm/migrate_device.c
> > > > > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > > >    				   struct mm_walk *walk)
> > > > > > > > > > >    {
> > > > > > > > > > >    	struct migrate_vma *migrate = walk->private;
> > > > > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> > > > > > > > > > > +		page_folio(migrate->fault_page) : NULL;
> > > > > > > > > > >    	struct vm_area_struct *vma = walk->vma;
> > > > > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> > > > > > > > > > >    	unsigned long addr = start, unmapped = 0;
> > > > > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > > >    			folio_get(folio);
> > > > > > > > > > >    			spin_unlock(ptl);
> > > > > > > > > > > -			if (unlikely(!folio_trylock(folio)))
> > > > > > > > > > > +			if (unlikely(fault_folio != folio &&
> > > > > > > > > > > +				     !folio_trylock(folio)))
> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > > > >    								walk);
> > > > > > > > > > >    			ret = split_folio(folio);
> > > > > > > > > > > -			folio_unlock(folio);
> > > > > > > > > > > +			if (fault_folio != folio)
> > > > > > > > > > > +				folio_unlock(folio);
> > > > > > > > > > >    			folio_put(folio);
> > > > > > > > > > >    			if (ret)
> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> > > > > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > > >    		 * optimisation to avoid walking the rmap later with
> > > > > > > > > > >    		 * try_to_migrate().
> > > > > > > > > > >    		 */
> > > > > > > > > > > -		if (folio_trylock(folio)) {
> > > > > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> > > > > > > > > > >    			bool anon_exclusive;
> > > > > > > > > > >    			pte_t swp_pte;
> > > > > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> > > > > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> > > > > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> > > > > > > > > > > -					folio_unlock(folio);
> > > > > > > > > > > +					if (fault_folio != folio)
> > > > > > > > > > > +						folio_unlock(folio);
> > > > > > > > > > >    					folio_put(folio);
> > > > > > > > > > >    					mpfn = 0;
> > > > > > > > > > >    					goto next;
> > > > > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > > > >    					  unsigned long npages,
> > > > > > > > > > >    					  struct page *fault_page)
> > > > > > > > > > >    {
> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > > > >    	unsigned long i, restore = 0;
> > > > > > > > > > >    	bool allow_drain = true;
> > > > > > > > > > >    	unsigned long unmapped = 0;
> > > > > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> > > > > > > > > > >    		remove_migration_ptes(folio, folio, false);
> > > > > > > > > > >    		src_pfns[i] = 0;
> > > > > > > > > > > -		folio_unlock(folio);
> > > > > > > > > > > +		if (fault_folio != folio)
> > > > > > > > > > > +			folio_unlock(folio);
> > > > > > > > > > >    		folio_put(folio);
> > > > > > > > > > >    		restore--;
> > > > > > > > > > >    	}
> > > > > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> > > > > > > > > > >    		return -EINVAL;
> > > > > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> > > > > > > > > > >    		return -EINVAL;
> > > > > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> > > > > > > > > > > +		return -EINVAL;
> > > > > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> > > > > > > > > > >    	args->cpages = 0;
> > > > > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> > > > > > > > > > >    }
> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> > > > > > > > > > > -/*
> > > > > > > > > > > - * migrate_device_finalize() - complete page migration
> > > > > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > > > - * @npages: number of pages in the range
> > > > > > > > > > > - *
> > > > > > > > > > > - * Completes migration of the page by removing special migration entries.
> > > > > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > > > - * before calling this.
> > > > > > > > > > > - */
> > > > > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > > +				      unsigned long *dst_pfns,
> > > > > > > > > > > +				      unsigned long npages,
> > > > > > > > > > > +				      struct page *fault_page)
> > > > > > > > > > >    {
> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> > > > > > > > > > > +		page_folio(fault_page) : NULL;
> > > > > > > > > > >    	unsigned long i;
> > > > > > > > > > >    	for (i = 0; i < npages; i++) {
> > > > > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > >    		src = page_folio(page);
> > > > > > > > > > >    		dst = page_folio(newpage);
> > > > > > > > > > >    		remove_migration_ptes(src, dst, false);
> > > > > > > > > > > -		folio_unlock(src);
> > > > > > > > > > > +		if (fault_folio != src)
> > > > > > > > > > > +			folio_unlock(src);
> > > > > > > > > > >    		if (is_zone_device_page(page))
> > > > > > > > > > >    			put_page(page);
> > > > > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > >    		}
> > > > > > > > > > >    	}
> > > > > > > > > > >    }
> > > > > > > > > > > +
> > > > > > > > > > > +/*
> > > > > > > > > > > + * migrate_device_finalize() - complete page migration
> > > > > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> > > > > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> > > > > > > > > > > + * @npages: number of pages in the range
> > > > > > > > > > > + *
> > > > > > > > > > > + * Completes migration of the page by removing special migration entries.
> > > > > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> > > > > > > > > > > + * before calling this.
> > > > > > > > > > > + */
> > > > > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> > > > > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> > > > > > > > > > > +{
> > > > > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> > > > > > > > > > > +}
> > > > > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > > > >    /**
> > > > > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> > > > > > > > > > >     */
> > > > > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> > > > > > > > > > >    {
> > > > > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> > > > > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> > > > > > > > > > > +				  migrate->fault_page);
> > > > > > > > > > >    }
> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);
> > > 
> > > -- 
> > > Simona Vetter
> > > Software Engineer, Intel Corporation
> > > http://blog.ffwll.ch
> 
> -- 
> Simona Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch

-- 
Simona Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Alistair Popple]

Simona Vetter <simona.vetter@ffwll.ch> writes:

> On Wed, Sep 25, 2024 at 01:44:37PM +0200, Simona Vetter wrote:
>> On Tue, Sep 24, 2024 at 04:42:19PM +0000, Matthew Brost wrote:
>> > On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
>> > > On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
>> > > > On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
>> > > > > 
>> > > > > On 2024-09-20 17:23, Matthew Brost wrote:
>> > > > > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
>> > > > > > > On 2024-09-18 11:10, Alistair Popple wrote:
>> > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
>> > > > > > > > 
>> > > > > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
>> > > > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
>> > > > > > > > > > 
>> > > > > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
>> > > > > > > > > Still a driver.
>> > > > > > > > Indeed, and I'm happy to answer any questions about our implementation.
>> > > > > > > > 
>> > > > > > > > > > but theoretically it seems like it should be possible. However we
>> > > > > > > > > > serialize migrations of the same virtual address range to avoid these
>> > > > > > > > > > kind of issues as they can happen the other way too (ie. multiple
>> > > > > > > > > > threads trying to migrate to GPU).
>> > > > > > > > > > 
>> > > > > > > > > > So I suspect what happens in UVM is that one thread wins and installs
>> > > > > > > > > > the migration entry while the others fail to get the driver migration
>> > > > > > > > > > lock and bail out sufficiently early in the fault path to avoid the
>> > > > > > > > > > live-lock.
>> > > > > > > > > > 
>> > > > > > > > > I had to try hard to show this, doubt an actual user could trigger this.
>> > > > > > > > > 
>> > > > > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
>> > > > > > > > > and then tried to read the same page. This repeats in loop for like 512
>> > > > > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
>> > > > > > > > > for it to livelock. Without an exclusive lock I think on average I saw
>> > > > > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
>> > > > > > > > > thread won this race.
>> > > > > > > > > 
>> > > > > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
>> > > > > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
>> > > > > > > > Yes, I suspect you're correct here and that we just haven't tried hard
>> > > > > > > > enough to trigger it.
>> > > > > > > > 
>> > > > > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
>> > > > > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
>> > > > > > > > > > > Cc: Christian König <christian.koenig@amd.com>
>> > > > > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
>> > > > > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
>> > > > > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>> > > > > > > > > > > ---
>> > > > > > > > > > >    mm/memory.c         | 13 +++++++---
>> > > > > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
>> > > > > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
>> > > > > > > > > > > 
>> > > > > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
>> > > > > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
>> > > > > > > > > > > --- a/mm/memory.c
>> > > > > > > > > > > +++ b/mm/memory.c
>> > > > > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
>> > > > > > > > > > >    			 * Get a page reference while we know the page can't be
>> > > > > > > > > > >    			 * freed.
>> > > > > > > > > > >    			 */
>> > > > > > > > > > > -			get_page(vmf->page);
>> > > > > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
>> > > > > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>> > > > > > > > > > > -			put_page(vmf->page);
>> > > > > > > > > > > +			if (trylock_page(vmf->page)) {
>> > > > > > > > > > > +				get_page(vmf->page);
>> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>> > > > > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
>> > > > > > > > > > rather than do this and then have to pass all this context
>> > > > > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
>> > > > > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
>> > > > > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
>> > > > > > > > > > the migration entry prior to calling migrate_to_ram().
>> > > > > > > > > > 
>> > > > > > > > > > Obviously calling migrate_vma_setup() would show the page as not
>> > > > > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
>> > > > > > > > > > calling migrate_vma_setup().
>> > > > > > > > > > 
>> > > > > > > > > > This would eliminate the whole fault_page ugliness.
>> > > > > > > > > > 
>> > > > > > > > > This seems like it would work and agree it likely be cleaner. Let me
>> > > > > > > > > play around with this and see what I come up with. Multi-tasking a bit
>> > > > > > > > > so expect a bit of delay here.
>> > > > > > > > > 
>> > > > > > > > > Thanks for the input,
>> > > > > > > > > Matt
>> > > > > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
>> > > > > > > Christian and myself in the loop with future patches in this area.
>> > > > > > > 
>> > > > > > Will do. Already have another local patch set which helps drivers dma
>> > > > > > map 2M pages for migrations if SRAM is physically contiguous. Seems
>> > > > > > helpful for performance on Intel hardware. Probably post that soon for
>> > > > > > early feedack.
>> > > > > 
>> > > > > OK.
>> > > > > 
>> > > > > 
>> > > > > > 
>> > > > > > Longer term I thinking 2M migration entries, 2M device pages, and being
>> > > > > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
>> > > > > > finding migrate_vma_* functions take up like 80-90% of the time in the
>> > > > > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
>> > > > > > ideal. Seems like 2M entries for everything would really help here. No
>> > > > > > idea how feasible this is as the core MM stuff gets confusing fast. Any
>> > > > > > input on this idea?
>> > > > > 
>> > > > > I agree with your observations. We found that the migrate_vma_* code was the
>> > > > > bottle neck for migration performance as well, and not breaking 2M pages
>> > > > > could reduce that overhead a lot. I don't have any specific ideas. I'm not
>> > > > > familiar with the details of that code myself. Philip has looked at this
>> > > > > (and some old NVidia patches from a few years ago) in the past but never had
>> > > > > enough uninterrupted time to make it past prototyping.
>> > > > > 
>> > > > 
>> > > > Cool good to know this isn't some crazy idea. Doubt it happen anytime
>> > > > soon as I need to get a working baseline in before anything then start
>> > > > applying optimizations and help in get other features to get the design
>> > > > complete. But eventually will probably try to look at this. May ping
>> > > > Philip and Nvidia when I have time to dig in here.

Apologies for my late reply here, I have just returned from vacation.

We (Nvidia) are actively looking at this as we have the same bottle
necks. Mostly I've been doing some clean-ups in MM to make compound
ZONE_DEVICE pages possible.

>> > > 
>> > > I think the big step will be moving hmm.c and migrate.c apis over from
>> > > struct page to folios. That should also give us some nice benefits on the
>> > > gpu side, since instead of 4k pages to track we could allocate 2m gpu
>> > > pages.
>> > > 
>> > 
>> > I think was thinking just encode the order in the migration PFN like HMM
>> > does. Really only Nth order entry in the page array needs to be
>> > populated then - HMM populates every entry though which doesn't seem
>> > like that is required. Maybe having a folio API makes more sense?
>> 
>> Both I'd say, as a first attempt at least. An array of folios, but only
>> populate the ones we need and jump over empty entries. A bit wasteful, but
>> eh it's just allocations.
>
> Ok thought some more, I think there's two things going on:
>
> - spot contig memory sections so that the gpu side is more efficient and
>   can user bigger pagetables and stuff like that. this is what
>   hmm_range_fault does.
>
> - optimize the core mm book-keeping by working on folios instead of
>   individual pages. hmm_range_fault does not care because it doesn't grab
>   references or lock pages or do anything else interesting with them, the
>   entire synchronization is provided by mmu notifier retry loops. But the
>   migration code does do a lot of expensive stuff, so it would need that.
>   For hmm_range_fault it's probably not so important, so maybe we could
>   leave the folio conversion of that to later.
>
> I think we need both, meaning:
> - switch to folio, leave out entries as NULL for compound folios
> - on top of compoung folios still track contig ranges so that the gpu side
>   can additionally benefit when e.g. 2M pages are split into smaller
>   folios but happen to be contig

Definitely I think the folio approach makes sense and I was prototpying
a series to allow compound device private pages and to migrate
them. However I got caught up on similar questions around migrate_vma_*
API (specifically around whether to allow splitting/merging) so never
got around to posting them. Probably the simple array based approach
makes sense though.

In any case to get something like that merged I'd been asked if we could
fix the ZONE_DEVICE refcounting mess for DAX (as it was the only one
still relying on the off-by-one refcounts for compound pages). The good
news is that series[1] is getting close to merged, and as a side-effect
it allows for compound ZONE_DEVICE pages so extending it to
DEVICE_PRIVATE pages to allow THP migration shouldn't be too difficult
and was going to be my next step once this was merged. So it's nice to
know other people might care about this too.

[1] - https://lore.kernel.org/linux-mm/cover.9f0e45d52f5cff58807831b6b867084d0b14b61c.1725941415.git-series.apopple@nvidia.com/

> Cheers, Sima
>
>> 
>> > > Once we have folios at the driver/core mm api level doing all the fancy
>> > > thp stuff should be at least a well-contained problem. But I might be
>> > > dellusionally optimistic here :-)
>> > 
>> > I think it contained in the sense is the DRM SVM layer just allocates a
>> > THP or large continous device memory and hands it off to migrate layer
>> > and that layer does the right thing. The 'right thing' here I believe is
>> > a decent amount of core MM work though.
>> 
>> Yeah that's what I meant, once we have switched the interfaces to be
>> arrays of folios, where for larger folios we leave the entries in between
>> NULL and have some appropriate iterators, then the driver side is done
>> mostly.  The core mm side to actually make use of that will be fairly
>> gnarly though.
>> -Sima
>> 
>> > 
>> > Matt
>> > 
>> > > -Sima
>> > > 
>> > > > 
>> > > > Matt
>> > > > 
>> > > > > Regards,
>> > > > >   Felix
>> > > > > 
>> > > > > 
>> > > > > > 
>> > > > > > Matt
>> > > > > > 
>> > > > > > > Regards,
>> > > > > > >    Felix
>> > > > > > > 
>> > > > > > > 
>> > > > > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
>> > > > > > > > > > > +				put_page(vmf->page);
>> > > > > > > > > > > +				unlock_page(vmf->page);
>> > > > > > > > > > > +			} else {
>> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
>> > > > > > > > > > > +			}
>> > > > > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
>> > > > > > > > > > >    			ret = VM_FAULT_HWPOISON;
>> > > > > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
>> > > > > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
>> > > > > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
>> > > > > > > > > > > --- a/mm/migrate_device.c
>> > > > > > > > > > > +++ b/mm/migrate_device.c
>> > > > > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> > > > > > > > > > >    				   struct mm_walk *walk)
>> > > > > > > > > > >    {
>> > > > > > > > > > >    	struct migrate_vma *migrate = walk->private;
>> > > > > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
>> > > > > > > > > > > +		page_folio(migrate->fault_page) : NULL;
>> > > > > > > > > > >    	struct vm_area_struct *vma = walk->vma;
>> > > > > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
>> > > > > > > > > > >    	unsigned long addr = start, unmapped = 0;
>> > > > > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> > > > > > > > > > >    			folio_get(folio);
>> > > > > > > > > > >    			spin_unlock(ptl);
>> > > > > > > > > > > -			if (unlikely(!folio_trylock(folio)))
>> > > > > > > > > > > +			if (unlikely(fault_folio != folio &&
>> > > > > > > > > > > +				     !folio_trylock(folio)))
>> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
>> > > > > > > > > > >    								walk);
>> > > > > > > > > > >    			ret = split_folio(folio);
>> > > > > > > > > > > -			folio_unlock(folio);
>> > > > > > > > > > > +			if (fault_folio != folio)
>> > > > > > > > > > > +				folio_unlock(folio);
>> > > > > > > > > > >    			folio_put(folio);
>> > > > > > > > > > >    			if (ret)
>> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
>> > > > > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> > > > > > > > > > >    		 * optimisation to avoid walking the rmap later with
>> > > > > > > > > > >    		 * try_to_migrate().
>> > > > > > > > > > >    		 */
>> > > > > > > > > > > -		if (folio_trylock(folio)) {
>> > > > > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
>> > > > > > > > > > >    			bool anon_exclusive;
>> > > > > > > > > > >    			pte_t swp_pte;
>> > > > > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>> > > > > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
>> > > > > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
>> > > > > > > > > > > -					folio_unlock(folio);
>> > > > > > > > > > > +					if (fault_folio != folio)
>> > > > > > > > > > > +						folio_unlock(folio);
>> > > > > > > > > > >    					folio_put(folio);
>> > > > > > > > > > >    					mpfn = 0;
>> > > > > > > > > > >    					goto next;
>> > > > > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>> > > > > > > > > > >    					  unsigned long npages,
>> > > > > > > > > > >    					  struct page *fault_page)
>> > > > > > > > > > >    {
>> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
>> > > > > > > > > > > +		page_folio(fault_page) : NULL;
>> > > > > > > > > > >    	unsigned long i, restore = 0;
>> > > > > > > > > > >    	bool allow_drain = true;
>> > > > > > > > > > >    	unsigned long unmapped = 0;
>> > > > > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
>> > > > > > > > > > >    		remove_migration_ptes(folio, folio, false);
>> > > > > > > > > > >    		src_pfns[i] = 0;
>> > > > > > > > > > > -		folio_unlock(folio);
>> > > > > > > > > > > +		if (fault_folio != folio)
>> > > > > > > > > > > +			folio_unlock(folio);
>> > > > > > > > > > >    		folio_put(folio);
>> > > > > > > > > > >    		restore--;
>> > > > > > > > > > >    	}
>> > > > > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
>> > > > > > > > > > >    		return -EINVAL;
>> > > > > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
>> > > > > > > > > > >    		return -EINVAL;
>> > > > > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
>> > > > > > > > > > > +		return -EINVAL;
>> > > > > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
>> > > > > > > > > > >    	args->cpages = 0;
>> > > > > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
>> > > > > > > > > > >    }
>> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
>> > > > > > > > > > > -/*
>> > > > > > > > > > > - * migrate_device_finalize() - complete page migration
>> > > > > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
>> > > > > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>> > > > > > > > > > > - * @npages: number of pages in the range
>> > > > > > > > > > > - *
>> > > > > > > > > > > - * Completes migration of the page by removing special migration entries.
>> > > > > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
>> > > > > > > > > > > - * before calling this.
>> > > > > > > > > > > - */
>> > > > > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
>> > > > > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
>> > > > > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
>> > > > > > > > > > > +				      unsigned long *dst_pfns,
>> > > > > > > > > > > +				      unsigned long npages,
>> > > > > > > > > > > +				      struct page *fault_page)
>> > > > > > > > > > >    {
>> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
>> > > > > > > > > > > +		page_folio(fault_page) : NULL;
>> > > > > > > > > > >    	unsigned long i;
>> > > > > > > > > > >    	for (i = 0; i < npages; i++) {
>> > > > > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
>> > > > > > > > > > >    		src = page_folio(page);
>> > > > > > > > > > >    		dst = page_folio(newpage);
>> > > > > > > > > > >    		remove_migration_ptes(src, dst, false);
>> > > > > > > > > > > -		folio_unlock(src);
>> > > > > > > > > > > +		if (fault_folio != src)
>> > > > > > > > > > > +			folio_unlock(src);
>> > > > > > > > > > >    		if (is_zone_device_page(page))
>> > > > > > > > > > >    			put_page(page);
>> > > > > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
>> > > > > > > > > > >    		}
>> > > > > > > > > > >    	}
>> > > > > > > > > > >    }
>> > > > > > > > > > > +
>> > > > > > > > > > > +/*
>> > > > > > > > > > > + * migrate_device_finalize() - complete page migration
>> > > > > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
>> > > > > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
>> > > > > > > > > > > + * @npages: number of pages in the range
>> > > > > > > > > > > + *
>> > > > > > > > > > > + * Completes migration of the page by removing special migration entries.
>> > > > > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
>> > > > > > > > > > > + * before calling this.
>> > > > > > > > > > > + */
>> > > > > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
>> > > > > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
>> > > > > > > > > > > +{
>> > > > > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
>> > > > > > > > > > > +}
>> > > > > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
>> > > > > > > > > > >    /**
>> > > > > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
>> > > > > > > > > > >     */
>> > > > > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
>> > > > > > > > > > >    {
>> > > > > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
>> > > > > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
>> > > > > > > > > > > +				  migrate->fault_page);
>> > > > > > > > > > >    }
>> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);
>> > > 
>> > > -- 
>> > > Simona Vetter
>> > > Software Engineer, Intel Corporation
>> > > http://blog.ffwll.ch
>> 
>> -- 
>> Simona Vetter
>> Software Engineer, Intel Corporation
>> http://blog.ffwll.ch

Re: [PATCH 1/1] mm/migrate: Trylock device page in do_swap_page

[Matthew Brost]

On Tue, Oct 08, 2024 at 12:33:51PM +1100, Alistair Popple wrote:
> 
> Simona Vetter <simona.vetter@ffwll.ch> writes:
> 
> > On Wed, Sep 25, 2024 at 01:44:37PM +0200, Simona Vetter wrote:
> >> On Tue, Sep 24, 2024 at 04:42:19PM +0000, Matthew Brost wrote:
> >> > On Tue, Sep 24, 2024 at 01:48:29PM +0200, Simona Vetter wrote:
> >> > > On Fri, Sep 20, 2024 at 09:59:51PM +0000, Matthew Brost wrote:
> >> > > > On Fri, Sep 20, 2024 at 05:50:10PM -0400, Felix Kuehling wrote:
> >> > > > > 
> >> > > > > On 2024-09-20 17:23, Matthew Brost wrote:
> >> > > > > > On Fri, Sep 20, 2024 at 04:26:50PM -0400, Felix Kuehling wrote:
> >> > > > > > > On 2024-09-18 11:10, Alistair Popple wrote:
> >> > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> >> > > > > > > > 
> >> > > > > > > > > On Wed, Sep 11, 2024 at 02:53:31PM +1000, Alistair Popple wrote:
> >> > > > > > > > > > Matthew Brost <matthew.brost@intel.com> writes:
> >> > > > > > > > > > 
> >> > > > > > > > > > I haven't seen the same in the NVIDIA UVM driver (out-of-tree, I know)
> >> > > > > > > > > Still a driver.
> >> > > > > > > > Indeed, and I'm happy to answer any questions about our implementation.
> >> > > > > > > > 
> >> > > > > > > > > > but theoretically it seems like it should be possible. However we
> >> > > > > > > > > > serialize migrations of the same virtual address range to avoid these
> >> > > > > > > > > > kind of issues as they can happen the other way too (ie. multiple
> >> > > > > > > > > > threads trying to migrate to GPU).
> >> > > > > > > > > > 
> >> > > > > > > > > > So I suspect what happens in UVM is that one thread wins and installs
> >> > > > > > > > > > the migration entry while the others fail to get the driver migration
> >> > > > > > > > > > lock and bail out sufficiently early in the fault path to avoid the
> >> > > > > > > > > > live-lock.
> >> > > > > > > > > > 
> >> > > > > > > > > I had to try hard to show this, doubt an actual user could trigger this.
> >> > > > > > > > > 
> >> > > > > > > > > I wrote a test which kicked 8 threads, each thread did a pthread join,
> >> > > > > > > > > and then tried to read the same page. This repeats in loop for like 512
> >> > > > > > > > > pages or something. I needed an exclusive lock in migrate_to_ram vfunc
> >> > > > > > > > > for it to livelock. Without an exclusive lock I think on average I saw
> >> > > > > > > > > about 32k retries (i.e. migrate_to_ram calls on the same page) before a
> >> > > > > > > > > thread won this race.
> >> > > > > > > > > 
> >> > > > > > > > >   From reading UVM, pretty sure if you tried hard enough you could trigger
> >> > > > > > > > > a livelock given it appears you take excluvise locks in migrate_to_ram.
> >> > > > > > > > Yes, I suspect you're correct here and that we just haven't tried hard
> >> > > > > > > > enough to trigger it.
> >> > > > > > > > 
> >> > > > > > > > > > > Cc: Philip Yang <Philip.Yang@amd.com>
> >> > > > > > > > > > > Cc: Felix Kuehling <felix.kuehling@amd.com>
> >> > > > > > > > > > > Cc: Christian König <christian.koenig@amd.com>
> >> > > > > > > > > > > Cc: Andrew Morton <akpm@linux-foundation.org>
> >> > > > > > > > > > > Suggessted-by: Simona Vetter <simona.vetter@ffwll.ch>
> >> > > > > > > > > > > Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> >> > > > > > > > > > > ---
> >> > > > > > > > > > >    mm/memory.c         | 13 +++++++---
> >> > > > > > > > > > >    mm/migrate_device.c | 60 +++++++++++++++++++++++++++++++--------------
> >> > > > > > > > > > >    2 files changed, 50 insertions(+), 23 deletions(-)
> >> > > > > > > > > > > 
> >> > > > > > > > > > > diff --git a/mm/memory.c b/mm/memory.c
> >> > > > > > > > > > > index 3c01d68065be..bbd97d16a96a 100644
> >> > > > > > > > > > > --- a/mm/memory.c
> >> > > > > > > > > > > +++ b/mm/memory.c
> >> > > > > > > > > > > @@ -4046,10 +4046,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
> >> > > > > > > > > > >    			 * Get a page reference while we know the page can't be
> >> > > > > > > > > > >    			 * freed.
> >> > > > > > > > > > >    			 */
> >> > > > > > > > > > > -			get_page(vmf->page);
> >> > > > > > > > > > > -			pte_unmap_unlock(vmf->pte, vmf->ptl);
> >> > > > > > > > > > > -			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> >> > > > > > > > > > > -			put_page(vmf->page);
> >> > > > > > > > > > > +			if (trylock_page(vmf->page)) {
> >> > > > > > > > > > > +				get_page(vmf->page);
> >> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> >> > > > > > > > > > This is all beginning to look a lot like migrate_vma_collect_pmd(). So
> >> > > > > > > > > > rather than do this and then have to pass all this context
> >> > > > > > > > > > (ie. fault_page) down to the migrate_vma_* functions could we instead
> >> > > > > > > > > > just do what migrate_vma_collect_pmd() does here? Ie. we already have
> >> > > > > > > > > > the PTL and the page lock so there's no reason we couldn't just setup
> >> > > > > > > > > > the migration entry prior to calling migrate_to_ram().
> >> > > > > > > > > > 
> >> > > > > > > > > > Obviously calling migrate_vma_setup() would show the page as not
> >> > > > > > > > > > migrating, but drivers could easily just fill in the src_pfn info after
> >> > > > > > > > > > calling migrate_vma_setup().
> >> > > > > > > > > > 
> >> > > > > > > > > > This would eliminate the whole fault_page ugliness.
> >> > > > > > > > > > 
> >> > > > > > > > > This seems like it would work and agree it likely be cleaner. Let me
> >> > > > > > > > > play around with this and see what I come up with. Multi-tasking a bit
> >> > > > > > > > > so expect a bit of delay here.
> >> > > > > > > > > 
> >> > > > > > > > > Thanks for the input,
> >> > > > > > > > > Matt
> >> > > > > > > Thanks! Sorry, I'm late catching up after a vacation. Please keep Philip,
> >> > > > > > > Christian and myself in the loop with future patches in this area.
> >> > > > > > > 
> >> > > > > > Will do. Already have another local patch set which helps drivers dma
> >> > > > > > map 2M pages for migrations if SRAM is physically contiguous. Seems
> >> > > > > > helpful for performance on Intel hardware. Probably post that soon for
> >> > > > > > early feedack.
> >> > > > > 
> >> > > > > OK.
> >> > > > > 
> >> > > > > 
> >> > > > > > 
> >> > > > > > Longer term I thinking 2M migration entries, 2M device pages, and being
> >> > > > > > able to install 2M THP on VRAM -> SRAM could be really helpful. I'm
> >> > > > > > finding migrate_vma_* functions take up like 80-90% of the time in the
> >> > > > > > CPU / GPU fault handlers on a fault (or prefetch) which doesn't seem
> >> > > > > > ideal. Seems like 2M entries for everything would really help here. No
> >> > > > > > idea how feasible this is as the core MM stuff gets confusing fast. Any
> >> > > > > > input on this idea?
> >> > > > > 
> >> > > > > I agree with your observations. We found that the migrate_vma_* code was the
> >> > > > > bottle neck for migration performance as well, and not breaking 2M pages
> >> > > > > could reduce that overhead a lot. I don't have any specific ideas. I'm not
> >> > > > > familiar with the details of that code myself. Philip has looked at this
> >> > > > > (and some old NVidia patches from a few years ago) in the past but never had
> >> > > > > enough uninterrupted time to make it past prototyping.
> >> > > > > 
> >> > > > 
> >> > > > Cool good to know this isn't some crazy idea. Doubt it happen anytime
> >> > > > soon as I need to get a working baseline in before anything then start
> >> > > > applying optimizations and help in get other features to get the design
> >> > > > complete. But eventually will probably try to look at this. May ping
> >> > > > Philip and Nvidia when I have time to dig in here.
> 
> Apologies for my late reply here, I have just returned from vacation.
> 
> We (Nvidia) are actively looking at this as we have the same bottle
> necks. Mostly I've been doing some clean-ups in MM to make compound
> ZONE_DEVICE pages possible.
> 

That's good to know all of us (Intel, AMD, Nvidia) are in agreement this
is a bottleneck and needs fixing. 

> >> > > 
> >> > > I think the big step will be moving hmm.c and migrate.c apis over from
> >> > > struct page to folios. That should also give us some nice benefits on the
> >> > > gpu side, since instead of 4k pages to track we could allocate 2m gpu
> >> > > pages.
> >> > > 
> >> > 
> >> > I think was thinking just encode the order in the migration PFN like HMM
> >> > does. Really only Nth order entry in the page array needs to be
> >> > populated then - HMM populates every entry though which doesn't seem
> >> > like that is required. Maybe having a folio API makes more sense?
> >> 
> >> Both I'd say, as a first attempt at least. An array of folios, but only
> >> populate the ones we need and jump over empty entries. A bit wasteful, but
> >> eh it's just allocations.
> >
> > Ok thought some more, I think there's two things going on:
> >
> > - spot contig memory sections so that the gpu side is more efficient and
> >   can user bigger pagetables and stuff like that. this is what
> >   hmm_range_fault does.
> >
> > - optimize the core mm book-keeping by working on folios instead of
> >   individual pages. hmm_range_fault does not care because it doesn't grab
> >   references or lock pages or do anything else interesting with them, the
> >   entire synchronization is provided by mmu notifier retry loops. But the
> >   migration code does do a lot of expensive stuff, so it would need that.
> >   For hmm_range_fault it's probably not so important, so maybe we could
> >   leave the folio conversion of that to later.
> >
> > I think we need both, meaning:
> > - switch to folio, leave out entries as NULL for compound folios
> > - on top of compoung folios still track contig ranges so that the gpu side
> >   can additionally benefit when e.g. 2M pages are split into smaller
> >   folios but happen to be contig
> 
> Definitely I think the folio approach makes sense and I was prototpying
> a series to allow compound device private pages and to migrate
> them. However I got caught up on similar questions around migrate_vma_*
> API (specifically around whether to allow splitting/merging) so never
> got around to posting them. Probably the simple array based approach
> makes sense though.
> 

If / when we (Intel) have time to look at this we will reach out but
seems like you may beat us to it. Happy to pull in code to test out and
with reviews.

> In any case to get something like that merged I'd been asked if we could
> fix the ZONE_DEVICE refcounting mess for DAX (as it was the only one
> still relying on the off-by-one refcounts for compound pages). The good
> news is that series[1] is getting close to merged, and as a side-effect
> it allows for compound ZONE_DEVICE pages so extending it to
> DEVICE_PRIVATE pages to allow THP migration shouldn't be too difficult
> and was going to be my next step once this was merged. So it's nice to
> know other people might care about this too.
> 
> [1] - https://lore.kernel.org/linux-mm/cover.9f0e45d52f5cff58807831b6b867084d0b14b61c.1725941415.git-series.apopple@nvidia.com/
>

Let me look here.

Matt

> > Cheers, Sima
> >
> >> 
> >> > > Once we have folios at the driver/core mm api level doing all the fancy
> >> > > thp stuff should be at least a well-contained problem. But I might be
> >> > > dellusionally optimistic here :-)
> >> > 
> >> > I think it contained in the sense is the DRM SVM layer just allocates a
> >> > THP or large continous device memory and hands it off to migrate layer
> >> > and that layer does the right thing. The 'right thing' here I believe is
> >> > a decent amount of core MM work though.
> >> 
> >> Yeah that's what I meant, once we have switched the interfaces to be
> >> arrays of folios, where for larger folios we leave the entries in between
> >> NULL and have some appropriate iterators, then the driver side is done
> >> mostly.  The core mm side to actually make use of that will be fairly
> >> gnarly though.
> >> -Sima
> >> 
> >> > 
> >> > Matt
> >> > 
> >> > > -Sima
> >> > > 
> >> > > > 
> >> > > > Matt
> >> > > > 
> >> > > > > Regards,
> >> > > > >   Felix
> >> > > > > 
> >> > > > > 
> >> > > > > > 
> >> > > > > > Matt
> >> > > > > > 
> >> > > > > > > Regards,
> >> > > > > > >    Felix
> >> > > > > > > 
> >> > > > > > > 
> >> > > > > > > > > > > +				ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
> >> > > > > > > > > > > +				put_page(vmf->page);
> >> > > > > > > > > > > +				unlock_page(vmf->page);
> >> > > > > > > > > > > +			} else {
> >> > > > > > > > > > > +				pte_unmap_unlock(vmf->pte, vmf->ptl);
> >> > > > > > > > > > > +			}
> >> > > > > > > > > > >    		} else if (is_hwpoison_entry(entry)) {
> >> > > > > > > > > > >    			ret = VM_FAULT_HWPOISON;
> >> > > > > > > > > > >    		} else if (is_pte_marker_entry(entry)) {
> >> > > > > > > > > > > diff --git a/mm/migrate_device.c b/mm/migrate_device.c
> >> > > > > > > > > > > index 6d66dc1c6ffa..049893a5a179 100644
> >> > > > > > > > > > > --- a/mm/migrate_device.c
> >> > > > > > > > > > > +++ b/mm/migrate_device.c
> >> > > > > > > > > > > @@ -60,6 +60,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >> > > > > > > > > > >    				   struct mm_walk *walk)
> >> > > > > > > > > > >    {
> >> > > > > > > > > > >    	struct migrate_vma *migrate = walk->private;
> >> > > > > > > > > > > +	struct folio *fault_folio = migrate->fault_page ?
> >> > > > > > > > > > > +		page_folio(migrate->fault_page) : NULL;
> >> > > > > > > > > > >    	struct vm_area_struct *vma = walk->vma;
> >> > > > > > > > > > >    	struct mm_struct *mm = vma->vm_mm;
> >> > > > > > > > > > >    	unsigned long addr = start, unmapped = 0;
> >> > > > > > > > > > > @@ -88,11 +90,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >> > > > > > > > > > >    			folio_get(folio);
> >> > > > > > > > > > >    			spin_unlock(ptl);
> >> > > > > > > > > > > -			if (unlikely(!folio_trylock(folio)))
> >> > > > > > > > > > > +			if (unlikely(fault_folio != folio &&
> >> > > > > > > > > > > +				     !folio_trylock(folio)))
> >> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> >> > > > > > > > > > >    								walk);
> >> > > > > > > > > > >    			ret = split_folio(folio);
> >> > > > > > > > > > > -			folio_unlock(folio);
> >> > > > > > > > > > > +			if (fault_folio != folio)
> >> > > > > > > > > > > +				folio_unlock(folio);
> >> > > > > > > > > > >    			folio_put(folio);
> >> > > > > > > > > > >    			if (ret)
> >> > > > > > > > > > >    				return migrate_vma_collect_skip(start, end,
> >> > > > > > > > > > > @@ -192,7 +196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >> > > > > > > > > > >    		 * optimisation to avoid walking the rmap later with
> >> > > > > > > > > > >    		 * try_to_migrate().
> >> > > > > > > > > > >    		 */
> >> > > > > > > > > > > -		if (folio_trylock(folio)) {
> >> > > > > > > > > > > +		if (fault_folio == folio || folio_trylock(folio)) {
> >> > > > > > > > > > >    			bool anon_exclusive;
> >> > > > > > > > > > >    			pte_t swp_pte;
> >> > > > > > > > > > > @@ -204,7 +208,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> >> > > > > > > > > > >    				if (folio_try_share_anon_rmap_pte(folio, page)) {
> >> > > > > > > > > > >    					set_pte_at(mm, addr, ptep, pte);
> >> > > > > > > > > > > -					folio_unlock(folio);
> >> > > > > > > > > > > +					if (fault_folio != folio)
> >> > > > > > > > > > > +						folio_unlock(folio);
> >> > > > > > > > > > >    					folio_put(folio);
> >> > > > > > > > > > >    					mpfn = 0;
> >> > > > > > > > > > >    					goto next;
> >> > > > > > > > > > > @@ -363,6 +368,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >> > > > > > > > > > >    					  unsigned long npages,
> >> > > > > > > > > > >    					  struct page *fault_page)
> >> > > > > > > > > > >    {
> >> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> >> > > > > > > > > > > +		page_folio(fault_page) : NULL;
> >> > > > > > > > > > >    	unsigned long i, restore = 0;
> >> > > > > > > > > > >    	bool allow_drain = true;
> >> > > > > > > > > > >    	unsigned long unmapped = 0;
> >> > > > > > > > > > > @@ -427,7 +434,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
> >> > > > > > > > > > >    		remove_migration_ptes(folio, folio, false);
> >> > > > > > > > > > >    		src_pfns[i] = 0;
> >> > > > > > > > > > > -		folio_unlock(folio);
> >> > > > > > > > > > > +		if (fault_folio != folio)
> >> > > > > > > > > > > +			folio_unlock(folio);
> >> > > > > > > > > > >    		folio_put(folio);
> >> > > > > > > > > > >    		restore--;
> >> > > > > > > > > > >    	}
> >> > > > > > > > > > > @@ -536,6 +544,8 @@ int migrate_vma_setup(struct migrate_vma *args)
> >> > > > > > > > > > >    		return -EINVAL;
> >> > > > > > > > > > >    	if (args->fault_page && !is_device_private_page(args->fault_page))
> >> > > > > > > > > > >    		return -EINVAL;
> >> > > > > > > > > > > +	if (args->fault_page && !PageLocked(args->fault_page))
> >> > > > > > > > > > > +		return -EINVAL;
> >> > > > > > > > > > >    	memset(args->src, 0, sizeof(*args->src) * nr_pages);
> >> > > > > > > > > > >    	args->cpages = 0;
> >> > > > > > > > > > > @@ -799,19 +809,13 @@ void migrate_vma_pages(struct migrate_vma *migrate)
> >> > > > > > > > > > >    }
> >> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_pages);
> >> > > > > > > > > > > -/*
> >> > > > > > > > > > > - * migrate_device_finalize() - complete page migration
> >> > > > > > > > > > > - * @src_pfns: src_pfns returned from migrate_device_range()
> >> > > > > > > > > > > - * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> >> > > > > > > > > > > - * @npages: number of pages in the range
> >> > > > > > > > > > > - *
> >> > > > > > > > > > > - * Completes migration of the page by removing special migration entries.
> >> > > > > > > > > > > - * Drivers must ensure copying of page data is complete and visible to the CPU
> >> > > > > > > > > > > - * before calling this.
> >> > > > > > > > > > > - */
> >> > > > > > > > > > > -void migrate_device_finalize(unsigned long *src_pfns,
> >> > > > > > > > > > > -			unsigned long *dst_pfns, unsigned long npages)
> >> > > > > > > > > > > +static void __migrate_device_finalize(unsigned long *src_pfns,
> >> > > > > > > > > > > +				      unsigned long *dst_pfns,
> >> > > > > > > > > > > +				      unsigned long npages,
> >> > > > > > > > > > > +				      struct page *fault_page)
> >> > > > > > > > > > >    {
> >> > > > > > > > > > > +	struct folio *fault_folio = fault_page ?
> >> > > > > > > > > > > +		page_folio(fault_page) : NULL;
> >> > > > > > > > > > >    	unsigned long i;
> >> > > > > > > > > > >    	for (i = 0; i < npages; i++) {
> >> > > > > > > > > > > @@ -838,7 +842,8 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >> > > > > > > > > > >    		src = page_folio(page);
> >> > > > > > > > > > >    		dst = page_folio(newpage);
> >> > > > > > > > > > >    		remove_migration_ptes(src, dst, false);
> >> > > > > > > > > > > -		folio_unlock(src);
> >> > > > > > > > > > > +		if (fault_folio != src)
> >> > > > > > > > > > > +			folio_unlock(src);
> >> > > > > > > > > > >    		if (is_zone_device_page(page))
> >> > > > > > > > > > >    			put_page(page);
> >> > > > > > > > > > > @@ -854,6 +859,22 @@ void migrate_device_finalize(unsigned long *src_pfns,
> >> > > > > > > > > > >    		}
> >> > > > > > > > > > >    	}
> >> > > > > > > > > > >    }
> >> > > > > > > > > > > +
> >> > > > > > > > > > > +/*
> >> > > > > > > > > > > + * migrate_device_finalize() - complete page migration
> >> > > > > > > > > > > + * @src_pfns: src_pfns returned from migrate_device_range()
> >> > > > > > > > > > > + * @dst_pfns: array of pfns allocated by the driver to migrate memory to
> >> > > > > > > > > > > + * @npages: number of pages in the range
> >> > > > > > > > > > > + *
> >> > > > > > > > > > > + * Completes migration of the page by removing special migration entries.
> >> > > > > > > > > > > + * Drivers must ensure copying of page data is complete and visible to the CPU
> >> > > > > > > > > > > + * before calling this.
> >> > > > > > > > > > > + */
> >> > > > > > > > > > > +void migrate_device_finalize(unsigned long *src_pfns,
> >> > > > > > > > > > > +			unsigned long *dst_pfns, unsigned long npages)
> >> > > > > > > > > > > +{
> >> > > > > > > > > > > +	return __migrate_device_finalize(src_pfns, dst_pfns, npages, NULL);
> >> > > > > > > > > > > +}
> >> > > > > > > > > > >    EXPORT_SYMBOL(migrate_device_finalize);
> >> > > > > > > > > > >    /**
> >> > > > > > > > > > > @@ -869,7 +890,8 @@ EXPORT_SYMBOL(migrate_device_finalize);
> >> > > > > > > > > > >     */
> >> > > > > > > > > > >    void migrate_vma_finalize(struct migrate_vma *migrate)
> >> > > > > > > > > > >    {
> >> > > > > > > > > > > -	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
> >> > > > > > > > > > > +	__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
> >> > > > > > > > > > > +				  migrate->fault_page);
> >> > > > > > > > > > >    }
> >> > > > > > > > > > >    EXPORT_SYMBOL(migrate_vma_finalize);
> >> > > 
> >> > > -- 
> >> > > Simona Vetter
> >> > > Software Engineer, Intel Corporation
> >> > > http://blog.ffwll.ch
> >> 
> >> -- 
> >> Simona Vetter
> >> Software Engineer, Intel Corporation
> >> http://blog.ffwll.ch
>