[PATCH] mm/hugetlb: fix deadlock in __hugetlb_zap_begin() by using trylock

[PATCH] mm/hugetlb: fix deadlock in __hugetlb_zap_begin() by using trylock

[Kartik Nair]

syzbot reported a circular locking dependency involving
resv_map->rw_sema and mmap_lock:

  CPU0                          CPU1
  lock(&mm->mmap_lock)
                                lock(sk_lock-AF_INET6)
                                lock(&mm->mmap_lock)
  lock(&resv_map->rw_sema)

__hugetlb_zap_begin() calls hugetlb_vma_lock_write() which does a
blocking down_write() on either vma_lock->rw_sema or
resv_map->rw_sema while mmap_lock is already held for write by the
caller chain (vm_mmap_pgoff -> mmap_region -> __mmap_region ->
unmap_region -> unmap_vmas -> hugetlb_zap_begin).

Fix this by converting __hugetlb_zap_begin() to use
hugetlb_vma_trylock_write() instead of hugetlb_vma_lock_write().
If the trylock fails, return false to the callers so they can skip
the zap operation safely. Update hugetlb_zap_begin() and its callers
in unmap_vmas() and zap_vma_range_batched() accordingly.

Reported-by: syzbot+bd6aaf99e8443d8a9034@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=bd6aaf99e8443d8a9034
Signed-off-by: Kartik Nair <contact.kartikn@gmail.com>
---
 include/linux/hugetlb.h | 10 ++++++----
 mm/hugetlb.c            |  8 +++++---
 mm/memory.c             | 10 ++++++----
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 93418625d3c5..1972464bd92f 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -244,16 +244,17 @@ void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma);
 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
 				unsigned long *start, unsigned long *end);
 
-extern void __hugetlb_zap_begin(struct vm_area_struct *vma,
+extern bool __hugetlb_zap_begin(struct vm_area_struct *vma,
 				unsigned long *begin, unsigned long *end);
 extern void __hugetlb_zap_end(struct vm_area_struct *vma,
 			      struct zap_details *details);
 
-static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
+static inline bool hugetlb_zap_begin(struct vm_area_struct *vma,
 				     unsigned long *start, unsigned long *end)
 {
 	if (is_vm_hugetlb_page(vma))
-		__hugetlb_zap_begin(vma, start, end);
+		return __hugetlb_zap_begin(vma, start, end);
+	return true;
 }
 
 static inline void hugetlb_zap_end(struct vm_area_struct *vma,
@@ -318,10 +319,11 @@ static inline void adjust_range_if_pmd_sharing_possible(
 {
 }
 
-static inline void hugetlb_zap_begin(
+static inline bool hugetlb_zap_begin(
 				struct vm_area_struct *vma,
 				unsigned long *start, unsigned long *end)
 {
+	return true;
 }
 
 static inline void hugetlb_zap_end(
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index f24bf49be047..dd55ec2ef007 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5309,16 +5309,18 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	huge_pmd_unshare_flush(tlb, vma);
 }
 
-void __hugetlb_zap_begin(struct vm_area_struct *vma,
+bool __hugetlb_zap_begin(struct vm_area_struct *vma,
 			 unsigned long *start, unsigned long *end)
 {
 	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
-		return;
+		return false;
 
 	adjust_range_if_pmd_sharing_possible(vma, start, end);
-	hugetlb_vma_lock_write(vma);
+	if (!hugetlb_vma_trylock_write(vma))
+		return false;
 	if (vma->vm_file)
 		i_mmap_lock_write(vma->vm_file->f_mapping);
+	return true;
 }
 
 void __hugetlb_zap_end(struct vm_area_struct *vma,
diff --git a/mm/memory.c b/mm/memory.c
index ea6568571131..c1451e5b6ee7 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2158,9 +2158,10 @@ void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap)
 		unsigned long start = max(vma->vm_start, unmap->vma_start);
 		unsigned long end = min(vma->vm_end, unmap->vma_end);
 
-		hugetlb_zap_begin(vma, &start, &end);
-		__zap_vma_range(tlb, vma, start, end, &details);
-		hugetlb_zap_end(vma, &details);
+		if (hugetlb_zap_begin(vma, &start, &end)) {
+			__zap_vma_range(tlb, vma, start, end, &details);
+			hugetlb_zap_end(vma, &details);
+		}
 		vma = mas_find(unmap->mas, unmap->tree_end - 1);
 	} while (vma);
 	mmu_notifier_invalidate_range_end(&range);
@@ -2194,7 +2195,8 @@ void zap_vma_range_batched(struct mmu_gather *tlb,
 
 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
 				address, end);
-	hugetlb_zap_begin(vma, &range.start, &range.end);
+	if (!hugetlb_zap_begin(vma, &range.start, &range.end))
+		return;
 	update_hiwater_rss(vma->vm_mm);
 	mmu_notifier_invalidate_range_start(&range);
 	/*
-- 
2.39.5 (Apple Git-154)

Re: [PATCH] mm/hugetlb: fix deadlock in __hugetlb_zap_begin() by using trylock

[Hillf Danton]

On Thu, 14 May 2026 02:49:27 +0530 Kartik Nair wrote:
> syzbot reported a circular locking dependency involving
> resv_map->rw_sema and mmap_lock:
> 
>   CPU0                          CPU1
>   lock(&mm->mmap_lock)
>                                 lock(sk_lock-AF_INET6)
>                                 lock(&mm->mmap_lock)
>   lock(&resv_map->rw_sema)
> 
> __hugetlb_zap_begin() calls hugetlb_vma_lock_write() which does a
> blocking down_write() on either vma_lock->rw_sema or
> resv_map->rw_sema while mmap_lock is already held for write by the
> caller chain (vm_mmap_pgoff -> mmap_region -> __mmap_region ->
> unmap_region -> unmap_vmas -> hugetlb_zap_begin).
> 
> Fix this by converting __hugetlb_zap_begin() to use
> hugetlb_vma_trylock_write() instead of hugetlb_vma_lock_write().
> If the trylock fails, return false to the callers so they can skip
> the zap operation safely. Update hugetlb_zap_begin() and its callers
> in unmap_vmas() and zap_vma_range_batched() accordingly.
>
Given q->q_usage_counter in the syzbot report [1] and the correct
locking order in ffa1e7ada456 ("block: Make request_queue lockdep
splats show up earlier"), I suspect change to hugetlb is needed.

[1] https://lore.kernel.org/lkml/6a02edcf.170a0220.7f9b3.000c.GAE@google.com/