Re: [PATCH] mm/hugetlb: fix deadlock in __hugetlb_zap_begin() by using trylock

[jane.chu@oracle.com]

On 5/13/2026 2:19 PM, Kartik Nair wrote:
> syzbot reported a circular locking dependency involving
> resv_map->rw_sema and mmap_lock:
> 
>    CPU0                          CPU1
>    lock(&mm->mmap_lock)
>                                  lock(sk_lock-AF_INET6)
>                                  lock(&mm->mmap_lock)
>    lock(&resv_map->rw_sema)
> 
> __hugetlb_zap_begin() calls hugetlb_vma_lock_write() which does a
> blocking down_write() on either vma_lock->rw_sema or
> resv_map->rw_sema while mmap_lock is already held for write by the
> caller chain (vm_mmap_pgoff -> mmap_region -> __mmap_region ->
> unmap_region -> unmap_vmas -> hugetlb_zap_begin).
> 
> Fix this by converting __hugetlb_zap_begin() to use
> hugetlb_vma_trylock_write() instead of hugetlb_vma_lock_write().
> If the trylock fails, return false to the callers so they can skip
> the zap operation safely. Update hugetlb_zap_begin() and its callers
> in unmap_vmas() and zap_vma_range_batched() accordingly.

But where exactly is the Chain
   &resv_map->rw_sema --> sk_lock-AF_INET6 --> &mm->mmap_lock
in the calling stack on CPU1 ?

thanks,
-jane


> 
> Reported-by: syzbot+bd6aaf99e8443d8a9034@syzkaller.appspotmail.com
> Closes: https://syzkaller.appspot.com/bug?extid=bd6aaf99e8443d8a9034
> Signed-off-by: Kartik Nair <contact.kartikn@gmail.com>
> ---
>   include/linux/hugetlb.h | 10 ++++++----
>   mm/hugetlb.c            |  8 +++++---
>   mm/memory.c             | 10 ++++++----
>   3 files changed, 17 insertions(+), 11 deletions(-)
> 
> diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
> index 93418625d3c5..1972464bd92f 100644
> --- a/include/linux/hugetlb.h
> +++ b/include/linux/hugetlb.h
> @@ -244,16 +244,17 @@ void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma);
>   void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
>   				unsigned long *start, unsigned long *end);
>   
> -extern void __hugetlb_zap_begin(struct vm_area_struct *vma,
> +extern bool __hugetlb_zap_begin(struct vm_area_struct *vma,
>   				unsigned long *begin, unsigned long *end);
>   extern void __hugetlb_zap_end(struct vm_area_struct *vma,
>   			      struct zap_details *details);
>   
> -static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
> +static inline bool hugetlb_zap_begin(struct vm_area_struct *vma,
>   				     unsigned long *start, unsigned long *end)
>   {
>   	if (is_vm_hugetlb_page(vma))
> -		__hugetlb_zap_begin(vma, start, end);
> +		return __hugetlb_zap_begin(vma, start, end);
> +	return true;
>   }
>   
>   static inline void hugetlb_zap_end(struct vm_area_struct *vma,
> @@ -318,10 +319,11 @@ static inline void adjust_range_if_pmd_sharing_possible(
>   {
>   }
>   
> -static inline void hugetlb_zap_begin(
> +static inline bool hugetlb_zap_begin(
>   				struct vm_area_struct *vma,
>   				unsigned long *start, unsigned long *end)
>   {
> +	return true;
>   }
>   
>   static inline void hugetlb_zap_end(
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index f24bf49be047..dd55ec2ef007 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -5309,16 +5309,18 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
>   	huge_pmd_unshare_flush(tlb, vma);
>   }
>   
> -void __hugetlb_zap_begin(struct vm_area_struct *vma,
> +bool __hugetlb_zap_begin(struct vm_area_struct *vma,
>   			 unsigned long *start, unsigned long *end)
>   {
>   	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
> -		return;
> +		return false;
>   
>   	adjust_range_if_pmd_sharing_possible(vma, start, end);
> -	hugetlb_vma_lock_write(vma);
> +	if (!hugetlb_vma_trylock_write(vma))
> +		return false;
>   	if (vma->vm_file)
>   		i_mmap_lock_write(vma->vm_file->f_mapping);
> +	return true;
>   }
>   
>   void __hugetlb_zap_end(struct vm_area_struct *vma,
> diff --git a/mm/memory.c b/mm/memory.c
> index ea6568571131..c1451e5b6ee7 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -2158,9 +2158,10 @@ void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap)
>   		unsigned long start = max(vma->vm_start, unmap->vma_start);
>   		unsigned long end = min(vma->vm_end, unmap->vma_end);
>   
> -		hugetlb_zap_begin(vma, &start, &end);
> -		__zap_vma_range(tlb, vma, start, end, &details);
> -		hugetlb_zap_end(vma, &details);
> +		if (hugetlb_zap_begin(vma, &start, &end)) {
> +			__zap_vma_range(tlb, vma, start, end, &details);
> +			hugetlb_zap_end(vma, &details);
> +		}
>   		vma = mas_find(unmap->mas, unmap->tree_end - 1);
>   	} while (vma);
>   	mmu_notifier_invalidate_range_end(&range);
> @@ -2194,7 +2195,8 @@ void zap_vma_range_batched(struct mmu_gather *tlb,
>   
>   	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
>   				address, end);
> -	hugetlb_zap_begin(vma, &range.start, &range.end);
> +	if (!hugetlb_zap_begin(vma, &range.start, &range.end))
> +		return;
>   	update_hiwater_rss(vma->vm_mm);
>   	mmu_notifier_invalidate_range_start(&range);
>   	/*