Linux-mm Archive on lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] mm/ksm: fix ksm_exit() deadlock on exec error path
@ 2026-11-06  1:39 Jinjiang Tu
  2026-11-06  8:28 ` Xu Xin (vivo)
                   ` (2 more replies)
  0 siblings, 3 replies; 7+ messages in thread
From: Jinjiang Tu @ 2026-11-06  1:39 UTC (permalink / raw)
  To: akpm, david, xu.xin, chengming.zhou, liam, ljs, vbabka, jannh,
	pfalcato, kees, ziy, gourry, anthony.yznaga, linux-mm
  Cc: wangkefeng.wang, sunnanyong, tujinjiang

create_init_stack_vma() registers the new mm with ksmd via ksm_execve()
while holding mmap_write_lock(), and calls ksm_exit() before dropping the
lock. When insert_vm_struct() fails, __ksm_exit() takes mmap_write_lock()
again when the mm is ksmd's current scan cursor, leading to deadlock.

The early registration is no longer needed after commit 590c03ca6a3f
("mm/ksm: fix exec/fork inheritance support for prctl"). ksm_vma_flags()
calls __ksm_enter() when the first mergeable VMA is created. Remove
ksm_execve() and its stubs.

Tested with ksm_functional_tests, all tests pass.

Fixes: 3a9e567ca45f ("mm/ksm: fix ksm exec support for prctl")
Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com>
---
 include/linux/ksm.h               | 13 -------------
 mm/ksm.c                          | 16 +++++++++-------
 mm/vma_exec.c                     | 10 ----------
 tools/testing/vma/include/stubs.h |  5 -----
 4 files changed, 9 insertions(+), 35 deletions(-)

diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index d39d0d5483a2..5e228935cede 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -66,14 +66,6 @@ static inline void ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
 	}
 }
 
-static inline int ksm_execve(struct mm_struct *mm)
-{
-	if (mm_flags_test(MMF_VM_MERGE_ANY, mm))
-		return __ksm_enter(mm);
-
-	return 0;
-}
-
 static inline void ksm_exit(struct mm_struct *mm)
 {
 	if (mm_flags_test(MMF_VM_MERGEABLE, mm))
@@ -118,11 +110,6 @@ static inline void ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
 {
 }
 
-static inline int ksm_execve(struct mm_struct *mm)
-{
-	return 0;
-}
-
 static inline void ksm_exit(struct mm_struct *mm)
 {
 }
diff --git a/mm/ksm.c b/mm/ksm.c
index dbfb8f98b189..2e0f894208d8 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -2777,11 +2777,11 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
 
 		mm_slot_free(mm_slot_cache, mm_slot);
 		/*
-		 * Only clear MMF_VM_MERGEABLE. We must not clear
-		 * MMF_VM_MERGE_ANY, because for those MMF_VM_MERGE_ANY process,
-		 * perhaps their mm_struct has just been added to ksm_mm_slot
-		 * list, and its process has not yet officially started running
-		 * or has not yet performed mmap/brk to allocate anonymous VMAS.
+		 * Only clear MMF_VM_MERGEABLE. MMF_VM_MERGE_ANY is the
+		 * per-process setting inherited across exec, so it must
+		 * stay set until userspace disables it with
+		 * prctl(PR_SET_MEMORY_MERGE, 0). A later mergeable VMA
+		 * re-registers the mm through ksm_vma_flags().
 		 */
 		mm_flags_clear(MMF_VM_MERGEABLE, mm);
 		mmap_read_unlock(mm);
@@ -2909,8 +2909,10 @@ vma_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file,
 		vma_flags_set(&vma_flags, VMA_MERGEABLE_BIT);
 		/*
 		 * Generally, the flags here always include MMF_VM_MERGEABLE.
-		 * However, in rare cases, this flag may be cleared by ksmd who
-		 * scans a cycle without finding any mergeable vma.
+		 * However, a newly execed process has not been added to
+		 * ksm_mm_slot. The first mergeable VMA registers the mm
+		 * with ksmd here. ksmd can also clear it after a full scan
+		 * cycle without finding any mergeable VMA.
 		 */
 		if (unlikely(!mm_flags_test(MMF_VM_MERGEABLE, mm)))
 			__ksm_enter(mm);
diff --git a/mm/vma_exec.c b/mm/vma_exec.c
index 586c52155942..2c644e18b55d 100644
--- a/mm/vma_exec.c
+++ b/mm/vma_exec.c
@@ -128,14 +128,6 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
 		goto err_free;
 	}
 
-	/*
-	 * Need to be called with mmap write lock
-	 * held, to avoid race with ksmd.
-	 */
-	err = ksm_execve(mm);
-	if (err)
-		goto err_ksm;
-
 	vma_flags_set_mask(&flags, VMA_STACK_FLAGS);
 	vma_set_anonymous(vma);
 
@@ -163,8 +155,6 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
 	return 0;
 
 err:
-	ksm_exit(mm);
-err_ksm:
 	mmap_write_unlock(mm);
 err_free:
 	*vmap = NULL;
diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h
index f786a585caf0..9ab4e524f78e 100644
--- a/tools/testing/vma/include/stubs.h
+++ b/tools/testing/vma/include/stubs.h
@@ -55,11 +55,6 @@ static inline void free_pgd_range(struct mmu_gather *tlb,
 {
 }
 
-static inline int ksm_execve(struct mm_struct *mm)
-{
-	return 0;
-}
-
 static inline void ksm_exit(struct mm_struct *mm)
 {
 }
-- 
2.54.0



^ permalink raw reply related	[flat|nested] 7+ messages in thread

end of thread, other threads:[~2026-11-09  1:53 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-11-06  1:39 [PATCH] mm/ksm: fix ksm_exit() deadlock on exec error path Jinjiang Tu
2026-11-06  8:28 ` Xu Xin (vivo)
2026-11-07  2:48   ` Jinjiang Tu
2026-11-06  8:46 ` Xu Xin (vivo)
2026-11-07  2:56   ` Jinjiang Tu
2026-11-07 19:06 ` David Hildenbrand (Arm)
2026-11-09  1:53   ` Jinjiang Tu

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox