[PATCH 2/4] task_work: don't rely on PF_EXITING

[Oleg Nesterov <oleg@redhat.com>]

task_work_add() checks PF_EXITING to ensure it can't insert the new
twork after exit_task_work(). This means that the exiting task can
not use it after exit_signals().

Change task_work_add() and task_work_run() to use the fake
TWORK_EXITED pointer to synchronize with each other, now we can
move exit_task_work() down and make task_work_add() more useful.

Unfortunately, this means that the exiting task needs to take
pi_lock unconditionally.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
---
 include/linux/task_work.h |    6 ++----
 kernel/task_work.c        |   15 ++++++++-------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/include/linux/task_work.h b/include/linux/task_work.h
index 03640fb..ef70b01 100644
--- a/include/linux/task_work.h
+++ b/include/linux/task_work.h
@@ -1,9 +1,8 @@
 #ifndef _LINUX_TASK_WORK_H
 #define _LINUX_TASK_WORK_H
 
-#include <linux/sched.h>
-
 struct task_work;
+struct task_struct;
 typedef void (*task_work_func_t)(struct task_work *);
 
 struct task_work {
@@ -25,8 +24,7 @@ void task_work_run(void);
 
 static inline void exit_task_work(struct task_struct *task)
 {
-	if (unlikely(task->last_twork))
-		task_work_run();
+	task_work_run();
 }
 
 #endif	/* _LINUX_TASK_WORK_H */
diff --git a/kernel/task_work.c b/kernel/task_work.c
index a0a3133..b2ff3b7 100644
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -6,6 +6,8 @@
  * task->last_twork points to the last node in the circular single-linked list.
  */
 
+#define TWORK_EXITED	((struct task_work *)1)
+
 int
 task_work_add(struct task_struct *task, struct task_work *twork, bool notify)
 {
@@ -18,12 +20,11 @@ task_work_add(struct task_struct *task, struct task_work *twork, bool notify)
 		return -ENOTSUPP;
 #endif
 	/*
-	 * We must not insert the new work if the task has already passed
-	 * exit_task_work(). We rely on do_exit()->raw_spin_unlock_wait()
-	 * and check PF_EXITING under pi_lock.
+	 * We must not insert the new work if the exiting task has already
+	 * passed task_work_run().
 	 */
 	raw_spin_lock_irqsave(&task->pi_lock, flags);
-	if (likely(!(task->flags & PF_EXITING))) {
+	if (likely(task->last_twork != TWORK_EXITED)) {
 		last = task->last_twork ?: twork;
 		task->last_twork = twork;
 		twork->next = last->next;
@@ -46,7 +47,7 @@ task_work_cancel(struct task_struct *task, task_work_func_t func)
 
 	raw_spin_lock_irqsave(&task->pi_lock, flags);
 	last = task->last_twork;
-	if (last) {
+	if (last && last != TWORK_EXITED) {
 		twork = last;
 		do {
 			prev = twork;
@@ -77,10 +78,10 @@ void task_work_run(void)
 
 	raw_spin_lock_irq(&task->pi_lock);
 	last = task->last_twork;
-	task->last_twork = NULL;
+	task->last_twork = (task->flags & PF_EXITING) ? TWORK_EXITED : NULL;
 	raw_spin_unlock_irq(&task->pi_lock);
 
-	if (unlikely(!last))
+	if (!last)
 		return;
 
 	next = last->next;
-- 
1.5.5.1