From: Andrei Vagin <avagin@gmail.com>
To: linux-kernel@vger.kernel.org
Cc: Andrei Vagin <avagin@gmail.com>,
Andy Lutomirski <luto@amacapital.net>,
Christian Brauner <brauner@kernel.org>,
Dietmar Eggemann <dietmar.eggemann@arm.com>,
Kees Cook <keescook@chromium.org>, Ingo Molnar <mingo@redhat.com>,
Juri Lelli <juri.lelli@redhat.com>,
Peter Oskolkov <posk@google.com>,
Peter Zijlstra <peterz@infradead.org>,
Tycho Andersen <tycho@tycho.pizza>,
Will Drewry <wad@chromium.org>,
Vincent Guittot <vincent.guittot@linaro.org>
Subject: [PATCH 3/5] sched: add a few helpers to wake up tasks on the current cpu
Date: Wed, 19 Oct 2022 18:10:46 -0700 [thread overview]
Message-ID: <20221020011048.156415-4-avagin@gmail.com> (raw)
In-Reply-To: <20221020011048.156415-1-avagin@gmail.com>
Add complete_on_current_cpu, wake_up_poll_on_current_cpu helpers to wake
up processes on the current CPU.
Signed-off-by: Andrei Vagin <avagin@gmail.com>
---
include/linux/completion.h | 1 +
include/linux/swait.h | 1 +
include/linux/wait.h | 3 +++
kernel/sched/completion.c | 12 ++++++++++++
kernel/sched/core.c | 2 +-
kernel/sched/swait.c | 11 +++++++++++
kernel/sched/wait.c | 5 +++++
7 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/include/linux/completion.h b/include/linux/completion.h
index 62b32b19e0a8..fb2915676574 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -116,6 +116,7 @@ extern bool try_wait_for_completion(struct completion *x);
extern bool completion_done(struct completion *x);
extern void complete(struct completion *);
+extern void complete_on_current_cpu(struct completion *x);
extern void complete_all(struct completion *);
#endif
diff --git a/include/linux/swait.h b/include/linux/swait.h
index 6a8c22b8c2a5..1f27b254adf5 100644
--- a/include/linux/swait.h
+++ b/include/linux/swait.h
@@ -147,6 +147,7 @@ static inline bool swq_has_sleeper(struct swait_queue_head *wq)
extern void swake_up_one(struct swait_queue_head *q);
extern void swake_up_all(struct swait_queue_head *q);
extern void swake_up_locked(struct swait_queue_head *q);
+extern void swake_up_locked_on_current_cpu(struct swait_queue_head *q);
extern void prepare_to_swait_exclusive(struct swait_queue_head *q, struct swait_queue *wait, int state);
extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state);
diff --git a/include/linux/wait.h b/include/linux/wait.h
index 7f5a51aae0a7..c7d3e78a500d 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -210,6 +210,7 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq
}
void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
+void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
unsigned int mode, void *key, wait_queue_entry_t *bookmark);
@@ -237,6 +238,8 @@ void __wake_up_pollfree(struct wait_queue_head *wq_head);
#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
#define wake_up_poll(x, m) \
__wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
+#define wake_up_poll_on_current_cpu(x, m) \
+ __wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m))
#define wake_up_locked_poll(x, m) \
__wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
#define wake_up_interruptible_poll(x, m) \
diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c
index d57a5c1c1cd9..a1931a79c05a 100644
--- a/kernel/sched/completion.c
+++ b/kernel/sched/completion.c
@@ -38,6 +38,18 @@ void complete(struct completion *x)
}
EXPORT_SYMBOL(complete);
+void complete_on_current_cpu(struct completion *x)
+{
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&x->wait.lock, flags);
+
+ if (x->done != UINT_MAX)
+ x->done++;
+ swake_up_locked_on_current_cpu(&x->wait);
+ raw_spin_unlock_irqrestore(&x->wait.lock, flags);
+}
+
/**
* complete_all: - signals all threads waiting on this completion
* @x: holds the state of this particular completion
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index cffa8f314c9a..1412470216c3 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6822,7 +6822,7 @@ asmlinkage __visible void __sched preempt_schedule_irq(void)
int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
void *key)
{
- WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC);
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~(WF_SYNC|WF_CURRENT_CPU));
return try_to_wake_up(curr->private, mode, wake_flags);
}
EXPORT_SYMBOL(default_wake_function);
diff --git a/kernel/sched/swait.c b/kernel/sched/swait.c
index 76b9b796e695..9ebe23868942 100644
--- a/kernel/sched/swait.c
+++ b/kernel/sched/swait.c
@@ -31,6 +31,17 @@ void swake_up_locked(struct swait_queue_head *q)
}
EXPORT_SYMBOL(swake_up_locked);
+void swake_up_locked_on_current_cpu(struct swait_queue_head *q)
+{
+ struct swait_queue *curr;
+
+ if (list_empty(&q->task_list))
+ return;
+
+ curr = list_first_entry(&q->task_list, typeof(*curr), task_list);
+ try_to_wake_up(curr->task, TASK_NORMAL, WF_CURRENT_CPU);
+ list_del_init(&curr->task_list);
+}
/*
* Wake up all waiters. This is an interface which is solely exposed for
* completions and not for general usage.
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index 9860bb9a847c..9a78bca79419 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -157,6 +157,11 @@ void __wake_up(struct wait_queue_head *wq_head, unsigned int mode,
}
EXPORT_SYMBOL(__wake_up);
+void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key)
+{
+ __wake_up_common_lock(wq_head, mode, 1, WF_CURRENT_CPU, key);
+}
+
/*
* Same as __wake_up but called with the spinlock in wait_queue_head_t held.
*/
--
2.37.2
next prev parent reply other threads:[~2022-10-20 1:11 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-20 1:10 [PATCH 0/5 v2] seccomp: add the synchronous mode for seccomp_unotify Andrei Vagin
2022-10-20 1:10 ` [PATCH 1/5] seccomp: don't use semaphore and wait_queue together Andrei Vagin
2022-10-20 5:10 ` Kees Cook
2022-10-21 4:30 ` Andrei Vagin
2022-10-20 1:10 ` [PATCH 2/5] sched: add WF_CURRENT_CPU and externise ttwu Andrei Vagin
2022-10-20 4:54 ` Kees Cook
2022-10-21 0:48 ` Andrei Vagin
2022-10-20 1:10 ` Andrei Vagin [this message]
2022-10-20 14:15 ` [PATCH 3/5] sched: add a few helpers to wake up tasks on the current cpu Peter Zijlstra
2022-10-21 0:44 ` Andrei Vagin
2022-10-27 6:51 ` Andrei Vagin
2022-10-20 1:10 ` [PATCH 4/5] seccomp: add the synchronous mode for seccomp_unotify Andrei Vagin
2022-10-20 1:10 ` [PATCH 5/5] selftest/seccomp: add a new test for the sync mode of seccomp_user_notify Andrei Vagin
2022-10-20 5:04 ` Kees Cook
2022-10-21 0:53 ` Andrei Vagin
-- strict thread matches above, loose matches on Subject: below --
2022-11-11 7:31 [PATCH 0/5 v3] seccomp: add the synchronous mode for seccomp_unotify Andrei Vagin
2022-11-11 7:31 ` [PATCH 3/5] sched: add a few helpers to wake up tasks on the current cpu Andrei Vagin
2023-01-10 21:30 [PATCH 0/5 v3 RESEND] seccomp: add the synchronous mode for seccomp_unotify Andrei Vagin
2023-01-10 21:30 ` [PATCH 3/5] sched: add a few helpers to wake up tasks on the current cpu Andrei Vagin
2023-01-16 9:59 ` Peter Zijlstra
2023-01-19 6:45 ` Andrei Vagin
2023-01-19 10:09 ` Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221020011048.156415-4-avagin@gmail.com \
--to=avagin@gmail.com \
--cc=brauner@kernel.org \
--cc=dietmar.eggemann@arm.com \
--cc=juri.lelli@redhat.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@amacapital.net \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=posk@google.com \
--cc=tycho@tycho.pizza \
--cc=vincent.guittot@linaro.org \
--cc=wad@chromium.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox