* [PATCH 1/2] sbitmap: add helpers for add/del wait queue handling
2018-12-20 16:01 [PATCHSET v2] Fix kyber hang Jens Axboe
@ 2018-12-20 16:01 ` Jens Axboe
2018-12-20 16:01 ` [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers Jens Axboe
` (2 subsequent siblings)
3 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 16:01 UTC (permalink / raw)
To: linux-block; +Cc: ming.lei, osandov, Jens Axboe
After commit 5d2ee7122c73, users of sbitmap that need wait queue
handling must use the provided helpers. But we only added
prepare_to_wait()/finish_wait() style helpers, add the equivalent
add_wait_queue/list_del wrappers as we..
This is needed to ensure kyber plays by the sbitmap waitqueue
rules.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
include/linux/sbitmap.h | 16 ++++++++++++++--
lib/sbitmap.c | 30 ++++++++++++++++++++++++++----
2 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h
index 03f50fcedc79..14d558146aea 100644
--- a/include/linux/sbitmap.h
+++ b/include/linux/sbitmap.h
@@ -560,13 +560,13 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq);
void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m);
struct sbq_wait {
- int accounted;
+ struct sbitmap_queue *sbq; /* if set, sbq_wait is accounted */
struct wait_queue_entry wait;
};
#define DEFINE_SBQ_WAIT(name) \
struct sbq_wait name = { \
- .accounted = 0, \
+ .sbq = NULL, \
.wait = { \
.private = current, \
.func = autoremove_wake_function, \
@@ -588,4 +588,16 @@ void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait);
+/*
+ * Wrapper around add_wait_queue(), which maintains some extra internal state
+ */
+void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
+ struct sbq_wait_state *ws,
+ struct sbq_wait *sbq_wait);
+
+/*
+ * Must be paired with sbitmap_add_wait_queue()
+ */
+void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait);
+
#endif /* __LINUX_SCALE_BITMAP_H */
diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index 5b3e56d68dab..65c2d06250a6 100644
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -671,13 +671,35 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
}
EXPORT_SYMBOL_GPL(sbitmap_queue_show);
+void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
+ struct sbq_wait_state *ws,
+ struct sbq_wait *sbq_wait)
+{
+ if (!sbq_wait->sbq) {
+ sbq_wait->sbq = sbq;
+ atomic_inc(&sbq->ws_active);
+ }
+ add_wait_queue(&ws->wait, &sbq_wait->wait);
+}
+EXPORT_SYMBOL_GPL(sbitmap_add_wait_queue);
+
+void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait)
+{
+ list_del_init(&sbq_wait->wait.entry);
+ if (sbq_wait->sbq) {
+ atomic_dec(&sbq_wait->sbq->ws_active);
+ sbq_wait->sbq = NULL;
+ }
+}
+EXPORT_SYMBOL_GPL(sbitmap_del_wait_queue);
+
void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait, int state)
{
- if (!sbq_wait->accounted) {
+ if (!sbq_wait->sbq) {
atomic_inc(&sbq->ws_active);
- sbq_wait->accounted = 1;
+ sbq_wait->sbq = sbq;
}
prepare_to_wait_exclusive(&ws->wait, &sbq_wait->wait, state);
}
@@ -687,9 +709,9 @@ void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait)
{
finish_wait(&ws->wait, &sbq_wait->wait);
- if (sbq_wait->accounted) {
+ if (sbq_wait->sbq) {
atomic_dec(&sbq->ws_active);
- sbq_wait->accounted = 0;
+ sbq_wait->sbq = NULL;
}
}
EXPORT_SYMBOL_GPL(sbitmap_finish_wait);
--
2.17.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers
2018-12-20 16:01 [PATCHSET v2] Fix kyber hang Jens Axboe
2018-12-20 16:01 ` [PATCH 1/2] sbitmap: add helpers for add/del wait queue handling Jens Axboe
@ 2018-12-20 16:01 ` Jens Axboe
2018-12-20 16:56 ` [PATCHSET v2] Fix kyber hang Ming Lei
2018-12-20 19:15 ` Omar Sandoval
3 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 16:01 UTC (permalink / raw)
To: linux-block; +Cc: ming.lei, osandov, Jens Axboe
sbq_wake_ptr() checks sbq->ws_active to know if it needs to loop
the wait indexes or not. This requires the use of the sbitmap
waitqueue wrappers, but kyber doesn't use those for its domain
token waitqueue handling.
Convert kyber to use the helpers. This fixes a hang with waiting
for domain tokens.
Fixes: 5d2ee7122c73 ("sbitmap: optimize wakeup check")
Reported-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
block/kyber-iosched.c | 28 +++++++++++++++-------------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index de78e8aa7b0a..ec6a04e01bc1 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -195,7 +195,7 @@ struct kyber_hctx_data {
unsigned int batching;
struct kyber_ctx_queue *kcqs;
struct sbitmap kcq_map[KYBER_NUM_DOMAINS];
- wait_queue_entry_t domain_wait[KYBER_NUM_DOMAINS];
+ struct sbq_wait domain_wait[KYBER_NUM_DOMAINS];
struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];
atomic_t wait_index[KYBER_NUM_DOMAINS];
};
@@ -501,10 +501,11 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
INIT_LIST_HEAD(&khd->rqs[i]);
- init_waitqueue_func_entry(&khd->domain_wait[i],
+ khd->domain_wait[i].sbq = NULL;
+ init_waitqueue_func_entry(&khd->domain_wait[i].wait,
kyber_domain_wake);
- khd->domain_wait[i].private = hctx;
- INIT_LIST_HEAD(&khd->domain_wait[i].entry);
+ khd->domain_wait[i].wait.private = hctx;
+ INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);
atomic_set(&khd->wait_index[i], 0);
}
@@ -698,12 +699,13 @@ static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,
flush_busy_kcq, &data);
}
-static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
+static int kyber_domain_wake(wait_queue_entry_t *wqe, unsigned mode, int flags,
void *key)
{
- struct blk_mq_hw_ctx *hctx = READ_ONCE(wait->private);
+ struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);
+ struct sbq_wait *wait = container_of(wqe, struct sbq_wait, wait);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
blk_mq_run_hw_queue(hctx, true);
return 1;
}
@@ -714,7 +716,7 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
{
unsigned int sched_domain = khd->cur_domain;
struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];
- wait_queue_entry_t *wait = &khd->domain_wait[sched_domain];
+ struct sbq_wait *wait = &khd->domain_wait[sched_domain];
struct sbq_wait_state *ws;
int nr;
@@ -725,11 +727,11 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* run when one becomes available. Note that this is serialized on
* khd->lock, but we still need to be careful about the waker.
*/
- if (nr < 0 && list_empty_careful(&wait->entry)) {
+ if (nr < 0 && list_empty_careful(&wait->wait.entry)) {
ws = sbq_wait_ptr(domain_tokens,
&khd->wait_index[sched_domain]);
khd->domain_ws[sched_domain] = ws;
- add_wait_queue(&ws->wait, wait);
+ sbitmap_add_wait_queue(domain_tokens, ws, wait);
/*
* Try again in case a token was freed before we got on the wait
@@ -745,10 +747,10 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* between the !list_empty_careful() check and us grabbing the lock, but
* list_del_init() is okay with that.
*/
- if (nr >= 0 && !list_empty_careful(&wait->entry)) {
+ if (nr >= 0 && !list_empty_careful(&wait->wait.entry)) {
ws = khd->domain_ws[sched_domain];
spin_lock_irq(&ws->wait.lock);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
spin_unlock_irq(&ws->wait.lock);
}
@@ -951,7 +953,7 @@ static int kyber_##name##_waiting_show(void *data, struct seq_file *m) \
{ \
struct blk_mq_hw_ctx *hctx = data; \
struct kyber_hctx_data *khd = hctx->sched_data; \
- wait_queue_entry_t *wait = &khd->domain_wait[domain]; \
+ wait_queue_entry_t *wait = &khd->domain_wait[domain].wait; \
\
seq_printf(m, "%d\n", !list_empty_careful(&wait->entry)); \
return 0; \
--
2.17.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [PATCHSET v2] Fix kyber hang
2018-12-20 16:01 [PATCHSET v2] Fix kyber hang Jens Axboe
2018-12-20 16:01 ` [PATCH 1/2] sbitmap: add helpers for add/del wait queue handling Jens Axboe
2018-12-20 16:01 ` [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers Jens Axboe
@ 2018-12-20 16:56 ` Ming Lei
2018-12-20 17:37 ` Jens Axboe
2018-12-20 19:15 ` Omar Sandoval
3 siblings, 1 reply; 9+ messages in thread
From: Ming Lei @ 2018-12-20 16:56 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, osandov
On Thu, Dec 20, 2018 at 09:01:47AM -0700, Jens Axboe wrote:
> Since the wrong kyber patch was included in the first one, here's a
> v2 with just that one line corrected.
>
> First one adds the necessary sbitmap helpers for users to be able to
> do add_wait_queue() and list_del(wait_entry) type operations, and
> still work with the optimized wakeup checking.
>
> Second one converts kyber to use it, fixing a hang with domain tokens.
With the two patches, can't reproduce the io hang issue any more when
kyber is used.
Tested-by: Ming Lei <ming.lei@redhat.com>
Thanks,
Ming
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCHSET v2] Fix kyber hang
2018-12-20 16:56 ` [PATCHSET v2] Fix kyber hang Ming Lei
@ 2018-12-20 17:37 ` Jens Axboe
0 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 17:37 UTC (permalink / raw)
To: Ming Lei; +Cc: linux-block, osandov
On 12/20/18 9:56 AM, Ming Lei wrote:
> On Thu, Dec 20, 2018 at 09:01:47AM -0700, Jens Axboe wrote:
>> Since the wrong kyber patch was included in the first one, here's a
>> v2 with just that one line corrected.
>>
>> First one adds the necessary sbitmap helpers for users to be able to
>> do add_wait_queue() and list_del(wait_entry) type operations, and
>> still work with the optimized wakeup checking.
>>
>> Second one converts kyber to use it, fixing a hang with domain tokens.
>
> With the two patches, can't reproduce the io hang issue any more when
> kyber is used.
>
> Tested-by: Ming Lei <ming.lei@redhat.com>
Awesome, thanks for reporting and testing the patches!
--
Jens Axboe
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCHSET v2] Fix kyber hang
2018-12-20 16:01 [PATCHSET v2] Fix kyber hang Jens Axboe
` (2 preceding siblings ...)
2018-12-20 16:56 ` [PATCHSET v2] Fix kyber hang Ming Lei
@ 2018-12-20 19:15 ` Omar Sandoval
2018-12-20 19:16 ` Jens Axboe
3 siblings, 1 reply; 9+ messages in thread
From: Omar Sandoval @ 2018-12-20 19:15 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, ming.lei
On Thu, Dec 20, 2018 at 09:01:47AM -0700, Jens Axboe wrote:
> Since the wrong kyber patch was included in the first one, here's a
> v2 with just that one line corrected.
>
> First one adds the necessary sbitmap helpers for users to be able to
> do add_wait_queue() and list_del(wait_entry) type operations, and
> still work with the optimized wakeup checking.
>
> Second one converts kyber to use it, fixing a hang with domain tokens.
Lame, I should've caught this on the original patch. For the series:
Reviewed-by: Omar Sandoval <osandov@fb.com>
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCHSET v2] Fix kyber hang
2018-12-20 19:15 ` Omar Sandoval
@ 2018-12-20 19:16 ` Jens Axboe
0 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 19:16 UTC (permalink / raw)
To: Omar Sandoval; +Cc: linux-block, ming.lei
On 12/20/18 12:15 PM, Omar Sandoval wrote:
> On Thu, Dec 20, 2018 at 09:01:47AM -0700, Jens Axboe wrote:
>> Since the wrong kyber patch was included in the first one, here's a
>> v2 with just that one line corrected.
>>
>> First one adds the necessary sbitmap helpers for users to be able to
>> do add_wait_queue() and list_del(wait_entry) type operations, and
>> still work with the optimized wakeup checking.
>>
>> Second one converts kyber to use it, fixing a hang with domain tokens.
>
> Lame, I should've caught this on the original patch. For the series:
Yeah, somewhat puzzled that it wasn't...
> Reviewed-by: Omar Sandoval <osandov@fb.com>
Thanks!
--
Jens Axboe
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers
2018-12-20 15:55 [PATCHSET] " Jens Axboe
@ 2018-12-20 15:55 ` Jens Axboe
2018-12-20 16:00 ` Jens Axboe
0 siblings, 1 reply; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 15:55 UTC (permalink / raw)
To: linux-block; +Cc: ming.lei, osandov, Jens Axboe
sbq_wake_ptr() checks sbq->ws_active to know if it needs to loop
the wait indexes or not. This requires the use of the sbitmap
waitqueue wrappers, but kyber doesn't use those for its domain
token waitqueue handling.
Convert kyber to use the helpers. This fixes a hang with waiting
for domain tokens.
Fixes: 5d2ee7122c73 ("sbitmap: optimize wakeup check")
Reported-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
block/kyber-iosched.c | 28 +++++++++++++++-------------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index de78e8aa7b0a..b8b11e85b4f1 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -195,7 +195,7 @@ struct kyber_hctx_data {
unsigned int batching;
struct kyber_ctx_queue *kcqs;
struct sbitmap kcq_map[KYBER_NUM_DOMAINS];
- wait_queue_entry_t domain_wait[KYBER_NUM_DOMAINS];
+ struct sbq_wait domain_wait[KYBER_NUM_DOMAINS];
struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];
atomic_t wait_index[KYBER_NUM_DOMAINS];
};
@@ -501,10 +501,11 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
INIT_LIST_HEAD(&khd->rqs[i]);
- init_waitqueue_func_entry(&khd->domain_wait[i],
+ khd->domain_wait[i].accounted = 0;
+ init_waitqueue_func_entry(&khd->domain_wait[i].wait,
kyber_domain_wake);
- khd->domain_wait[i].private = hctx;
- INIT_LIST_HEAD(&khd->domain_wait[i].entry);
+ khd->domain_wait[i].wait.private = hctx;
+ INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);
atomic_set(&khd->wait_index[i], 0);
}
@@ -698,12 +699,13 @@ static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,
flush_busy_kcq, &data);
}
-static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
+static int kyber_domain_wake(wait_queue_entry_t *wqe, unsigned mode, int flags,
void *key)
{
- struct blk_mq_hw_ctx *hctx = READ_ONCE(wait->private);
+ struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);
+ struct sbq_wait *wait = container_of(wqe, struct sbq_wait, wait);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
blk_mq_run_hw_queue(hctx, true);
return 1;
}
@@ -714,7 +716,7 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
{
unsigned int sched_domain = khd->cur_domain;
struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];
- wait_queue_entry_t *wait = &khd->domain_wait[sched_domain];
+ struct sbq_wait *wait = &khd->domain_wait[sched_domain];
struct sbq_wait_state *ws;
int nr;
@@ -725,11 +727,11 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* run when one becomes available. Note that this is serialized on
* khd->lock, but we still need to be careful about the waker.
*/
- if (nr < 0 && list_empty_careful(&wait->entry)) {
+ if (nr < 0 && list_empty_careful(&wait->wait.entry)) {
ws = sbq_wait_ptr(domain_tokens,
&khd->wait_index[sched_domain]);
khd->domain_ws[sched_domain] = ws;
- add_wait_queue(&ws->wait, wait);
+ sbitmap_add_wait_queue(domain_tokens, ws, wait);
/*
* Try again in case a token was freed before we got on the wait
@@ -745,10 +747,10 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* between the !list_empty_careful() check and us grabbing the lock, but
* list_del_init() is okay with that.
*/
- if (nr >= 0 && !list_empty_careful(&wait->entry)) {
+ if (nr >= 0 && !list_empty_careful(&wait->wait.entry)) {
ws = khd->domain_ws[sched_domain];
spin_lock_irq(&ws->wait.lock);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
spin_unlock_irq(&ws->wait.lock);
}
@@ -951,7 +953,7 @@ static int kyber_##name##_waiting_show(void *data, struct seq_file *m) \
{ \
struct blk_mq_hw_ctx *hctx = data; \
struct kyber_hctx_data *khd = hctx->sched_data; \
- wait_queue_entry_t *wait = &khd->domain_wait[domain]; \
+ wait_queue_entry_t *wait = &khd->domain_wait[domain].wait; \
\
seq_printf(m, "%d\n", !list_empty_careful(&wait->entry)); \
return 0; \
--
2.17.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers
2018-12-20 15:55 ` [PATCH 2/2] kyber: use sbitmap add_wait_queue/list_del wait helpers Jens Axboe
@ 2018-12-20 16:00 ` Jens Axboe
0 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2018-12-20 16:00 UTC (permalink / raw)
To: linux-block; +Cc: ming.lei, osandov
On 12/20/18 8:55 AM, Jens Axboe wrote:
> sbq_wake_ptr() checks sbq->ws_active to know if it needs to loop
> the wait indexes or not. This requires the use of the sbitmap
> waitqueue wrappers, but kyber doesn't use those for its domain
> token waitqueue handling.
>
> Convert kyber to use the helpers. This fixes a hang with waiting
> for domain tokens.
>
> Fixes: 5d2ee7122c73 ("sbitmap: optimize wakeup check")
> Reported-by: Ming Lei <ming.lei@redhat.com>
> Signed-off-by: Jens Axboe <axboe@kernel.dk>
> ---
That sent out a wrong one, here's the right one...
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index de78e8aa7b0a..ec6a04e01bc1 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -195,7 +195,7 @@ struct kyber_hctx_data {
unsigned int batching;
struct kyber_ctx_queue *kcqs;
struct sbitmap kcq_map[KYBER_NUM_DOMAINS];
- wait_queue_entry_t domain_wait[KYBER_NUM_DOMAINS];
+ struct sbq_wait domain_wait[KYBER_NUM_DOMAINS];
struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];
atomic_t wait_index[KYBER_NUM_DOMAINS];
};
@@ -501,10 +501,11 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
INIT_LIST_HEAD(&khd->rqs[i]);
- init_waitqueue_func_entry(&khd->domain_wait[i],
+ khd->domain_wait[i].sbq = NULL;
+ init_waitqueue_func_entry(&khd->domain_wait[i].wait,
kyber_domain_wake);
- khd->domain_wait[i].private = hctx;
- INIT_LIST_HEAD(&khd->domain_wait[i].entry);
+ khd->domain_wait[i].wait.private = hctx;
+ INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);
atomic_set(&khd->wait_index[i], 0);
}
@@ -698,12 +699,13 @@ static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,
flush_busy_kcq, &data);
}
-static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
+static int kyber_domain_wake(wait_queue_entry_t *wqe, unsigned mode, int flags,
void *key)
{
- struct blk_mq_hw_ctx *hctx = READ_ONCE(wait->private);
+ struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);
+ struct sbq_wait *wait = container_of(wqe, struct sbq_wait, wait);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
blk_mq_run_hw_queue(hctx, true);
return 1;
}
@@ -714,7 +716,7 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
{
unsigned int sched_domain = khd->cur_domain;
struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];
- wait_queue_entry_t *wait = &khd->domain_wait[sched_domain];
+ struct sbq_wait *wait = &khd->domain_wait[sched_domain];
struct sbq_wait_state *ws;
int nr;
@@ -725,11 +727,11 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* run when one becomes available. Note that this is serialized on
* khd->lock, but we still need to be careful about the waker.
*/
- if (nr < 0 && list_empty_careful(&wait->entry)) {
+ if (nr < 0 && list_empty_careful(&wait->wait.entry)) {
ws = sbq_wait_ptr(domain_tokens,
&khd->wait_index[sched_domain]);
khd->domain_ws[sched_domain] = ws;
- add_wait_queue(&ws->wait, wait);
+ sbitmap_add_wait_queue(domain_tokens, ws, wait);
/*
* Try again in case a token was freed before we got on the wait
@@ -745,10 +747,10 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
* between the !list_empty_careful() check and us grabbing the lock, but
* list_del_init() is okay with that.
*/
- if (nr >= 0 && !list_empty_careful(&wait->entry)) {
+ if (nr >= 0 && !list_empty_careful(&wait->wait.entry)) {
ws = khd->domain_ws[sched_domain];
spin_lock_irq(&ws->wait.lock);
- list_del_init(&wait->entry);
+ sbitmap_del_wait_queue(wait);
spin_unlock_irq(&ws->wait.lock);
}
@@ -951,7 +953,7 @@ static int kyber_##name##_waiting_show(void *data, struct seq_file *m) \
{ \
struct blk_mq_hw_ctx *hctx = data; \
struct kyber_hctx_data *khd = hctx->sched_data; \
- wait_queue_entry_t *wait = &khd->domain_wait[domain]; \
+ wait_queue_entry_t *wait = &khd->domain_wait[domain].wait; \
\
seq_printf(m, "%d\n", !list_empty_careful(&wait->entry)); \
return 0; \
--
Jens Axboe
^ permalink raw reply related [flat|nested] 9+ messages in thread