[PATCH 0/2] disable I/O priority when prio_aging_expire is zero

[PATCH 0/2] disable I/O priority when prio_aging_expire is zero

[Ye Bin]

From: Ye Bin <yebin10@huawei.com>

Since the mq-deadline scheduler introduced support for I/O priorities, if
a process does not have an I/O priority configured, it becomes bound to
the process's scheduling priority. This change forces applications to
re-plan their I/O priorities. However, applications do not care about
I/O priorities, yet now they cannot disable priorities.

Ye Bin (2):
  block/mq-deadline: disable I/O priority when prio_aging_expire is zero
  block/mq-deadline: add module parameter for prio_aging_expire default

 block/mq-deadline.c | 84 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 71 insertions(+), 13 deletions(-)

-- 
2.34.1

[PATCH 1/2] block/mq-deadline: disable I/O priority when prio_aging_expire is zero

[Ye Bin]

From: Ye Bin <yebin10@huawei.com>

Since the mq-deadline scheduler introduced support for I/O priorities,
if a process does not have an I/O priority configured, it becomes bound
to the process's scheduling priority. This change forces applications to
re-plan their I/O priorities. However, applications do not care about I/O
priorities, yet now they cannot disable priorities.

Setting prio_aging_expire to zero does not actually turn off I/O
priority in mq-deadline. Instead the priority aging path in
dd_dispatch_prio_aged_requests() is invoked with "now - 0 == now",
which causes best-effort and idle requests to be dispatched ahead of
pending real-time requests through the aging path -- a classic
priority inversion, not the "priority disabled" behavior users expect
when writing zero.

Treat a zero prio_aging_expire as an explicit request to disable I/O
priority:

  * dd_insert_request() and dd_request_merge() file every request in
    the DD_BE_PRIO bucket, so the scheduler no longer distinguishes
    between RT, BE and IDLE classes.

  * dd_dispatch_prio_aged_requests() short-circuits when
    prio_aging_expire is zero, closing the aging path that caused the
    inversion.

  * dd_request_merged() and dd_merged_requests() now look up the
    per-priority bucket from rq->elv.priv[0] instead of recomputing it
    from the request ioprio.  Once priority is disabled the request
    ioprio no longer reflects the bucket the request lives in, so the
    old computation would touch the wrong rb-tree and FIFO list.

Switching the mode while I/O is in flight could itself invert
priorities, because requests already queued in the RT or IDLE buckets
would keep being dispatched by priority until they drain.  Follow the
same sequence used by elevator_switch(): when the sysfs store observes
a transition from a non-zero value to zero it freezes the queue (which
blocks new upper-layer I/O in blk_queue_enter() and waits for every
outstanding request to complete, draining the scheduler), quiesces the
queue so that no dispatch is in progress, flips prio_aging_expire to
zero, then unquiesces and unfreezes.  New I/O arriving after the
switch lands in the best-effort bucket.  Writing a non-zero value
re-enables priority without draining.

Signed-off-by: Ye Bin <yebin10@huawei.com>
---
 block/mq-deadline.c | 77 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 66 insertions(+), 11 deletions(-)

diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 824bfc17b2c6..e3314f7aed8b 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -83,6 +83,7 @@ struct deadline_data {
 	 * run time data
 	 */
 
+	struct request_queue *q;	/* associated request queue */
 	struct list_head dispatch;
 	struct dd_per_prio per_prio[DD_PRIO_COUNT];
 
@@ -184,10 +185,7 @@ static void deadline_remove_request(struct request_queue *q,
 static void dd_request_merged(struct request_queue *q, struct request *req,
 			      enum elv_merge type)
 {
-	struct deadline_data *dd = q->elevator->elevator_data;
-	const u8 ioprio_class = dd_rq_ioclass(req);
-	const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];
-	struct dd_per_prio *per_prio = &dd->per_prio[prio];
+	struct dd_per_prio *per_prio = req->elv.priv[0];
 
 	/*
 	 * if the merge was a front merge, we need to reposition request
@@ -205,12 +203,11 @@ static void dd_merged_requests(struct request_queue *q, struct request *req,
 			       struct request *next)
 {
 	struct deadline_data *dd = q->elevator->elevator_data;
-	const u8 ioprio_class = dd_rq_ioclass(next);
-	const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];
+	struct dd_per_prio *per_prio = next->elv.priv[0];
 
 	lockdep_assert_held(&dd->lock);
 
-	dd->per_prio[prio].stats.merged++;
+	per_prio->stats.merged++;
 
 	/*
 	 * if next expires before rq, assign its expire time to rq
@@ -227,7 +224,7 @@ static void dd_merged_requests(struct request_queue *q, struct request *req,
 	/*
 	 * kill knowledge of next, this one is a goner
 	 */
-	deadline_remove_request(q, &dd->per_prio[prio], next);
+	deadline_remove_request(q, per_prio, next);
 }
 
 /*
@@ -426,6 +423,15 @@ static struct request *dd_dispatch_prio_aged_requests(struct deadline_data *dd,
 
 	lockdep_assert_held(&dd->lock);
 
+	/*
+	 * A prio_aging_expire of zero disables I/O priority: all requests are
+	 * filed in the DD_BE_PRIO bucket and must never be dispatched ahead of
+	 * higher priority requests through this aging path, which would cause
+	 * priority inversion.
+	 */
+	if (dd->prio_aging_expire == 0)
+		return NULL;
+
 	prio_cnt = !!dd_queued(dd, DD_RT_PRIO) + !!dd_queued(dd, DD_BE_PRIO) +
 		   !!dd_queued(dd, DD_IDLE_PRIO);
 	if (prio_cnt < 2)
@@ -536,6 +542,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq)
 		return -ENOMEM;
 
 	eq->elevator_data = dd;
+	dd->q = q;
 
 	INIT_LIST_HEAD(&dd->dispatch);
 	for (prio = 0; prio <= DD_PRIO_MAX; prio++) {
@@ -573,7 +580,8 @@ static int dd_request_merge(struct request_queue *q, struct request **rq,
 {
 	struct deadline_data *dd = q->elevator->elevator_data;
 	const u8 ioprio_class = IOPRIO_PRIO_CLASS(bio->bi_ioprio);
-	const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];
+	const enum dd_prio prio = dd->prio_aging_expire == 0 ? DD_BE_PRIO :
+					  ioprio_class_to_prio[ioprio_class];
 	struct dd_per_prio *per_prio = &dd->per_prio[prio];
 	sector_t sector = bio_end_sector(bio);
 	struct request *__rq;
@@ -633,7 +641,15 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 
 	lockdep_assert_held(&dd->lock);
 
-	prio = ioprio_class_to_prio[ioprio_class];
+	/*
+	 * When prio_aging_expire is zero, I/O priority is disabled: file every
+	 * request in the best-effort bucket so that the dispatch path no longer
+	 * distinguishes between RT, BE and IDLE classes.
+	 */
+	if (dd->prio_aging_expire == 0)
+		prio = DD_BE_PRIO;
+	else
+		prio = ioprio_class_to_prio[ioprio_class];
 	per_prio = &dd->per_prio[prio];
 	if (!rq->elv.priv[0])
 		per_prio->stats.inserted++;
@@ -773,7 +789,6 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)
 	STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, msecs_to_jiffies)
 STORE_JIFFIES(deadline_read_expire_store, &dd->fifo_expire[DD_READ], 0, INT_MAX);
 STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MAX);
-STORE_JIFFIES(deadline_prio_aging_expire_store, &dd->prio_aging_expire, 0, INT_MAX);
 STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX);
 STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1);
 STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX);
@@ -781,6 +796,46 @@ STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX);
 #undef STORE_INT
 #undef STORE_JIFFIES
 
+/*
+ * Writing zero to prio_aging_expire disables I/O priority: all requests are
+ * treated as best-effort. To avoid priority inversion while the mode is being
+ * switched, first drain all in-flight I/O by following the same sequence used
+ * by elevator_switch(): freeze the queue so that new upper-layer I/O is
+ * blocked and all outstanding requests complete, quiesce the queue so that no
+ * dispatch is in progress, then flip prio_aging_expire to zero. New I/O
+ * queued after the switch lands in the best-effort bucket.
+ * Writing a non-zero value (re-)enables priority without draining.
+ */
+static ssize_t deadline_prio_aging_expire_store(struct elevator_queue *e,
+						 const char *page, size_t count)
+{
+	struct deadline_data *dd = e->elevator_data;
+	int val, ret;
+
+	ret = kstrtoint(page, 0, &val);
+	if (ret < 0)
+		return ret;
+	if (val < 0)
+		val = 0;
+	else if (val > INT_MAX)
+		val = INT_MAX;
+	val = msecs_to_jiffies(val);
+
+	if (val == 0 && dd->prio_aging_expire != 0) {
+		unsigned int memflags;
+
+		memflags = blk_mq_freeze_queue(dd->q);
+		blk_mq_quiesce_queue(dd->q);
+		dd->prio_aging_expire = 0;
+		blk_mq_unquiesce_queue(dd->q);
+		blk_mq_unfreeze_queue(dd->q, memflags);
+	} else {
+		dd->prio_aging_expire = val;
+	}
+
+	return count;
+}
+
 #define DD_ATTR(name) \
 	__ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store)
 
-- 
2.34.1

[PATCH 2/2] block/mq-deadline: add module parameter for prio_aging_expire default

[Ye Bin]

From: Ye Bin <yebin10@huawei.com>

Allow the default value of prio_aging_expire to be overridden at load
time, in milliseconds to match the sysfs attribute:

  - built-in:  mq_deadline.prio_aging_expire=0 on the kernel command line
  - module:    modprobe mq_deadline prio_aging_expire=0

A value of zero disables I/O priority from boot/load: every request is
filed in the best-effort bucket and the priority aging path is
bypassed, so systems that do not want RT/BE/IDLE distinction can opt
out without writing to sysfs after every queue creation.

Previously prio_aging_expire was a compile-time constant (10 * HZ)
with no way to change the default before the first request queue was
initialized.  Make the variable a module_param so that the override
works whether mq-deadline is built-in or compiled as a module, and
convert the millisecond value to jiffies in dd_init_sched() when
assigning the per-queue default.

Signed-off-by: Ye Bin <yebin10@huawei.com>
---
 block/mq-deadline.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index e3314f7aed8b..95a00a673be5 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -33,7 +33,10 @@ static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SO
  * Time after which to dispatch lower priority requests even if higher
  * priority requests are pending.
  */
-static const int prio_aging_expire = 10 * HZ;
+static int prio_aging_expire = 10 * MSEC_PER_SEC;
+module_param(prio_aging_expire, int, 0644);
+MODULE_PARM_DESC(prio_aging_expire,
+		 "Default prio_aging_expire in milliseconds; 0 disables I/O priority.");
 static const int writes_starved = 2;    /* max times reads can starve a write */
 static const int fifo_batch = 16;       /* # of sequential requests treated as one
 				     by the above parameters. For throughput. */
@@ -559,7 +562,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq)
 	dd->front_merges = 1;
 	dd->last_dir = DD_WRITE;
 	dd->fifo_batch = fifo_batch;
-	dd->prio_aging_expire = prio_aging_expire;
+	dd->prio_aging_expire = msecs_to_jiffies(prio_aging_expire);
 	spin_lock_init(&dd->lock);
 
 	/* We dispatch from request queue wide instead of hw queue */
-- 
2.34.1