[RFC for-next 0/3] block: delay support for error injection

[RFC for-next 0/3] block: delay support for error injection

[Md Haris Iqbal]

Error injection can only fail a bio today.  This adds a delay_us option so
that a rule can hold a bio back first, to model a slow device.

The delay happens above the driver, so it is invisible to the I/O
statistics and never reaches the blk-mq timeout handler or SCSI error
handling.  What it does exercise is the code waiting above the block
layer: io_uring cancellation, hung task detection, and filesystem or
userspace timeouts.

Two things are worth a look.  A delayed bio is resubmitted below the
injection hook, so the rules are not applied to it again and it can never
pick up a status from another rule.  And holding a bio back reorders it
against bios submitted later, which breaks sequential write ordering on
zoned devices.  Both are documented in patch 3.

Patch 1 is a prep cleanup.  It moves the rejection of an unknown status
tag into the parser, because patch 2 makes a rule without a status valid.

Tested in a VM.

Md Haris Iqbal (3):
  block: reject unknown status tags in error injection rules
  block: allow error injection rules to delay bios
  Documentation: block: document error injection delays

 Documentation/block/error-injection.rst |  56 ++++++++-
 block/blk-core.c                        |  13 +-
 block/blk.h                             |   1 +
 block/error-injection.c                 | 154 +++++++++++++++++++++---
 4 files changed, 201 insertions(+), 23 deletions(-)

-- 
2.53.0

[RFC for-next 1/3] block: reject unknown status tags in error injection rules

[Md Haris Iqbal]

An unknown status tag leaves *status at BLK_STS_OK, which
error_inject_add() then rejects.  Fail in match_status() instead, so that
rejecting a bad tag does not rely on BLK_STS_OK being invalid for a rule.

For a single status= this does not change behaviour: an unknown tag still
fails the write with -EINVAL.  A repeated status= where an invalid tag
comes first is now rejected instead of being overridden by the later one.

Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Md Haris Iqbal <haris.iqbal@linux.dev>
---
 block/error-injection.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/block/error-injection.c b/block/error-injection.c
index e14bc4b723ef..47cdd8973adc 100644
--- a/block/error-injection.c
+++ b/block/error-injection.c
@@ -171,15 +171,18 @@ static int match_op(substring_t *args, enum req_op *op)
 static int match_status(substring_t *args, blk_status_t *status)
 {
 	const char *tag;
+	int ret = 0;
 
 	tag = match_strdup(args);
 	if (!tag)
 		return -ENOMEM;
 	*status = tag_to_blk_status(tag);
-	if (!*status)
+	if (!*status) {
 		pr_warn("invalid status '%s'\n", tag);
+		ret = -EINVAL;
+	}
 	kfree(tag);
-	return 0;
+	return ret;
 }
 
 static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
-- 
2.53.0

[RFC for-next 2/3] block: allow error injection rules to delay bios

[Md Haris Iqbal]

Error injection can only fail a bio today.  Add a delay_us option so that
a rule can hold a bio back first, to model a slow device.

If a matching rule has delay_us set, the bio is held for that long.  It is
then failed if the rule also has a status, or resubmitted below the
injection hook so that the rules are not applied to it again.

The bio is submitted from a workqueue rather than from the timer, because
submitting a bio can sleep.  Bios with REQ_NOWAIT are never delayed, and
values above 600 seconds are rejected.

Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Md Haris Iqbal <haris.iqbal@linux.dev>
---
 block/blk-core.c        |  13 ++--
 block/blk.h             |   1 +
 block/error-injection.c | 147 ++++++++++++++++++++++++++++++++++++----
 3 files changed, 142 insertions(+), 19 deletions(-)

diff --git a/block/blk-core.c b/block/blk-core.c
index 196bccf27f58..f28ccc4633d4 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -759,11 +759,8 @@ static void __submit_bio_noacct_mq(struct bio *bio)
 	current->bio_list = NULL;
 }
 
-void submit_bio_noacct_nocheck(struct bio *bio, bool split)
+void __submit_bio_noacct_nocheck(struct bio *bio, bool split)
 {
-	if (unlikely(blk_error_inject(bio)))
-		return;
-
 	blk_cgroup_bio_start(bio);
 
 	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
@@ -793,6 +790,14 @@ void submit_bio_noacct_nocheck(struct bio *bio, bool split)
 	}
 }
 
+void submit_bio_noacct_nocheck(struct bio *bio, bool split)
+{
+	if (unlikely(blk_error_inject(bio)))
+		return;
+
+	__submit_bio_noacct_nocheck(bio, split);
+}
+
 static blk_status_t blk_validate_atomic_write_op_size(struct request_queue *q,
 						 struct bio *bio)
 {
diff --git a/block/blk.h b/block/blk.h
index 50abfd932886..cdf6d8964da6 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -61,6 +61,7 @@ bool __blk_freeze_queue_start(struct request_queue *q,
 			      struct task_struct *owner);
 int __bio_queue_enter(struct request_queue *q, struct bio *bio);
 void submit_bio_noacct_nocheck(struct bio *bio, bool split);
+void __submit_bio_noacct_nocheck(struct bio *bio, bool split);
 int bio_submit_or_kill(struct bio *bio, unsigned int flags);
 
 static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
diff --git a/block/error-injection.c b/block/error-injection.c
index 47cdd8973adc..9b823edf3f8a 100644
--- a/block/error-injection.c
+++ b/block/error-injection.c
@@ -6,9 +6,17 @@
 #include <linux/blkdev.h>
 #include <linux/parser.h>
 #include <linux/seq_file.h>
+#include <linux/workqueue.h>
 #include "blk.h"
 #include "error-injection.h"
 
+/*
+ * Cap the delay so that a typo can't wedge a device for good.  This is still
+ * well beyond the default hung task timeout, which is one of the things a
+ * delay is useful for triggering.
+ */
+#define BLK_ERROR_INJECT_MAX_DELAY_US	(600 * USEC_PER_SEC)
+
 struct blk_error_inject {
 	struct list_head		entry;
 	sector_t			start;
@@ -18,14 +26,91 @@ struct blk_error_inject {
 
 	/* only inject every 1 / chance times */
 	unsigned int			chance;
+
+	/* hold the bio for this long before submitting or failing it */
+	unsigned int			delay_us;
 };
 
+/*
+ * A bio held by a delay rule.  This is self-contained on purpose: it does not
+ * point back at the rule, so rules can be removed while delayed bios are
+ * outstanding, and it does not point at the gendisk, so nothing has to be
+ * cleaned up when the disk goes away.  A delayed bio holds no queue usage
+ * counter reference either, so one that outlives its disk is failed by the
+ * GD_DEAD check in __bio_queue_enter() once it is finally submitted.
+ */
+struct blk_error_inject_delay {
+	struct delayed_work		dwork;
+	struct bio			*bio;
+	blk_status_t			status;
+};
+
+static struct workqueue_struct *blk_error_inject_wq;
+
 DEFINE_STATIC_KEY_FALSE(blk_error_injection_enabled);
 
+static void blk_error_inject_delay_work(struct work_struct *work)
+{
+	struct blk_error_inject_delay *d = container_of(to_delayed_work(work),
+			struct blk_error_inject_delay, dwork);
+	struct bio *bio = d->bio;
+	blk_status_t status = d->status;
+
+	kfree(d);
+
+	if (status != BLK_STS_OK) {
+		bio->bi_status = status;
+		bio_endio(bio);
+	} else {
+		/*
+		 * Submit below the injection hook.  Re-entering it would match
+		 * the same rule again and the bio would never be issued, so a
+		 * bio that was delayed once skips error injection entirely
+		 * from here on, including any other rule that covers it.
+		 */
+		__submit_bio_noacct_nocheck(bio, false);
+	}
+}
+
+/*
+ * Hand the bio to a workqueue that submits or fails it once the delay has
+ * expired.  Both blk_mq_submit_bio() and ->submit_bio can sleep, so this can't
+ * be completed from the timer itself.
+ *
+ * Returns false if the bio can't be delayed, in which case the caller handles
+ * it immediately instead.
+ */
+static bool blk_error_inject_delay(struct gendisk *disk, struct bio *bio,
+		blk_status_t status, unsigned int delay_us)
+{
+	struct blk_error_inject_delay *d;
+
+	/* never block a bio that asked not to be blocked */
+	if (bio->bi_opf & REQ_NOWAIT)
+		return false;
+
+	d = kmalloc_obj(*d, GFP_NOIO);
+	if (!d)
+		return false;
+
+	pr_info_ratelimited("%pg: delaying %s at sector %llu:%u by %uus\n",
+			disk->part0, blk_op_str(bio_op(bio)),
+			bio->bi_iter.bi_sector, bio_sectors(bio), delay_us);
+
+	d->bio = bio;
+	d->status = status;
+	INIT_DELAYED_WORK(&d->dwork, blk_error_inject_delay_work);
+	queue_delayed_work(blk_error_inject_wq, &d->dwork,
+			usecs_to_jiffies(delay_us));
+	return true;
+}
+
 bool __blk_error_inject(struct bio *bio)
 {
 	struct gendisk *disk = bio->bi_bdev->bd_disk;
 	struct blk_error_inject *inj;
+	blk_status_t status = BLK_STS_OK;
+	unsigned int delay_us = 0;
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
@@ -45,29 +130,38 @@ bool __blk_error_inject(struct bio *bio)
 		if (inj->chance > 1 && (get_random_u32() % inj->chance) != 0)
 			continue;
 
-		pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
-				disk->part0, blk_status_to_str(inj->status),
-				blk_op_str(inj->op), bio->bi_iter.bi_sector,
-				bio_sectors(bio));
-		bio->bi_status = inj->status;
-		rcu_read_unlock();
-		bio_endio(bio);
-		return true;
+		status = inj->status;
+		delay_us = inj->delay_us;
+		break;
 	}
 	rcu_read_unlock();
-	return false;
+
+	if (delay_us && blk_error_inject_delay(disk, bio, status, delay_us))
+		return true;
+	if (status == BLK_STS_OK)
+		return false;
+
+	pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
+			disk->part0, blk_status_to_str(status),
+			blk_op_str(bio_op(bio)), bio->bi_iter.bi_sector,
+			bio_sectors(bio));
+	bio->bi_status = status;
+	bio_endio(bio);
+	return true;
 }
 
 static int error_inject_add(struct gendisk *disk, enum req_op op,
 		sector_t start, u64 nr_sectors, blk_status_t status,
-		unsigned int chance)
+		unsigned int chance, unsigned int delay_us)
 {
 	struct blk_error_inject *inj;
 	int error = -EINVAL;
 
 	if (op == REQ_OP_LAST)
 		return -EINVAL;
-	if (status == BLK_STS_OK)
+	if (status == BLK_STS_OK && !delay_us)
+		return -EINVAL;
+	if (delay_us > BLK_ERROR_INJECT_MAX_DELAY_US)
 		return -EINVAL;
 
 	inj = kzalloc_obj(*inj);
@@ -86,6 +180,7 @@ static int error_inject_add(struct gendisk *disk, enum req_op op,
 	inj->start = start;
 	inj->status = status;
 	inj->chance = chance;
+	inj->delay_us = delay_us;
 
 	pr_debug_ratelimited("%pg: adding %s injection for %s at sector %llu:%llu\n",
 			disk->part0, blk_status_to_str(status),
@@ -139,6 +234,7 @@ enum options {
 	Opt_nr_sectors		= (1u << 18),
 	Opt_status		= (1u << 19),
 	Opt_chance		= (1u << 20),
+	Opt_delay_us		= (1u << 21),
 
 	Opt_invalid,
 };
@@ -151,6 +247,7 @@ static const match_table_t opt_tokens = {
 	{ Opt_nr_sectors,		"nr_sectors=%u"		},
 	{ Opt_status,			"status=%s"		},
 	{ Opt_chance,			"chance=%u"		},
+	{ Opt_delay_us,			"delay_us=%u"		},
 	{ Opt_invalid,			NULL,			},
 };
 
@@ -189,7 +286,7 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
 		char *options)
 {
 	enum { Unset, Add, Removeall } action = Unset;
-	unsigned int option_mask = 0, chance = 1;
+	unsigned int option_mask = 0, chance = 1, delay_us = 0;
 	enum req_op op = REQ_OP_LAST;
 	u64 start = 0, nr_sectors = 0;
 	blk_status_t status = BLK_STS_OK;
@@ -232,6 +329,9 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
 			if (!error && chance == 0)
 				error = -EINVAL;
 			break;
+		case Opt_delay_us:
+			error = match_uint(args, &delay_us);
+			break;
 		default:
 			pr_warn("unknown parameter or missing value '%s'\n", p);
 			error = -EINVAL;
@@ -243,7 +343,7 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
 	switch (action) {
 	case Add:
 		return error_inject_add(disk, op, start, nr_sectors, status,
-				chance);
+				chance, delay_us);
 	case Removeall:
 		if (option_mask & ~Opt_removeall)
 			return -EINVAL;
@@ -279,10 +379,11 @@ static int blk_error_injection_show(struct seq_file *s, void *private)
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
-		seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
+		seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u,delay_us=%u",
 			   inj->start, inj->end,
 			   blk_op_str(inj->op),
-			   blk_status_to_tag(inj->status), inj->chance);
+			   blk_status_to_tag(inj->status), inj->chance,
+			   inj->delay_us);
 		seq_putc(s, '\n');
 	}
 	rcu_read_unlock();
@@ -317,3 +418,19 @@ void blk_error_injection_exit(struct gendisk *disk)
 {
 	error_inject_removeall(disk);
 }
+
+static int __init blk_error_injection_init_wq(void)
+{
+	/*
+	 * WQ_MEM_RECLAIM so that a delayed bio on the reclaim path can still
+	 * find a worker under memory pressure.  Note that this only guarantees
+	 * a worker exists, not that it is free: submitting a bio can block on
+	 * a queue freeze or on tag allocation, so a delayed bio can still be
+	 * held up behind another one.
+	 */
+	blk_error_inject_wq = alloc_workqueue("blk_error_inject", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
+	if (!blk_error_inject_wq)
+		panic("Failed to create blk_error_inject wq\n");
+	return 0;
+}
+subsys_initcall(blk_error_injection_init_wq);
-- 
2.53.0

[RFC for-next 3/3] Documentation: block: document error injection delays

[Md Haris Iqbal]

Document the delay_us option: the delay happens above the driver, a
delayed bio is not run through the rules again, and holding a bio back
reorders it against bios submitted later.

Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Md Haris Iqbal <haris.iqbal@linux.dev>
---
 Documentation/block/error-injection.rst | 56 ++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/Documentation/block/error-injection.rst b/Documentation/block/error-injection.rst
index 81f31af82e65..54490c23cde7 100644
--- a/Documentation/block/error-injection.rst
+++ b/Documentation/block/error-injection.rst
@@ -9,7 +9,8 @@ Overview
 
 Configurable error injection allows injecting specific block layer status codes
 for sector ranges of a block device.  Errors can be injected unconditionally, or
-with a given probability.
+with a given probability.  Instead of, or before, failing a bio it can also be
+held back for a while to model a slow device.
 
 To use configurable error injection, CONFIG_BLK_ERROR_INJECTION must be enabled.
 
@@ -34,15 +35,58 @@ op=<string>		block layer operation this rule applies to.  This uses
 			the XYZ for each REQ_OP_XYZ operation, e.g. READ, WRITE
 			or DISCARD. Mandatory.
 status=<string>		Status to return.  This uses XYZ for each BLK_STS_XYZ
-			code, e.g. IOERR or MEDIUM. Mandatory.
+			code, e.g. IOERR or MEDIUM. Mandatory unless delay_us
+			is given.
 start=<number>		First block layer sector the rule applies to.
 			Optional, defaults to 0.
 nr_sectors=<number>	Number of sectors this rule applies.
 			Optional, defaults to the remainder of the device.
 chance=<number>		Only return a failure with a likelihood of 1/chance.
 			Optional, defaults to 1 (always).
+delay_us=<number>	Hold the bio back for this many microseconds.  Without
+			status the bio is then submitted to the device as
+			usual, with status it is failed once the delay has
+			expired.  Optional, defaults to 0 (no delay).
+			Values above 600 seconds are rejected.
 ===================	=======================================================
 
+Delays
+------
+
+A delayed bio is held before it is submitted, so the device itself never sees a
+slow I/O: the delay is not visible to the driver, to the I/O statistics, or to
+anything else below submission such as writeback throttling.  Throttling by
+blk-throttle happens before a bio can be delayed, so it is not affected either.
+What it does exercise is everything waiting above the block layer, for instance
+io_uring cancellation, hung task detection, and filesystem or userspace
+timeouts.  Because the low level driver is not involved, a delay does not reach
+the blk-mq timeout handler or SCSI error handling.
+
+Once its delay expires a bio is submitted below the injection hook, so no rule
+is evaluated for it a second time.  A bio that matched a delay rule therefore
+never gets an error from another rule, even one covering the same sectors.  Put
+the delay and the status in a single rule to fail a bio after holding it back.
+
+A delayed bio is issued after bios submitted while it was held, which reorders
+the I/O stream.  On zoned devices this breaks sequential write ordering: zone
+write plugging happens below the injection hook, so the writes issued while a
+write is held reach the zone out of order and are failed as misaligned.  Only
+delay reads there.  For the same reason, delaying one half of a split bio
+issues it out of order with the other half.
+
+Bios that must not block are never delayed.  A bio with REQ_NOWAIT set is
+submitted, or failed with the rule's status, immediately.
+
+The delay is a lower bound for anything longer than a timer tick, and the timer
+wheel adds further slack as the delay grows.  Values shorter than a tick are of
+little use: they expire on the next tick, which is anywhere between now and one
+tick away.
+
+Removing rules does not release bios that are already being delayed by them;
+those run out on their own.  A delayed bio whose disk is removed in the meantime
+is not submitted until its delay expires, by which point the queue no longer
+accepts I/O, so it fails with EIO.
+
 Example
 -------
 
@@ -54,6 +98,14 @@ Return BLK_STS_MEDIUM for every write to /dev/nvme0n1:
 
 	$ echo 'add,op=WRITE,start=0,status=MEDIUM' > /sys/kernel/debug/block/nvme0n1/error_injection
 
+Delay every read of /dev/nvme0n1 by 10 milliseconds, then issue it normally:
+
+	$ echo 'add,op=READ,delay_us=10000' > /sys/kernel/debug/block/nvme0n1/error_injection
+
+Fail one in 100 writes with BLK_STS_TIMEOUT, but only after 30 seconds:
+
+	$ echo 'add,op=WRITE,status=TIMEOUT,chance=100,delay_us=30000000' > /sys/kernel/debug/block/nvme0n1/error_injection
+
 Remove all rules for /dev/nvme0n1:
 
 	$ echo 'removeall' > /sys/kernel/debug/block/nvme0n1/error_injection
-- 
2.53.0

Re: [RFC for-next 0/3] block: delay support for error injection

[Keith Busch]

On Thu, Aug 27, 2026 at 02:01:12AM +0200, Md Haris Iqbal wrote:
> Two things are worth a look.  A delayed bio is resubmitted below the
> injection hook, so the rules are not applied to it again and it can never
> pick up a status from another rule.  And holding a bio back reorders it
> against bios submitted later, which breaks sequential write ordering on
> zoned devices.  Both are documented in patch 3.

Would it be possible to do the delay on the completion side instead?
That should avoid those submission order problems.

Re: [RFC for-next 2/3] block: allow error injection rules to delay bios

[Haris Iqbal]

On 8/27/26 02:01, Md Haris Iqbal wrote:
> Error injection can only fail a bio today.  Add a delay_us option so that
> a rule can hold a bio back first, to model a slow device.
> 
> If a matching rule has delay_us set, the bio is held for that long.  It is
> then failed if the rule also has a status, or resubmitted below the
> injection hook so that the rules are not applied to it again.
> 
> The bio is submitted from a workqueue rather than from the timer, because
> submitting a bio can sleep.  Bios with REQ_NOWAIT are never delayed, and
> values above 600 seconds are rejected.
> 
> Cc: Christoph Hellwig <hch@lst.de>
> Signed-off-by: Md Haris Iqbal <haris.iqbal@linux.dev>
> ---
>   block/blk-core.c        |  13 ++--
>   block/blk.h             |   1 +
>   block/error-injection.c | 147 ++++++++++++++++++++++++++++++++++++----
>   3 files changed, 142 insertions(+), 19 deletions(-)
> 
> diff --git a/block/blk-core.c b/block/blk-core.c
> index 196bccf27f58..f28ccc4633d4 100644
> --- a/block/blk-core.c
> +++ b/block/blk-core.c
> @@ -759,11 +759,8 @@ static void __submit_bio_noacct_mq(struct bio *bio)
>   	current->bio_list = NULL;
>   }
>   
> -void submit_bio_noacct_nocheck(struct bio *bio, bool split)
> +void __submit_bio_noacct_nocheck(struct bio *bio, bool split)
>   {
> -	if (unlikely(blk_error_inject(bio)))
> -		return;
> -
>   	blk_cgroup_bio_start(bio);
>   
>   	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
> @@ -793,6 +790,14 @@ void submit_bio_noacct_nocheck(struct bio *bio, bool split)
>   	}
>   }
>   
> +void submit_bio_noacct_nocheck(struct bio *bio, bool split)
> +{
> +	if (unlikely(blk_error_inject(bio)))
> +		return;
> +
> +	__submit_bio_noacct_nocheck(bio, split);
> +}
> +
>   static blk_status_t blk_validate_atomic_write_op_size(struct request_queue *q,
>   						 struct bio *bio)
>   {
> diff --git a/block/blk.h b/block/blk.h
> index 50abfd932886..cdf6d8964da6 100644
> --- a/block/blk.h
> +++ b/block/blk.h
> @@ -61,6 +61,7 @@ bool __blk_freeze_queue_start(struct request_queue *q,
>   			      struct task_struct *owner);
>   int __bio_queue_enter(struct request_queue *q, struct bio *bio);
>   void submit_bio_noacct_nocheck(struct bio *bio, bool split);
> +void __submit_bio_noacct_nocheck(struct bio *bio, bool split);
>   int bio_submit_or_kill(struct bio *bio, unsigned int flags);
>   
>   static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
> diff --git a/block/error-injection.c b/block/error-injection.c
> index 47cdd8973adc..9b823edf3f8a 100644
> --- a/block/error-injection.c
> +++ b/block/error-injection.c
> @@ -6,9 +6,17 @@
>   #include <linux/blkdev.h>
>   #include <linux/parser.h>
>   #include <linux/seq_file.h>
> +#include <linux/workqueue.h>
>   #include "blk.h"
>   #include "error-injection.h"
>   
> +/*
> + * Cap the delay so that a typo can't wedge a device for good.  This is still
> + * well beyond the default hung task timeout, which is one of the things a
> + * delay is useful for triggering.
> + */
> +#define BLK_ERROR_INJECT_MAX_DELAY_US	(600 * USEC_PER_SEC)
> +
>   struct blk_error_inject {
>   	struct list_head		entry;
>   	sector_t			start;
> @@ -18,14 +26,91 @@ struct blk_error_inject {
>   
>   	/* only inject every 1 / chance times */
>   	unsigned int			chance;
> +
> +	/* hold the bio for this long before submitting or failing it */
> +	unsigned int			delay_us;
>   };
>   
> +/*
> + * A bio held by a delay rule.  This is self-contained on purpose: it does not
> + * point back at the rule, so rules can be removed while delayed bios are
> + * outstanding, and it does not point at the gendisk, so nothing has to be
> + * cleaned up when the disk goes away.  A delayed bio holds no queue usage
> + * counter reference either, so one that outlives its disk is failed by the
> + * GD_DEAD check in __bio_queue_enter() once it is finally submitted.
> + */
> +struct blk_error_inject_delay {
> +	struct delayed_work		dwork;
> +	struct bio			*bio;
> +	blk_status_t			status;
> +};
> +
> +static struct workqueue_struct *blk_error_inject_wq;
> +
>   DEFINE_STATIC_KEY_FALSE(blk_error_injection_enabled);
>   
> +static void blk_error_inject_delay_work(struct work_struct *work)
> +{
> +	struct blk_error_inject_delay *d = container_of(to_delayed_work(work),
> +			struct blk_error_inject_delay, dwork);
> +	struct bio *bio = d->bio;
> +	blk_status_t status = d->status;
> +
> +	kfree(d);
> +
> +	if (status != BLK_STS_OK) {
> +		bio->bi_status = status;
> +		bio_endio(bio);
> +	} else {
> +		/*
> +		 * Submit below the injection hook.  Re-entering it would match
> +		 * the same rule again and the bio would never be issued, so a
> +		 * bio that was delayed once skips error injection entirely
> +		 * from here on, including any other rule that covers it.
> +		 */
> +		__submit_bio_noacct_nocheck(bio, false);

According to Sashiko, this will cause split bios to go through multiple 
delays.

Initial look says that what Sashiko is saying is true. I will take a 
deeper look and get back.

> +	}
> +}
> +
> +/*
> + * Hand the bio to a workqueue that submits or fails it once the delay has
> + * expired.  Both blk_mq_submit_bio() and ->submit_bio can sleep, so this can't
> + * be completed from the timer itself.
> + *
> + * Returns false if the bio can't be delayed, in which case the caller handles
> + * it immediately instead.
> + */
> +static bool blk_error_inject_delay(struct gendisk *disk, struct bio *bio,
> +		blk_status_t status, unsigned int delay_us)
> +{
> +	struct blk_error_inject_delay *d;
> +
> +	/* never block a bio that asked not to be blocked */
> +	if (bio->bi_opf & REQ_NOWAIT)
> +		return false;
> +
> +	d = kmalloc_obj(*d, GFP_NOIO);
> +	if (!d)
> +		return false;
> +
> +	pr_info_ratelimited("%pg: delaying %s at sector %llu:%u by %uus\n",
> +			disk->part0, blk_op_str(bio_op(bio)),
> +			bio->bi_iter.bi_sector, bio_sectors(bio), delay_us);
> +
> +	d->bio = bio;
> +	d->status = status;
> +	INIT_DELAYED_WORK(&d->dwork, blk_error_inject_delay_work);
> +	queue_delayed_work(blk_error_inject_wq, &d->dwork,
> +			usecs_to_jiffies(delay_us));
> +	return true;
> +}
> +
>   bool __blk_error_inject(struct bio *bio)
>   {
>   	struct gendisk *disk = bio->bi_bdev->bd_disk;
>   	struct blk_error_inject *inj;
> +	blk_status_t status = BLK_STS_OK;
> +	unsigned int delay_us = 0;
>   
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
> @@ -45,29 +130,38 @@ bool __blk_error_inject(struct bio *bio)
>   		if (inj->chance > 1 && (get_random_u32() % inj->chance) != 0)
>   			continue;
>   
> -		pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
> -				disk->part0, blk_status_to_str(inj->status),
> -				blk_op_str(inj->op), bio->bi_iter.bi_sector,
> -				bio_sectors(bio));
> -		bio->bi_status = inj->status;
> -		rcu_read_unlock();
> -		bio_endio(bio);
> -		return true;
> +		status = inj->status;
> +		delay_us = inj->delay_us;
> +		break;
>   	}
>   	rcu_read_unlock();
> -	return false;
> +
> +	if (delay_us && blk_error_inject_delay(disk, bio, status, delay_us))
> +		return true;
> +	if (status == BLK_STS_OK)
> +		return false;
> +
> +	pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
> +			disk->part0, blk_status_to_str(status),
> +			blk_op_str(bio_op(bio)), bio->bi_iter.bi_sector,
> +			bio_sectors(bio));
> +	bio->bi_status = status;
> +	bio_endio(bio);
> +	return true;
>   }
>   
>   static int error_inject_add(struct gendisk *disk, enum req_op op,
>   		sector_t start, u64 nr_sectors, blk_status_t status,
> -		unsigned int chance)
> +		unsigned int chance, unsigned int delay_us)
>   {
>   	struct blk_error_inject *inj;
>   	int error = -EINVAL;
>   
>   	if (op == REQ_OP_LAST)
>   		return -EINVAL;
> -	if (status == BLK_STS_OK)
> +	if (status == BLK_STS_OK && !delay_us)
> +		return -EINVAL;
> +	if (delay_us > BLK_ERROR_INJECT_MAX_DELAY_US)
>   		return -EINVAL;
>   
>   	inj = kzalloc_obj(*inj);
> @@ -86,6 +180,7 @@ static int error_inject_add(struct gendisk *disk, enum req_op op,
>   	inj->start = start;
>   	inj->status = status;
>   	inj->chance = chance;
> +	inj->delay_us = delay_us;
>   
>   	pr_debug_ratelimited("%pg: adding %s injection for %s at sector %llu:%llu\n",
>   			disk->part0, blk_status_to_str(status),
> @@ -139,6 +234,7 @@ enum options {
>   	Opt_nr_sectors		= (1u << 18),
>   	Opt_status		= (1u << 19),
>   	Opt_chance		= (1u << 20),
> +	Opt_delay_us		= (1u << 21),
>   
>   	Opt_invalid,
>   };
> @@ -151,6 +247,7 @@ static const match_table_t opt_tokens = {
>   	{ Opt_nr_sectors,		"nr_sectors=%u"		},
>   	{ Opt_status,			"status=%s"		},
>   	{ Opt_chance,			"chance=%u"		},
> +	{ Opt_delay_us,			"delay_us=%u"		},
>   	{ Opt_invalid,			NULL,			},
>   };
>   
> @@ -189,7 +286,7 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
>   		char *options)
>   {
>   	enum { Unset, Add, Removeall } action = Unset;
> -	unsigned int option_mask = 0, chance = 1;
> +	unsigned int option_mask = 0, chance = 1, delay_us = 0;
>   	enum req_op op = REQ_OP_LAST;
>   	u64 start = 0, nr_sectors = 0;
>   	blk_status_t status = BLK_STS_OK;
> @@ -232,6 +329,9 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
>   			if (!error && chance == 0)
>   				error = -EINVAL;
>   			break;
> +		case Opt_delay_us:
> +			error = match_uint(args, &delay_us);
> +			break;
>   		default:
>   			pr_warn("unknown parameter or missing value '%s'\n", p);
>   			error = -EINVAL;
> @@ -243,7 +343,7 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
>   	switch (action) {
>   	case Add:
>   		return error_inject_add(disk, op, start, nr_sectors, status,
> -				chance);
> +				chance, delay_us);
>   	case Removeall:
>   		if (option_mask & ~Opt_removeall)
>   			return -EINVAL;
> @@ -279,10 +379,11 @@ static int blk_error_injection_show(struct seq_file *s, void *private)
>   
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
> -		seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
> +		seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u,delay_us=%u",
>   			   inj->start, inj->end,
>   			   blk_op_str(inj->op),
> -			   blk_status_to_tag(inj->status), inj->chance);
> +			   blk_status_to_tag(inj->status), inj->chance,
> +			   inj->delay_us);
>   		seq_putc(s, '\n');
>   	}
>   	rcu_read_unlock();
> @@ -317,3 +418,19 @@ void blk_error_injection_exit(struct gendisk *disk)
>   {
>   	error_inject_removeall(disk);
>   }
> +
> +static int __init blk_error_injection_init_wq(void)
> +{
> +	/*
> +	 * WQ_MEM_RECLAIM so that a delayed bio on the reclaim path can still
> +	 * find a worker under memory pressure.  Note that this only guarantees
> +	 * a worker exists, not that it is free: submitting a bio can block on
> +	 * a queue freeze or on tag allocation, so a delayed bio can still be
> +	 * held up behind another one.
> +	 */
> +	blk_error_inject_wq = alloc_workqueue("blk_error_inject", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
> +	if (!blk_error_inject_wq)
> +		panic("Failed to create blk_error_inject wq\n");
> +	return 0;
> +}
> +subsys_initcall(blk_error_injection_init_wq);

Re: [RFC for-next 0/3] block: delay support for error injection

[Haris Iqbal]

On 8/27/26 05:49, Keith Busch wrote:
> On Thu, Aug 27, 2026 at 02:01:12AM +0200, Md Haris Iqbal wrote:
>> Two things are worth a look.  A delayed bio is resubmitted below the
>> injection hook, so the rules are not applied to it again and it can never
>> pick up a status from another rule.  And holding a bio back reorders it
>> against bios submitted later, which breaks sequential write ordering on
>> zoned devices.  Both are documented in patch 3.
> 
> Would it be possible to do the delay on the completion side instead?
> That should avoid those submission order problems.

Seems not. At bio_endio, bi_size is 0 hence the comparison rule cannot 
be calculated. What can be done is to capture the decision to delay or 
not at submit, and then execute it at completion, but something (the 
same kmalloc_obj?) needs to carry this all the way.

Besides, the bio would have been written to the device already, meaning 
if the rule said to delay and then fail, the upper layer will see the 
failure, but the data would have landed in the disk. Maybe not the worst 
idea, but still semantically incorrect since the documentation claims 
that nothing is seen by the device.

One way would be to omit delay injection for all bios meant for zoned 
block devices.

Re: [RFC for-next 2/3] block: allow error injection rules to delay bios

[Haris Iqbal]

On 8/27/26 15:02, Haris Iqbal wrote:
> 
> 
> On 8/27/26 02:01, Md Haris Iqbal wrote:
>> Error injection can only fail a bio today.  Add a delay_us option so that
>> a rule can hold a bio back first, to model a slow device.
>>
>> If a matching rule has delay_us set, the bio is held for that long.  
>> It is
>> then failed if the rule also has a status, or resubmitted below the
>> injection hook so that the rules are not applied to it again.
>>
>> The bio is submitted from a workqueue rather than from the timer, because
>> submitting a bio can sleep.  Bios with REQ_NOWAIT are never delayed, and
>> values above 600 seconds are rejected.
>>
>> Cc: Christoph Hellwig <hch@lst.de>
>> Signed-off-by: Md Haris Iqbal <haris.iqbal@linux.dev>
>> ---
>>   block/blk-core.c        |  13 ++--
>>   block/blk.h             |   1 +
>>   block/error-injection.c | 147 ++++++++++++++++++++++++++++++++++++----
>>   3 files changed, 142 insertions(+), 19 deletions(-)
>>
>> diff --git a/block/blk-core.c b/block/blk-core.c
>> index 196bccf27f58..f28ccc4633d4 100644
>> --- a/block/blk-core.c
>> +++ b/block/blk-core.c
>> @@ -759,11 +759,8 @@ static void __submit_bio_noacct_mq(struct bio *bio)
>>       current->bio_list = NULL;
>>   }
>> -void submit_bio_noacct_nocheck(struct bio *bio, bool split)
>> +void __submit_bio_noacct_nocheck(struct bio *bio, bool split)
>>   {
>> -    if (unlikely(blk_error_inject(bio)))
>> -        return;
>> -
>>       blk_cgroup_bio_start(bio);
>>       if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
>> @@ -793,6 +790,14 @@ void submit_bio_noacct_nocheck(struct bio *bio, 
>> bool split)
>>       }
>>   }
>> +void submit_bio_noacct_nocheck(struct bio *bio, bool split)
>> +{
>> +    if (unlikely(blk_error_inject(bio)))
>> +        return;
>> +
>> +    __submit_bio_noacct_nocheck(bio, split);
>> +}
>> +
>>   static blk_status_t blk_validate_atomic_write_op_size(struct 
>> request_queue *q,
>>                            struct bio *bio)
>>   {
>> diff --git a/block/blk.h b/block/blk.h
>> index 50abfd932886..cdf6d8964da6 100644
>> --- a/block/blk.h
>> +++ b/block/blk.h
>> @@ -61,6 +61,7 @@ bool __blk_freeze_queue_start(struct request_queue *q,
>>                     struct task_struct *owner);
>>   int __bio_queue_enter(struct request_queue *q, struct bio *bio);
>>   void submit_bio_noacct_nocheck(struct bio *bio, bool split);
>> +void __submit_bio_noacct_nocheck(struct bio *bio, bool split);
>>   int bio_submit_or_kill(struct bio *bio, unsigned int flags);
>>   static inline bool blk_try_enter_queue(struct request_queue *q, bool 
>> pm)
>> diff --git a/block/error-injection.c b/block/error-injection.c
>> index 47cdd8973adc..9b823edf3f8a 100644
>> --- a/block/error-injection.c
>> +++ b/block/error-injection.c
>> @@ -6,9 +6,17 @@
>>   #include <linux/blkdev.h>
>>   #include <linux/parser.h>
>>   #include <linux/seq_file.h>
>> +#include <linux/workqueue.h>
>>   #include "blk.h"
>>   #include "error-injection.h"
>> +/*
>> + * Cap the delay so that a typo can't wedge a device for good.  This 
>> is still
>> + * well beyond the default hung task timeout, which is one of the 
>> things a
>> + * delay is useful for triggering.
>> + */
>> +#define BLK_ERROR_INJECT_MAX_DELAY_US    (600 * USEC_PER_SEC)
>> +
>>   struct blk_error_inject {
>>       struct list_head        entry;
>>       sector_t            start;
>> @@ -18,14 +26,91 @@ struct blk_error_inject {
>>       /* only inject every 1 / chance times */
>>       unsigned int            chance;
>> +
>> +    /* hold the bio for this long before submitting or failing it */
>> +    unsigned int            delay_us;
>>   };
>> +/*
>> + * A bio held by a delay rule.  This is self-contained on purpose: it 
>> does not
>> + * point back at the rule, so rules can be removed while delayed bios 
>> are
>> + * outstanding, and it does not point at the gendisk, so nothing has 
>> to be
>> + * cleaned up when the disk goes away.  A delayed bio holds no queue 
>> usage
>> + * counter reference either, so one that outlives its disk is failed 
>> by the
>> + * GD_DEAD check in __bio_queue_enter() once it is finally submitted.
>> + */
>> +struct blk_error_inject_delay {
>> +    struct delayed_work        dwork;
>> +    struct bio            *bio;
>> +    blk_status_t            status;
>> +};
>> +
>> +static struct workqueue_struct *blk_error_inject_wq;
>> +
>>   DEFINE_STATIC_KEY_FALSE(blk_error_injection_enabled);
>> +static void blk_error_inject_delay_work(struct work_struct *work)
>> +{
>> +    struct blk_error_inject_delay *d = 
>> container_of(to_delayed_work(work),
>> +            struct blk_error_inject_delay, dwork);
>> +    struct bio *bio = d->bio;
>> +    blk_status_t status = d->status;
>> +
>> +    kfree(d);
>> +
>> +    if (status != BLK_STS_OK) {
>> +        bio->bi_status = status;
>> +        bio_endio(bio);
>> +    } else {
>> +        /*
>> +         * Submit below the injection hook.  Re-entering it would match
>> +         * the same rule again and the bio would never be issued, so a
>> +         * bio that was delayed once skips error injection entirely
>> +         * from here on, including any other rule that covers it.
>> +         */
>> +        __submit_bio_noacct_nocheck(bio, false);
> 
> According to Sashiko, this will cause split bios to go through multiple 
> delays.
> 
> Initial look says that what Sashiko is saying is true. I will take a 
> deeper look and get back.

Tested and Confirmed. Bios that split, compound the delays.

We can add a BIO_ERROR_INJECTED flag and set it upon the first entry to 
__blk_error_inject for bios that should be delayed. Later split bio 
submissions will then be skipped.

> 
>> +    }
>> +}
>> +
>> +/*
>> + * Hand the bio to a workqueue that submits or fails it once the 
>> delay has
>> + * expired.  Both blk_mq_submit_bio() and ->submit_bio can sleep, so 
>> this can't
>> + * be completed from the timer itself.
>> + *
>> + * Returns false if the bio can't be delayed, in which case the 
>> caller handles
>> + * it immediately instead.
>> + */
>> +static bool blk_error_inject_delay(struct gendisk *disk, struct bio 
>> *bio,
>> +        blk_status_t status, unsigned int delay_us)
>> +{
>> +    struct blk_error_inject_delay *d;
>> +
>> +    /* never block a bio that asked not to be blocked */
>> +    if (bio->bi_opf & REQ_NOWAIT)
>> +        return false;
>> +
>> +    d = kmalloc_obj(*d, GFP_NOIO);
>> +    if (!d)
>> +        return false;
>> +
>> +    pr_info_ratelimited("%pg: delaying %s at sector %llu:%u by %uus\n",
>> +            disk->part0, blk_op_str(bio_op(bio)),
>> +            bio->bi_iter.bi_sector, bio_sectors(bio), delay_us);
>> +
>> +    d->bio = bio;
>> +    d->status = status;
>> +    INIT_DELAYED_WORK(&d->dwork, blk_error_inject_delay_work);
>> +    queue_delayed_work(blk_error_inject_wq, &d->dwork,
>> +            usecs_to_jiffies(delay_us));
>> +    return true;
>> +}
>> +
>>   bool __blk_error_inject(struct bio *bio)
>>   {
>>       struct gendisk *disk = bio->bi_bdev->bd_disk;
>>       struct blk_error_inject *inj;
>> +    blk_status_t status = BLK_STS_OK;
>> +    unsigned int delay_us = 0;
>>       rcu_read_lock();
>>       list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
>> @@ -45,29 +130,38 @@ bool __blk_error_inject(struct bio *bio)
>>           if (inj->chance > 1 && (get_random_u32() % inj->chance) != 0)
>>               continue;
>> -        pr_info_ratelimited("%pg: injecting %s error for %s at sector 
>> %llu:%u\n",
>> -                disk->part0, blk_status_to_str(inj->status),
>> -                blk_op_str(inj->op), bio->bi_iter.bi_sector,
>> -                bio_sectors(bio));
>> -        bio->bi_status = inj->status;
>> -        rcu_read_unlock();
>> -        bio_endio(bio);
>> -        return true;
>> +        status = inj->status;
>> +        delay_us = inj->delay_us;
>> +        break;
>>       }
>>       rcu_read_unlock();
>> -    return false;
>> +
>> +    if (delay_us && blk_error_inject_delay(disk, bio, status, delay_us))
>> +        return true;
>> +    if (status == BLK_STS_OK)
>> +        return false;
>> +
>> +    pr_info_ratelimited("%pg: injecting %s error for %s at sector 
>> %llu:%u\n",
>> +            disk->part0, blk_status_to_str(status),
>> +            blk_op_str(bio_op(bio)), bio->bi_iter.bi_sector,
>> +            bio_sectors(bio));
>> +    bio->bi_status = status;
>> +    bio_endio(bio);
>> +    return true;
>>   }
>>   static int error_inject_add(struct gendisk *disk, enum req_op op,
>>           sector_t start, u64 nr_sectors, blk_status_t status,
>> -        unsigned int chance)
>> +        unsigned int chance, unsigned int delay_us)
>>   {
>>       struct blk_error_inject *inj;
>>       int error = -EINVAL;
>>       if (op == REQ_OP_LAST)
>>           return -EINVAL;
>> -    if (status == BLK_STS_OK)
>> +    if (status == BLK_STS_OK && !delay_us)
>> +        return -EINVAL;
>> +    if (delay_us > BLK_ERROR_INJECT_MAX_DELAY_US)
>>           return -EINVAL;
>>       inj = kzalloc_obj(*inj);
>> @@ -86,6 +180,7 @@ static int error_inject_add(struct gendisk *disk, 
>> enum req_op op,
>>       inj->start = start;
>>       inj->status = status;
>>       inj->chance = chance;
>> +    inj->delay_us = delay_us;
>>       pr_debug_ratelimited("%pg: adding %s injection for %s at sector 
>> %llu:%llu\n",
>>               disk->part0, blk_status_to_str(status),
>> @@ -139,6 +234,7 @@ enum options {
>>       Opt_nr_sectors        = (1u << 18),
>>       Opt_status        = (1u << 19),
>>       Opt_chance        = (1u << 20),
>> +    Opt_delay_us        = (1u << 21),
>>       Opt_invalid,
>>   };
>> @@ -151,6 +247,7 @@ static const match_table_t opt_tokens = {
>>       { Opt_nr_sectors,        "nr_sectors=%u"        },
>>       { Opt_status,            "status=%s"        },
>>       { Opt_chance,            "chance=%u"        },
>> +    { Opt_delay_us,            "delay_us=%u"        },
>>       { Opt_invalid,            NULL,            },
>>   };
>> @@ -189,7 +286,7 @@ static ssize_t 
>> blk_error_injection_parse_options(struct gendisk *disk,
>>           char *options)
>>   {
>>       enum { Unset, Add, Removeall } action = Unset;
>> -    unsigned int option_mask = 0, chance = 1;
>> +    unsigned int option_mask = 0, chance = 1, delay_us = 0;
>>       enum req_op op = REQ_OP_LAST;
>>       u64 start = 0, nr_sectors = 0;
>>       blk_status_t status = BLK_STS_OK;
>> @@ -232,6 +329,9 @@ static ssize_t 
>> blk_error_injection_parse_options(struct gendisk *disk,
>>               if (!error && chance == 0)
>>                   error = -EINVAL;
>>               break;
>> +        case Opt_delay_us:
>> +            error = match_uint(args, &delay_us);
>> +            break;
>>           default:
>>               pr_warn("unknown parameter or missing value '%s'\n", p);
>>               error = -EINVAL;
>> @@ -243,7 +343,7 @@ static ssize_t 
>> blk_error_injection_parse_options(struct gendisk *disk,
>>       switch (action) {
>>       case Add:
>>           return error_inject_add(disk, op, start, nr_sectors, status,
>> -                chance);
>> +                chance, delay_us);
>>       case Removeall:
>>           if (option_mask & ~Opt_removeall)
>>               return -EINVAL;
>> @@ -279,10 +379,11 @@ static int blk_error_injection_show(struct 
>> seq_file *s, void *private)
>>       rcu_read_lock();
>>       list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
>> -        seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
>> +        seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u,delay_us=%u",
>>                  inj->start, inj->end,
>>                  blk_op_str(inj->op),
>> -               blk_status_to_tag(inj->status), inj->chance);
>> +               blk_status_to_tag(inj->status), inj->chance,
>> +               inj->delay_us);
>>           seq_putc(s, '\n');
>>       }
>>       rcu_read_unlock();
>> @@ -317,3 +418,19 @@ void blk_error_injection_exit(struct gendisk *disk)
>>   {
>>       error_inject_removeall(disk);
>>   }
>> +
>> +static int __init blk_error_injection_init_wq(void)
>> +{
>> +    /*
>> +     * WQ_MEM_RECLAIM so that a delayed bio on the reclaim path can 
>> still
>> +     * find a worker under memory pressure.  Note that this only 
>> guarantees
>> +     * a worker exists, not that it is free: submitting a bio can 
>> block on
>> +     * a queue freeze or on tag allocation, so a delayed bio can 
>> still be
>> +     * held up behind another one.
>> +     */
>> +    blk_error_inject_wq = alloc_workqueue("blk_error_inject", 
>> WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
>> +    if (!blk_error_inject_wq)
>> +        panic("Failed to create blk_error_inject wq\n");
>> +    return 0;
>> +}
>> +subsys_initcall(blk_error_injection_init_wq);
> 
>