[PATCH] dm-delay: do not round a short delay up to the next timer tick

[Mike Snitzer <snitzer@kernel.org>]

For delays under 50 ms dm-delay hands its bios to a kthread, which
polls for expired bios eight times per delay period.  But the time a
bio is due is still kept in jiffies:

	delayed->expires = jiffies + msecs_to_jiffies(c->delay);

so however often the kthread looks, a bio is released at a tick.  With
HZ=1000 that is invisible.  With HZ=100 every delay under 10 ms becomes
a delay until the next tick, and a task that writes synchronously
falls into step with the tick and sees 10 ms each time.

Keep the due time in nanoseconds when the kthread is in use.  The
timer and workqueue path, for delays of 50 ms and more, is unchanged.

Single 4 KiB direct writes through a delay target on a memory-backed
null_blk device, arm64, HZ=100, mean latency in ms (300 writes each):

	delay	before	after
	 1	 9.97	 1.16
	 2	 9.97	 2.38
	 5	 9.99	 5.85
	20	20.06	21.29

What remains above the requested delay is the kthread's polling
interval, one eighth of the delay.

Signed-off-by: Mike Snitzer <snitzer@kernel.org>
---
 drivers/md/dm-delay.c | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index 4864671c2222f..4c732b240801f 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -15,6 +15,7 @@
 #include <linux/slab.h>
 #include <linux/kthread.h>
 #include <linux/delay.h>
+#include <linux/timekeeping.h>
 
 #include <linux/device-mapper.h>
 
@@ -51,7 +52,15 @@ struct dm_delay_info {
 	struct delay_c *context;
 	struct delay_class *class;
 	struct list_head list;
-	unsigned long expires;
+	/*
+	 * When the bio is due: in jiffies for the timer, or in nanoseconds
+	 * for the kthread, which polls more often than the tick and must
+	 * not round a short delay up to it.
+	 */
+	union {
+		unsigned long expires;
+		u64 expires_ns;
+	};
 };
 
 static void handle_delayed_timer(struct timer_list *t)
@@ -83,6 +92,13 @@ static void flush_bios(struct bio *bio)
 	}
 }
 
+static bool delay_expired(struct delay_c *dc, struct dm_delay_info *delayed)
+{
+	if (delay_is_fast(dc))
+		return ktime_get_ns() >= delayed->expires_ns;
+	return time_after_eq(jiffies, delayed->expires);
+}
+
 static void flush_delayed_bios(struct delay_c *dc, bool flush_all)
 {
 	struct dm_delay_info *delayed, *next;
@@ -98,7 +114,7 @@ static void flush_delayed_bios(struct delay_c *dc, bool flush_all)
 	spin_unlock(&dc->delayed_bios_lock);
 	list_for_each_entry_safe(delayed, next, &local_list, list) {
 		cond_resched();
-		if (flush_all || time_after_eq(jiffies, delayed->expires)) {
+		if (flush_all || delay_expired(dc, delayed)) {
 			struct bio *bio = dm_bio_from_per_bio_data(delayed,
 						sizeof(struct dm_delay_info));
 			list_del(&delayed->list);
@@ -322,7 +338,10 @@ static int delay_bio(struct delay_c *dc, struct delay_class *c, struct bio *bio)
 	delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info));
 
 	delayed->context = dc;
-	delayed->expires = expires = jiffies + msecs_to_jiffies(c->delay);
+	if (delay_is_fast(dc))
+		delayed->expires_ns = ktime_get_ns() + (u64)c->delay * NSEC_PER_MSEC;
+	else
+		delayed->expires = expires = jiffies + msecs_to_jiffies(c->delay);
 
 	spin_lock(&dc->delayed_bios_lock);
 	if (unlikely(!dc->may_delay)) {
-- 
2.44.0