[PATCH 2/3] block: switch to per-cpu in-flight counters

[PATCH 2/3] block: switch to per-cpu in-flight counters

[Mikulas Patocka]

[-- Attachment #1: block-percpu-inflight.patch --]
[-- Type: text/plain, Size: 8536 bytes --]

Now when part_round_stats is gone, we can switch to per-cpu in-flight
counters.

We use the local-atomic type local_t, so that if part_inc_in_flight or
part_dec_in_flight is reentrantly called from an interrupt, the value will
be correct.

The other counters could be corrupted due to reentrant interrupt, but the
corruption only results in slight counter skew - the in_flight counter
must be exact, so it needs local_t.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
 block/bio.c           |    4 ++--
 block/blk-core.c      |    4 ++--
 block/blk-merge.c     |    2 +-
 block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
 drivers/md/dm.c       |    4 +---
 include/linux/genhd.h |    7 ++++---
 6 files changed, 45 insertions(+), 23 deletions(-)

Index: linux-block/block/genhd.c
===================================================================
--- linux-block.orig/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
@@ -45,53 +45,76 @@ static void disk_add_events(struct gendi
 static void disk_del_events(struct gendisk *disk);
 static void disk_release_events(struct gendisk *disk);
 
-void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
+void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
 {
 	if (queue_is_mq(q))
 		return;
 
-	atomic_inc(&part->in_flight[rw]);
+	local_inc(per_cpu_ptr(part->dkstats, cpu)->in_flight);
 	if (part->partno)
-		atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);
+		local_inc(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
 }
 
-void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
+void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
 {
 	if (queue_is_mq(q))
 		return;
 
-	atomic_dec(&part->in_flight[rw]);
+	local_dec(per_cpu_ptr(part->dkstats, cpu)->in_flight);
 	if (part->partno)
-		atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);
+		local_dec(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
 }
 
 void part_in_flight(struct request_queue *q, struct hd_struct *part,
 		    unsigned int inflight[2])
 {
+	int cpu;
+
 	if (queue_is_mq(q)) {
 		blk_mq_in_flight(q, part, inflight);
 		return;
 	}
 
-	inflight[0] = atomic_read(&part->in_flight[0]) +
-			atomic_read(&part->in_flight[1]);
+	inflight[0] = 0;
+	for_each_possible_cpu(cpu) {
+		inflight[0] +=	local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[0]) +
+				local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[1]);
+	}
+	if ((int)inflight[0] < 0)
+		inflight[0] = 0;
+
 	if (part->partno) {
 		part = &part_to_disk(part)->part0;
-		inflight[1] = atomic_read(&part->in_flight[0]) +
-				atomic_read(&part->in_flight[1]);
+		inflight[1] = 0;
+		for_each_possible_cpu(cpu) {
+			inflight[1] +=	local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[0]) +
+					local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[1]);
+		}
+		if ((int)inflight[1] < 0)
+			inflight[1] = 0;
 	}
 }
 
 void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,
 		       unsigned int inflight[2])
 {
+	int cpu;
+
 	if (queue_is_mq(q)) {
 		blk_mq_in_flight_rw(q, part, inflight);
 		return;
 	}
 
-	inflight[0] = atomic_read(&part->in_flight[0]);
-	inflight[1] = atomic_read(&part->in_flight[1]);
+	inflight[0] = 0;
+	inflight[1] = 0;
+	for_each_possible_cpu(cpu) {
+		inflight[0] += local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[0]);
+		inflight[1] += local_read(&per_cpu_ptr(part->dkstats, cpu)->in_flight[1]);
+	}
+	if ((int)inflight[0] < 0)
+		inflight[0] = 0;
+	if ((int)inflight[1] < 0)
+		inflight[1] = 0;
 }
 
 struct hd_struct *__disk_get_part(struct gendisk *disk, int partno)
Index: linux-block/include/linux/genhd.h
===================================================================
--- linux-block.orig/include/linux/genhd.h	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/include/linux/genhd.h	2018-11-28 00:09:59.000000000 +0100
@@ -17,6 +17,7 @@
 #include <linux/percpu-refcount.h>
 #include <linux/uuid.h>
 #include <linux/blk_types.h>
+#include <asm/local.h>
 
 #ifdef CONFIG_BLOCK
 
@@ -89,6 +90,7 @@ struct disk_stats {
 	unsigned long merges[NR_STAT_GROUPS];
 	unsigned long io_ticks;
 	unsigned long time_in_queue;
+	local_t in_flight[2];
 };
 
 #define PARTITION_META_INFO_VOLNAMELTH	64
@@ -122,7 +124,6 @@ struct hd_struct {
 	int make_it_fail;
 #endif
 	unsigned long stamp;
-	atomic_t in_flight[2];
 #ifdef	CONFIG_SMP
 	struct disk_stats __percpu *dkstats;
 #else
@@ -380,9 +381,9 @@ void part_in_flight(struct request_queue
 		    unsigned int inflight[2]);
 void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,
 		       unsigned int inflight[2]);
-void part_dec_in_flight(struct request_queue *q, struct hd_struct *part,
+void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part,
 			int rw);
-void part_inc_in_flight(struct request_queue *q, struct hd_struct *part,
+void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part,
 			int rw);
 
 static inline struct partition_meta_info *alloc_part_info(struct gendisk *disk)
Index: linux-block/block/bio.c
===================================================================
--- linux-block.orig/block/bio.c	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/block/bio.c	2018-11-28 00:09:59.000000000 +0100
@@ -1688,7 +1688,7 @@ void generic_start_io_acct(struct reques
 	update_io_ticks(cpu, part, jiffies);
 	part_stat_inc(cpu, part, ios[sgrp]);
 	part_stat_add(cpu, part, sectors[sgrp], sectors);
-	part_inc_in_flight(q, part, op_is_write(op));
+	part_inc_in_flight(q, cpu, part, op_is_write(op));
 
 	part_stat_unlock();
 }
@@ -1707,7 +1707,7 @@ void generic_end_io_acct(struct request_
 	part_stat_add(cpu, part, time_in_queue, duration);
 	if (part->partno)
 		part_stat_add(cpu, &part_to_disk(part)->part0, time_in_queue, duration);
-	part_dec_in_flight(q, part, op_is_write(req_op));
+	part_dec_in_flight(q, cpu, part, op_is_write(req_op));
 
 	part_stat_unlock();
 }
Index: linux-block/block/blk-merge.c
===================================================================
--- linux-block.orig/block/blk-merge.c	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/block/blk-merge.c	2018-11-28 00:09:59.000000000 +0100
@@ -690,7 +690,7 @@ static void blk_account_io_merge(struct
 		cpu = part_stat_lock();
 		part = req->part;
 
-		part_dec_in_flight(req->q, part, rq_data_dir(req));
+		part_dec_in_flight(req->q, cpu, part, rq_data_dir(req));
 
 		hd_struct_put(part);
 		part_stat_unlock();
Index: linux-block/block/blk-core.c
===================================================================
--- linux-block.orig/block/blk-core.c	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/block/blk-core.c	2018-11-28 00:09:59.000000000 +0100
@@ -1356,7 +1356,7 @@ void blk_account_io_done(struct request
 		part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns);
 		part_stat_add(cpu, part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
 			part_stat_add(cpu, &part_to_disk(part)->part0, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
-		part_dec_in_flight(req->q, part, rq_data_dir(req));
+		part_dec_in_flight(req->q, cpu, part, rq_data_dir(req));
 
 		hd_struct_put(part);
 		part_stat_unlock();
@@ -1391,7 +1391,7 @@ void blk_account_io_start(struct request
 			part = &rq->rq_disk->part0;
 			hd_struct_get(part);
 		}
-		part_inc_in_flight(rq->q, part, rw);
+		part_inc_in_flight(rq->q, cpu, part, rw);
 		rq->part = part;
 	}
 
Index: linux-block/drivers/md/dm.c
===================================================================
--- linux-block.orig/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
+++ linux-block/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
@@ -663,8 +663,7 @@ static void start_io_acct(struct dm_io *
 	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
 			      &dm_disk(md)->part0);
 
-	atomic_set(&dm_disk(md)->part0.in_flight[rw],
-		   atomic_inc_return(&md->pending[rw]));
+	atomic_inc(&md->pending[rw]);
 
 	if (unlikely(dm_stats_used(&md->stats)))
 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
@@ -693,7 +692,6 @@ static void end_io_acct(struct dm_io *io
 	 * a flush.
 	 */
 	pending = atomic_dec_return(&md->pending[rw]);
-	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
 	pending += atomic_read(&md->pending[rw^0x1]);
 
 	/* nudge anyone waiting on suspend queue */

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Mike Snitzer]

On Tue, Nov 27 2018 at  7:42pm -0500,
Mikulas Patocka <mpatocka@redhat.com> wrote:

> Now when part_round_stats is gone, we can switch to per-cpu in-flight
> counters.
> 
> We use the local-atomic type local_t, so that if part_inc_in_flight or
> part_dec_in_flight is reentrantly called from an interrupt, the value will
> be correct.
> 
> The other counters could be corrupted due to reentrant interrupt, but the
> corruption only results in slight counter skew - the in_flight counter
> must be exact, so it needs local_t.
> 
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
> 
> ---
>  block/bio.c           |    4 ++--
>  block/blk-core.c      |    4 ++--
>  block/blk-merge.c     |    2 +-
>  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
>  drivers/md/dm.c       |    4 +---
>  include/linux/genhd.h |    7 ++++---
>  6 files changed, 45 insertions(+), 23 deletions(-)
> 
...
> Index: linux-block/drivers/md/dm.c
> ===================================================================
> --- linux-block.orig/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> +++ linux-block/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> @@ -663,8 +663,7 @@ static void start_io_acct(struct dm_io *
>  	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
>  			      &dm_disk(md)->part0);
>  
> -	atomic_set(&dm_disk(md)->part0.in_flight[rw],
> -		   atomic_inc_return(&md->pending[rw]));
> +	atomic_inc(&md->pending[rw]);
>  
>  	if (unlikely(dm_stats_used(&md->stats)))
>  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
> @@ -693,7 +692,6 @@ static void end_io_acct(struct dm_io *io
>  	 * a flush.
>  	 */
>  	pending = atomic_dec_return(&md->pending[rw]);
> -	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
>  	pending += atomic_read(&md->pending[rw^0x1]);
>  
>  	/* nudge anyone waiting on suspend queue */
> 


These dm.c hunks conflict with changes from you that I already staged in
dm-4.21, see:

https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=b5616f7a11592cc74860f4ec3e3c4fba6688eefa
https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=78f95b53c203c969bbe6b86e405f7a891a43b6be

I'd really like to get away from DM maintaining its own ->pending
counters.

Mike

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Mikulas Patocka]

On Thu, 29 Nov 2018, Mike Snitzer wrote:

> On Tue, Nov 27 2018 at  7:42pm -0500,
> Mikulas Patocka <mpatocka@redhat.com> wrote:
> 
> > Now when part_round_stats is gone, we can switch to per-cpu in-flight
> > counters.
> > 
> > We use the local-atomic type local_t, so that if part_inc_in_flight or
> > part_dec_in_flight is reentrantly called from an interrupt, the value will
> > be correct.
> > 
> > The other counters could be corrupted due to reentrant interrupt, but the
> > corruption only results in slight counter skew - the in_flight counter
> > must be exact, so it needs local_t.
> > 
> > Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
> > 
> > ---
> >  block/bio.c           |    4 ++--
> >  block/blk-core.c      |    4 ++--
> >  block/blk-merge.c     |    2 +-
> >  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
> >  drivers/md/dm.c       |    4 +---
> >  include/linux/genhd.h |    7 ++++---
> >  6 files changed, 45 insertions(+), 23 deletions(-)
> > 
> ...
> > Index: linux-block/drivers/md/dm.c
> > ===================================================================
> > --- linux-block.orig/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> > +++ linux-block/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> > @@ -663,8 +663,7 @@ static void start_io_acct(struct dm_io *
> >  	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
> >  			      &dm_disk(md)->part0);
> >  
> > -	atomic_set(&dm_disk(md)->part0.in_flight[rw],
> > -		   atomic_inc_return(&md->pending[rw]));
> > +	atomic_inc(&md->pending[rw]);
> >  
> >  	if (unlikely(dm_stats_used(&md->stats)))
> >  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
> > @@ -693,7 +692,6 @@ static void end_io_acct(struct dm_io *io
> >  	 * a flush.
> >  	 */
> >  	pending = atomic_dec_return(&md->pending[rw]);
> > -	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
> >  	pending += atomic_read(&md->pending[rw^0x1]);
> >  
> >  	/* nudge anyone waiting on suspend queue */
> > 
> 
> 
> These dm.c hunks conflict with changes from you that I already staged in
> dm-4.21, see:
> 
> https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=b5616f7a11592cc74860f4ec3e3c4fba6688eefa
> https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=78f95b53c203c969bbe6b86e405f7a891a43b6be
> 
> I'd really like to get away from DM maintaining its own ->pending
> counters.
> 
> Mike

I know.

It depends on whether Jens takes these patches or not. If he doesn't take 
them, dm will use percpu counters on its own and it will return zero in 
the "/sys/block/dm-*/inflight" file.

Mikulas

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Jens Axboe]

On 11/29/18 3:05 PM, Mikulas Patocka wrote:
> 
> 
> On Thu, 29 Nov 2018, Mike Snitzer wrote:
> 
>> On Tue, Nov 27 2018 at  7:42pm -0500,
>> Mikulas Patocka <mpatocka@redhat.com> wrote:
>>
>>> Now when part_round_stats is gone, we can switch to per-cpu in-flight
>>> counters.
>>>
>>> We use the local-atomic type local_t, so that if part_inc_in_flight or
>>> part_dec_in_flight is reentrantly called from an interrupt, the value will
>>> be correct.
>>>
>>> The other counters could be corrupted due to reentrant interrupt, but the
>>> corruption only results in slight counter skew - the in_flight counter
>>> must be exact, so it needs local_t.
>>>
>>> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>>>
>>> ---
>>>  block/bio.c           |    4 ++--
>>>  block/blk-core.c      |    4 ++--
>>>  block/blk-merge.c     |    2 +-
>>>  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
>>>  drivers/md/dm.c       |    4 +---
>>>  include/linux/genhd.h |    7 ++++---
>>>  6 files changed, 45 insertions(+), 23 deletions(-)
>>>
>> ...
>>> Index: linux-block/drivers/md/dm.c
>>> ===================================================================
>>> --- linux-block.orig/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
>>> +++ linux-block/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
>>> @@ -663,8 +663,7 @@ static void start_io_acct(struct dm_io *
>>>  	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
>>>  			      &dm_disk(md)->part0);
>>>  
>>> -	atomic_set(&dm_disk(md)->part0.in_flight[rw],
>>> -		   atomic_inc_return(&md->pending[rw]));
>>> +	atomic_inc(&md->pending[rw]);
>>>  
>>>  	if (unlikely(dm_stats_used(&md->stats)))
>>>  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
>>> @@ -693,7 +692,6 @@ static void end_io_acct(struct dm_io *io
>>>  	 * a flush.
>>>  	 */
>>>  	pending = atomic_dec_return(&md->pending[rw]);
>>> -	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
>>>  	pending += atomic_read(&md->pending[rw^0x1]);
>>>  
>>>  	/* nudge anyone waiting on suspend queue */
>>>
>>
>>
>> These dm.c hunks conflict with changes from you that I already staged in
>> dm-4.21, see:
>>
>> https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=b5616f7a11592cc74860f4ec3e3c4fba6688eefa
>> https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=78f95b53c203c969bbe6b86e405f7a891a43b6be
>>
>> I'd really like to get away from DM maintaining its own ->pending
>> counters.
>>
>> Mike
> 
> I know.
> 
> It depends on whether Jens takes these patches or not. If he doesn't take 
> them, dm will use percpu counters on its own and it will return zero in 
> the "/sys/block/dm-*/inflight" file.

This reply is the first I've seen of them?!

Oh, they are in gmail spam. This happened with your last patches as well,
you might want to look into that. I'll take a look at them later today.

-- 
Jens Axboe

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Mike Snitzer]

On Thu, Nov 29 2018 at  5:05pm -0500,
Mikulas Patocka <mpatocka@redhat.com> wrote:

> 
> 
> On Thu, 29 Nov 2018, Mike Snitzer wrote:
> 
> > On Tue, Nov 27 2018 at  7:42pm -0500,
> > Mikulas Patocka <mpatocka@redhat.com> wrote:
> > 
> > > Now when part_round_stats is gone, we can switch to per-cpu in-flight
> > > counters.
> > > 
> > > We use the local-atomic type local_t, so that if part_inc_in_flight or
> > > part_dec_in_flight is reentrantly called from an interrupt, the value will
> > > be correct.
> > > 
> > > The other counters could be corrupted due to reentrant interrupt, but the
> > > corruption only results in slight counter skew - the in_flight counter
> > > must be exact, so it needs local_t.
> > > 
> > > Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
> > > 
> > > ---
> > >  block/bio.c           |    4 ++--
> > >  block/blk-core.c      |    4 ++--
> > >  block/blk-merge.c     |    2 +-
> > >  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
> > >  drivers/md/dm.c       |    4 +---
> > >  include/linux/genhd.h |    7 ++++---
> > >  6 files changed, 45 insertions(+), 23 deletions(-)
> > > 
> > ...
> > > Index: linux-block/drivers/md/dm.c
> > > ===================================================================
> > > --- linux-block.orig/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> > > +++ linux-block/drivers/md/dm.c	2018-11-28 00:09:59.000000000 +0100
> > > @@ -663,8 +663,7 @@ static void start_io_acct(struct dm_io *
> > >  	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
> > >  			      &dm_disk(md)->part0);
> > >  
> > > -	atomic_set(&dm_disk(md)->part0.in_flight[rw],
> > > -		   atomic_inc_return(&md->pending[rw]));
> > > +	atomic_inc(&md->pending[rw]);
> > >  
> > >  	if (unlikely(dm_stats_used(&md->stats)))
> > >  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
> > > @@ -693,7 +692,6 @@ static void end_io_acct(struct dm_io *io
> > >  	 * a flush.
> > >  	 */
> > >  	pending = atomic_dec_return(&md->pending[rw]);
> > > -	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
> > >  	pending += atomic_read(&md->pending[rw^0x1]);
> > >  
> > >  	/* nudge anyone waiting on suspend queue */
> > > 
> > 
> > 
> > These dm.c hunks conflict with changes from you that I already staged in
> > dm-4.21, see:
> > 
> > https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=b5616f7a11592cc74860f4ec3e3c4fba6688eefa
> > https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git/commit/?h=dm-4.21&id=78f95b53c203c969bbe6b86e405f7a891a43b6be
> > 
> > I'd really like to get away from DM maintaining its own ->pending
> > counters.
> > 
> > Mike
> 
> I know.
> 
> It depends on whether Jens takes these patches or not. If he doesn't take 
> them, dm will use percpu counters on its own and it will return zero in 
> the "/sys/block/dm-*/inflight" file.

I see, yeah, I could easily rebase the changes I referenced.

I had a quick look at your proposed changes, they seem pretty clean to
me.  The fact that disk_stats is already percpu for other counters makes
this change straight forward.

But I'll review closer.

Mike

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Mike Snitzer]

On Tue, Nov 27 2018 at  7:42pm -0500,
Mikulas Patocka <mpatocka@redhat.com> wrote:

> Now when part_round_stats is gone, we can switch to per-cpu in-flight
> counters.
> 
> We use the local-atomic type local_t, so that if part_inc_in_flight or
> part_dec_in_flight is reentrantly called from an interrupt, the value will
> be correct.
> 
> The other counters could be corrupted due to reentrant interrupt, but the
> corruption only results in slight counter skew - the in_flight counter
> must be exact, so it needs local_t.
> 
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
> 
> ---
>  block/bio.c           |    4 ++--
>  block/blk-core.c      |    4 ++--
>  block/blk-merge.c     |    2 +-
>  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
>  drivers/md/dm.c       |    4 +---
>  include/linux/genhd.h |    7 ++++---
>  6 files changed, 45 insertions(+), 23 deletions(-)
> 
> Index: linux-block/block/genhd.c
> ===================================================================
> --- linux-block.orig/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
> +++ linux-block/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
> @@ -45,53 +45,76 @@ static void disk_add_events(struct gendi
>  static void disk_del_events(struct gendisk *disk);
>  static void disk_release_events(struct gendisk *disk);
>  
> -void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
> +void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
>  {
>  	if (queue_is_mq(q))
>  		return;
>  
> -	atomic_inc(&part->in_flight[rw]);
> +	local_inc(per_cpu_ptr(part->dkstats, cpu)->in_flight);
>  	if (part->partno)
> -		atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);
> +		local_inc(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
>  }
>  
> -void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
> +void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
>  {
>  	if (queue_is_mq(q))
>  		return;
>  
> -	atomic_dec(&part->in_flight[rw]);
> +	local_dec(per_cpu_ptr(part->dkstats, cpu)->in_flight);
>  	if (part->partno)
> -		atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);
> +		local_dec(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
>  }


The above is wrong, needs this incremental fix:

diff --git a/block/genhd.c b/block/genhd.c
index 2ed431098953..3397288a2926 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -50,9 +50,9 @@ void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part
 	if (queue_is_mq(q))
 		return;
 
-	local_inc(per_cpu_ptr(part->dkstats, cpu)->in_flight);
+	local_inc(&per_cpu_ptr(part->dkstats, cpu)->in_flight[rw]);
 	if (part->partno)
-		local_inc(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
+		local_inc(&per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight[rw]);
 }
 
 void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
@@ -60,9 +60,9 @@ void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part
 	if (queue_is_mq(q))
 		return;
 
-	local_dec(per_cpu_ptr(part->dkstats, cpu)->in_flight);
+	local_dec(&per_cpu_ptr(part->dkstats, cpu)->in_flight[rw]);
 	if (part->partno)
-		local_dec(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
+		local_dec(&per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight[rw]);
 }
 
 unsigned int part_in_flight(struct request_queue *q, struct hd_struct *part)

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Jens Axboe]

On 11/30/18 2:44 PM, Mike Snitzer wrote:
> On Tue, Nov 27 2018 at  7:42pm -0500,
> Mikulas Patocka <mpatocka@redhat.com> wrote:
> 
>> Now when part_round_stats is gone, we can switch to per-cpu in-flight
>> counters.
>>
>> We use the local-atomic type local_t, so that if part_inc_in_flight or
>> part_dec_in_flight is reentrantly called from an interrupt, the value will
>> be correct.
>>
>> The other counters could be corrupted due to reentrant interrupt, but the
>> corruption only results in slight counter skew - the in_flight counter
>> must be exact, so it needs local_t.
>>
>> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>>
>> ---
>>  block/bio.c           |    4 ++--
>>  block/blk-core.c      |    4 ++--
>>  block/blk-merge.c     |    2 +-
>>  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
>>  drivers/md/dm.c       |    4 +---
>>  include/linux/genhd.h |    7 ++++---
>>  6 files changed, 45 insertions(+), 23 deletions(-)
>>
>> Index: linux-block/block/genhd.c
>> ===================================================================
>> --- linux-block.orig/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
>> +++ linux-block/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
>> @@ -45,53 +45,76 @@ static void disk_add_events(struct gendi
>>  static void disk_del_events(struct gendisk *disk);
>>  static void disk_release_events(struct gendisk *disk);
>>  
>> -void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
>> +void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
>>  {
>>  	if (queue_is_mq(q))
>>  		return;
>>  
>> -	atomic_inc(&part->in_flight[rw]);
>> +	local_inc(per_cpu_ptr(part->dkstats, cpu)->in_flight);
>>  	if (part->partno)
>> -		atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);
>> +		local_inc(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
>>  }
>>  
>> -void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
>> +void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
>>  {
>>  	if (queue_is_mq(q))
>>  		return;
>>  
>> -	atomic_dec(&part->in_flight[rw]);
>> +	local_dec(per_cpu_ptr(part->dkstats, cpu)->in_flight);
>>  	if (part->partno)
>> -		atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);
>> +		local_dec(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
>>  }
> 
> 
> The above is wrong, needs this incremental fix:

Should also be using this_cpu_inc/dec instead.

-- 
Jens Axboe

Re: [PATCH 2/3] block: switch to per-cpu in-flight counters

[Mike Snitzer]

On Fri, Nov 30 2018 at  4:52pm -0500,
Jens Axboe <axboe@kernel.dk> wrote:

> On 11/30/18 2:44 PM, Mike Snitzer wrote:
> > On Tue, Nov 27 2018 at  7:42pm -0500,
> > Mikulas Patocka <mpatocka@redhat.com> wrote:
> > 
> >> Now when part_round_stats is gone, we can switch to per-cpu in-flight
> >> counters.
> >>
> >> We use the local-atomic type local_t, so that if part_inc_in_flight or
> >> part_dec_in_flight is reentrantly called from an interrupt, the value will
> >> be correct.
> >>
> >> The other counters could be corrupted due to reentrant interrupt, but the
> >> corruption only results in slight counter skew - the in_flight counter
> >> must be exact, so it needs local_t.
> >>
> >> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
> >>
> >> ---
> >>  block/bio.c           |    4 ++--
> >>  block/blk-core.c      |    4 ++--
> >>  block/blk-merge.c     |    2 +-
> >>  block/genhd.c         |   47 +++++++++++++++++++++++++++++++++++------------
> >>  drivers/md/dm.c       |    4 +---
> >>  include/linux/genhd.h |    7 ++++---
> >>  6 files changed, 45 insertions(+), 23 deletions(-)
> >>
> >> Index: linux-block/block/genhd.c
> >> ===================================================================
> >> --- linux-block.orig/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
> >> +++ linux-block/block/genhd.c	2018-11-28 00:09:59.000000000 +0100
> >> @@ -45,53 +45,76 @@ static void disk_add_events(struct gendi
> >>  static void disk_del_events(struct gendisk *disk);
> >>  static void disk_release_events(struct gendisk *disk);
> >>  
> >> -void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
> >> +void part_inc_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
> >>  {
> >>  	if (queue_is_mq(q))
> >>  		return;
> >>  
> >> -	atomic_inc(&part->in_flight[rw]);
> >> +	local_inc(per_cpu_ptr(part->dkstats, cpu)->in_flight);
> >>  	if (part->partno)
> >> -		atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);
> >> +		local_inc(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
> >>  }
> >>  
> >> -void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
> >> +void part_dec_in_flight(struct request_queue *q, int cpu, struct hd_struct *part, int rw)
> >>  {
> >>  	if (queue_is_mq(q))
> >>  		return;
> >>  
> >> -	atomic_dec(&part->in_flight[rw]);
> >> +	local_dec(per_cpu_ptr(part->dkstats, cpu)->in_flight);
> >>  	if (part->partno)
> >> -		atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);
> >> +		local_dec(per_cpu_ptr(part_to_disk(part)->part0.dkstats, cpu)->in_flight);
> >>  }
> > 
> > 
> > The above is wrong, needs this incremental fix:
> 
> Should also be using this_cpu_inc/dec instead.

How so?  Mikulas is using local_t (original header speaks to why)