Re: [PATCH v2 3/3] scsi: use percpu counters for iorequest_cnt and iodone_cnt

[John Garry <john.g.garry@oracle.com>]

On 20/04/2026 12:38, Sumit Saxena wrote:
> iorequest_cnt and iodone_cnt are updated on every command dispatch and
> completion, often from different CPUs on high queue depth workloads.
> Using adjacent atomic_t fields caused cache line contention between the
> submission and completion paths.
> 
> Represent these statistics with struct percpu_counter so increments are
> mostly local to each CPU, avoiding false sharing without growing
> struct scsi_device further for cache-line padding.
> 
> Suggested-by: Bart Van Assche <bvanassche@acm.org>
> Signed-off-by: Sumit Saxena <sumit.saxena@broadcom.com>
> ---
>   drivers/scsi/scsi_error.c  |  2 +-
>   drivers/scsi/scsi_lib.c    |  8 ++++----
>   drivers/scsi/scsi_scan.c   |  9 +++++++++
>   drivers/scsi/scsi_sysfs.c  | 27 +++++++++++++++++++++++----
>   include/scsi/scsi_device.h |  5 +++--
>   5 files changed, 40 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
> index 147127fb4db9..c7424ce92f3e 100644
> --- a/drivers/scsi/scsi_error.c
> +++ b/drivers/scsi/scsi_error.c
> @@ -370,7 +370,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req)
>   	 */
>   	if (test_and_set_bit(SCMD_STATE_COMPLETE, &scmd->state))
>   		return BLK_EH_DONE;
> -	atomic_inc(&scmd->device->iodone_cnt);
> +	percpu_counter_inc(&scmd->device->iodone_cnt);
>   	if (scsi_abort_command(scmd) != SUCCESS) {
>   		set_host_byte(scmd, DID_TIME_OUT);
>   		scsi_eh_scmd_add(scmd);
> diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
> index 6e8c7a42603e..0b05cb63f630 100644
> --- a/drivers/scsi/scsi_lib.c
> +++ b/drivers/scsi/scsi_lib.c
> @@ -1554,7 +1554,7 @@ static void scsi_complete(struct request *rq)
>   
>   	INIT_LIST_HEAD(&cmd->eh_entry);
>   
> -	atomic_inc(&cmd->device->iodone_cnt);
> +	percpu_counter_inc(&cmd->device->iodone_cnt);
>   	if (cmd->result)
>   		atomic_inc(&cmd->device->ioerr_cnt);
>   
> @@ -1592,7 +1592,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd)
>   	struct Scsi_Host *host = cmd->device->host;
>   	int rtn = 0;
>   
> -	atomic_inc(&cmd->device->iorequest_cnt);
> +	percpu_counter_inc(&cmd->device->iorequest_cnt);
>   
>   	/* check if the device is still usable */
>   	if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
> @@ -1614,7 +1614,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd)
>   		 */
>   		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
>   			"queuecommand : device blocked\n"));
> -		atomic_dec(&cmd->device->iorequest_cnt);
> +		percpu_counter_dec(&cmd->device->iorequest_cnt);
>   		return SCSI_MLQUEUE_DEVICE_BUSY;
>   	}
>   
> @@ -1647,7 +1647,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd)
>   	trace_scsi_dispatch_cmd_start(cmd);
>   	rtn = host->hostt->queuecommand(host, cmd);
>   	if (rtn) {
> -		atomic_dec(&cmd->device->iorequest_cnt);
> +		percpu_counter_dec(&cmd->device->iorequest_cnt);
>   		trace_scsi_dispatch_cmd_error(cmd, rtn);
>   		if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
>   		    rtn != SCSI_MLQUEUE_TARGET_BUSY)
> diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
> index 9749a8dbe964..0b4fa89149af 100644
> --- a/drivers/scsi/scsi_scan.c
> +++ b/drivers/scsi/scsi_scan.c
> @@ -351,6 +351,15 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget,
>   
>   	scsi_sysfs_device_initialize(sdev);
>   
> +	ret = percpu_counter_init(&sdev->iorequest_cnt, 0, GFP_KERNEL);
> +	if (ret)
> +		goto out_device_destroy;
> +	ret = percpu_counter_init(&sdev->iodone_cnt, 0, GFP_KERNEL);
> +	if (ret) {
> +		percpu_counter_destroy(&sdev->iorequest_cnt);
> +		goto out_device_destroy;
> +	}

it could be neater to have:
	if (percpu_counter_init(&sdev->iorequest_cnt, 0, GFP_KERNEL) ||
	    percpu_counter_init(&sdev->iodone_cnt, 0, GFP_KERNEL)) {
		err = some err;
		goto out_device_destroy;
	}	

> +
>   	if (scsi_device_is_pseudo_dev(sdev))
>   		return sdev;
>   
> diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
> index dfc3559e7e04..1f5b2dc156a8 100644
> --- a/drivers/scsi/scsi_sysfs.c
> +++ b/drivers/scsi/scsi_sysfs.c
> @@ -516,6 +516,10 @@ static void scsi_device_dev_release(struct device *dev)
>   	if (vpd_pgb7)
>   		kfree_rcu(vpd_pgb7, rcu);
>   	kfree(sdev->inquiry);
> +	if (percpu_counter_initialized(&sdev->iodone_cnt))
> +		percpu_counter_destroy(&sdev->iodone_cnt);
> +	if (percpu_counter_initialized(&sdev->iorequest_cnt))
> +		percpu_counter_destroy(&sdev->iorequest_cnt);

Maybe I am wrong, but doesn't percpu_counter_destroy() handle the case 
of the percpu counter not being initialized? In other words, do we need 
the percpu_counter_initialized() checks?

>   	kfree(sdev);
>   
>   	if (parent)
> @@ -936,11 +940,26 @@ static ssize_t
>   show_iostat_counterbits(struct device *dev, struct device_attribute *attr,
>   			char *buf)
>   {
> -	return snprintf(buf, 20, "%d\n", (int)sizeof(atomic_t) * 8);
> +	/*
> +	 * iorequest_cnt and iodone_cnt are per-CPU sums (s64); ioerr_cnt and
> +	 * iotmo_cnt remain atomic_t.  Report the widest counter for tools.
> +	 */
> +	return snprintf(buf, 20, "%zu\n", sizeof(s64) * 8);
>   }
>   
>   static DEVICE_ATTR(iocounterbits, S_IRUGO, show_iostat_counterbits, NULL);
>   
> +#define show_sdev_iostat_percpu(field)					\
> +static ssize_t								\
> +show_iostat_##field(struct device *dev, struct device_attribute *attr,	\
> +		    char *buf)						\
> +{									\
> +	struct scsi_device *sdev = to_scsi_device(dev);			\
> +	unsigned long long count = percpu_counter_sum(&sdev->field);	\
> +	return snprintf(buf, 20, "0x%llx\n", count);			\
> +}									\
> +static DEVICE_ATTR(field, 0444, show_iostat_##field, NULL)
> +
>   #define show_sdev_iostat(field)						\
>   static ssize_t								\
>   show_iostat_##field(struct device *dev, struct device_attribute *attr,	\
> @@ -950,10 +969,10 @@ show_iostat_##field(struct device *dev, struct device_attribute *attr,	\
>   	unsigned long long count = atomic_read(&sdev->field);		\
>   	return snprintf(buf, 20, "0x%llx\n", count);			\
>   }									\
> -static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL)
> +static DEVICE_ATTR(field, 0444, show_iostat_##field, NULL)
>   
> -show_sdev_iostat(iorequest_cnt);
> -show_sdev_iostat(iodone_cnt);
> +show_sdev_iostat_percpu(iorequest_cnt);
> +show_sdev_iostat_percpu(iodone_cnt);
>   show_sdev_iostat(ioerr_cnt);
>   show_sdev_iostat(iotmo_cnt);
>   
> diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
> index 9c2a7bbe5891..ad80b500ced9 100644
> --- a/include/scsi/scsi_device.h
> +++ b/include/scsi/scsi_device.h
> @@ -8,6 +8,7 @@
>   #include <linux/blk-mq.h>
>   #include <scsi/scsi.h>
>   #include <linux/atomic.h>
> +#include <linux/percpu_counter.h>
>   #include <linux/sbitmap.h>
>   
>   struct bsg_device;
> @@ -271,8 +272,8 @@ struct scsi_device {
>   	unsigned int max_device_blocked; /* what device_blocked counts down from  */
>   #define SCSI_DEFAULT_DEVICE_BLOCKED	3
>   
> -	atomic_t iorequest_cnt;
> -	atomic_t iodone_cnt;
> +	struct percpu_counter iorequest_cnt;
> +	struct percpu_counter iodone_cnt;
>   	atomic_t ioerr_cnt;
>   	atomic_t iotmo_cnt;
>   

Would it be simpler to make ioerr_cnt and iotmo_cnt also as 
percpu_counter? We could then drop some sysfs code for handling atomic_t 
counter.

I noticed that there is a percpu_counter_init_many() - maybe we could 
use that, and index into the array of counters. Note that I am not 
fimilar with that API, so it may not be a good idea.