Re: fix atomic limits check

[alan.adamson@oracle.com]

On 6/23/25 6:33 AM, Christoph Hellwig wrote:
> On Wed, Jun 18, 2025 at 11:30:20AM -0700, alan.adamson@oracle.com wrote:
>>> Can you double check this really is your CTLR2?  At least for me
>>> the async probing reorders nvme devices quite a lot with qemu.
>>>
>> CTRL 0 (nvme0) - AWUN=31 AWUPF=15       nvme1n2
>> CTRL 1 (nvme1) - AWUN=31 AWUPF=31       nvme1n1
>> CTRL 2 (nvme2) - AWUN=15 AWUPF=7         nvme1n3
>> CTRL 3 (nvme3) - AWUN=15 AWUPF=15
> Do you mean AWUN or AWUPF?  Because AWUN is totally irrelevant and
> not even parsed by Linux.

The above configuration is the HW/QEMU configuration. Both AWUN and 
AWUPF are setup.

>
>> I rebooted and it probed differently.  Now CTRL2 is nvme1n3.  We know
>> CTRL2 is the same as nvme1n3 because awun and awupf from nvme id-ctrl
>> matches the qemu config (atomic.awun=15,atomic.awupf=7) for CTRL2 (nvme2).
>>
>> # nvme id-ctrl /dev/nvme1n3 | grep awun
>> awun      : 15
>> # nvme id-ctrl /dev/nvme1n3 | grep awupf
>> awupf     : 7
>> # cat /sys/block/nvme1n3/queue/atomic_write_max_bytes
>> 8192
> The code in nvme_configure_atomic_write calculates the values pretty
> much directly from (N)AWUPF, so I don't see how it would get things
> wrong.  Can you run your patched qemu setup with this debug printk
> patch?
>
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index 3da5ac71a9b0..245397b217f4 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -2033,6 +2033,8 @@ static u32 nvme_configure_atomic_write(struct nvme_ns *ns,
>   		atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
>   		if (id->nabspf)
>   			boundary = (le16_to_cpu(id->nabspf) + 1) * bs;
> +		dev_info(ns->ctrl->device, "NAWUPF: %u, atomic_bs: %u\n",
> +			le16_to_cpu(id->nabspf), atomic_bs);
>   	} else {
>   		/*
>   		 * Use the controller wide atomic write unit.  This sucks
> @@ -2042,6 +2044,8 @@ static u32 nvme_configure_atomic_write(struct nvme_ns *ns,
>   		 * values for different controllers in the subsystem.
>   		 */
>   		atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
> +		dev_info(ns->ctrl->device, "AWUPF: %u, atomic_bs: %u\n",
> +			ns->ctrl->subsys->awupf, atomic_bs);
>   	}
>   
>   	lim->atomic_write_hw_max = atomic_bs;

Here you go:

CTRL 0 (nvme0) - AWUN=31 AWUPF=15       nvme0n1
CTRL 1 (nvme1) - AWUN=31 AWUPF=31       nvme0n3
CTRL 2 (nvme2) - AWUN=15 AWUPF=7        nvme0n2
CTRL 3 (nvme3) - AWUN=15 AWUPF=15
     NS - NAWUN=31 NAWUPF=15                 nvme0n4
     NS - NAWUN=127 NAWUPF=63               nvme0n5


[root@localhost ~]# nvme id-ctrl /dev/nvme0n1 | grep awupf
awupf     : 15
[root@localhost ~]# cat /sys/block/nvme0n1/queue/atomic_write_max_bytes
8192
[root@localhost ~]# nvme id-ctrl /dev/nvme0n2 | grep awupf
awupf     : 7
[root@localhost ~]# cat /sys/block/nvme0n2/queue/atomic_write_max_bytes
8192
[root@localhost ~]# nvme id-ctrl /dev/nvme0n3 | grep awupf
awupf     : 31
[root@localhost ~]# cat /sys/block/nvme0n3/queue/atomic_write_max_bytes
8192
[root@localhost ~]# nvme id-ns /dev/nvme0n4 | grep nawupf
nawupf  : 15
[root@localhost ~]# cat /sys/block/nvme0n4/queue/atomic_write_max_bytes
8192
[root@localhost ~]# nvme id-ns /dev/nvme0n5 | grep nawupf
nawupf  : 63
[root@localhost ~]# cat /sys/block/nvme0n5/queue/atomic_write_max_bytes
32768
[root@localhost ~]#

[root@localhost ~]# dmesg | grep nvme | grep atomic_bs
[    2.831882] nvme nvme0: AWUPF: 15, atomic_bs: 8192
[    2.840480] nvme nvme2: AWUPF: 15, atomic_bs: 8192
[    2.842418] nvme nvme1: AWUPF: 15, atomic_bs: 8192
[    2.861119] nvme nvme3: NAWUPF: 0, atomic_bs: 8192
[    2.862427] nvme nvme3: NAWUPF: 0, atomic_bs: 32768
[root@localhost ~]#