[PATCH v1 0/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[PATCH v1 0/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Added capability to connect to an Administrative Controller by 
preventing ioq creation for admin-controllers. Also prevent ioq 
creation for discovery-controllers as the spec prohibits them. 


Kamaljit Singh (1):
  nvme: add admin controller support. prohibit ioq creation for admin &
    disco ctrlrs

 drivers/nvme/host/core.c    | 25 +++++++++++++------------
 drivers/nvme/host/fabrics.h |  5 +++++
 drivers/nvme/host/nvme.h    |  5 +++++
 drivers/nvme/host/rdma.c    | 11 +++++++++++
 drivers/nvme/host/tcp.c     | 11 +++++++++++
 5 files changed, 45 insertions(+), 12 deletions(-)

-- 
2.43.0

[PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Added capability to connect to an administrative controller by
preventing ioq creation for admin-controllers. Also prevent ioq creation
for discovery-controllers as the spec prohibits them.

* Added nvme_admin_ctrl() to check for an administrative controller

* Renamed nvme_discovery_ctrl() to nvmf_discovery_ctrl() as discovery is
  more relevant to fabrics

* Similar to a discovery ctrl, prevent an admin-ctrl from getting a
  smart log (LID=2). LID 2 is optional for admin controllers but in the
  future when we add support for the newly added LID=0 (supported log
  pages), we can make GLP accesses smarter by basing such calls on
  response from LID=0 reads.

Signed-off-by: Kamaljit Singh <kamaljit.singh1@wdc.com>
---
 drivers/nvme/host/core.c    | 25 +++++++++++++------------
 drivers/nvme/host/fabrics.h |  5 +++++
 drivers/nvme/host/nvme.h    |  5 +++++
 drivers/nvme/host/rdma.c    | 11 +++++++++++
 drivers/nvme/host/tcp.c     | 11 +++++++++++
 5 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 60537c9224bf..417893c4e8e8 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -2800,11 +2800,6 @@ static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
 	return NULL;
 }
 
-static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
-{
-	return ctrl->opts && ctrl->opts->discovery_nqn;
-}
-
 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 {
@@ -2825,7 +2820,7 @@ static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
 		}
 
 		if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
-		    nvme_discovery_ctrl(ctrl))
+		    nvmf_discovery_ctrl(ctrl))
 			continue;
 
 		dev_err(ctrl->device,
@@ -2863,13 +2858,19 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 	else
 		subsys->subtype = NVME_NQN_NVME;
 
-	if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
+	if (nvmf_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
 		dev_err(ctrl->device,
 			"Subsystem %s is not a discovery controller",
 			subsys->subnqn);
 		kfree(subsys);
 		return -EINVAL;
 	}
+	if (nvme_admin_ctrl(ctrl)) {
+		dev_info(ctrl->device,
+			"Subsystem %s is an administrative controller",
+			subsys->subnqn);
+	}
+
 	subsys->awupf = le16_to_cpu(id->awupf);
 	nvme_mpath_default_iopolicy(subsys);
 
@@ -3093,20 +3094,20 @@ static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ct
 		return -EINVAL;
 	}
 
-	if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
+	if (!nvmf_discovery_ctrl(ctrl) && !ctrl->kas) {
 		dev_err(ctrl->device,
 			"keep-alive support is mandatory for fabrics\n");
 		return -EINVAL;
 	}
 
-	if (!nvme_discovery_ctrl(ctrl) && ctrl->ioccsz < 4) {
+	if (!nvmf_discovery_ctrl(ctrl) && ctrl->ioccsz < 4) {
 		dev_err(ctrl->device,
 			"I/O queue command capsule supported size %d < 4\n",
 			ctrl->ioccsz);
 		return -EINVAL;
 	}
 
-	if (!nvme_discovery_ctrl(ctrl) && ctrl->iorcsz < 1) {
+	if (!nvmf_discovery_ctrl(ctrl) && ctrl->iorcsz < 1) {
 		dev_err(ctrl->device,
 			"I/O queue response capsule supported size %d < 1\n",
 			ctrl->iorcsz);
@@ -3290,7 +3291,7 @@ int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
 
 	nvme_configure_opal(ctrl, was_suspended);
 
-	if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
+	if (!ctrl->identified && !nvmf_discovery_ctrl(ctrl) && !nvme_admin_ctrl(ctrl)) {
 		/*
 		 * Do not return errors unless we are in a controller reset,
 		 * the controller works perfectly fine without hwmon.
@@ -4492,7 +4493,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)
 	 * checking that they started once before, hence are reconnecting back.
 	 */
 	if (test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
-	    nvme_discovery_ctrl(ctrl))
+	    nvmf_discovery_ctrl(ctrl))
 		nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
 
 	if (ctrl->queue_count > 1) {
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index 06cc54851b1b..679cf5282cee 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -181,6 +181,11 @@ struct nvmf_transport_ops {
 					struct nvmf_ctrl_options *opts);
 };
 
+static inline bool nvmf_discovery_ctrl(struct nvme_ctrl *ctrl)
+{
+	return ctrl->opts && ctrl->opts->discovery_nqn;
+}
+
 static inline bool
 nvmf_ctlr_matches_baseopts(struct nvme_ctrl *ctrl,
 			struct nvmf_ctrl_options *opts)
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 7b87763e2f8a..7c2d896a754c 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1135,6 +1135,11 @@ struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
 void nvme_put_ns(struct nvme_ns *ns);
 
+static inline bool nvme_admin_ctrl(struct nvme_ctrl *ctrl)
+{
+	return (ctrl->cntrltype == NVME_CTRL_ADMIN);
+}
+
 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
 {
 	return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 299e3c19df9d..0f3150411bd5 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -1030,6 +1030,17 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
 		goto destroy_admin;
 	}
 
+	/* An admin or discovery controller has one admin queue, but no I/O queues */
+	if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
+		ctrl->ctrl.queue_count = 1;
+	} else if (ctrl->ctrl.queue_count < 2) {
+		/* I/O controller with no I/O queues is not allowed */
+		ret = -EOPNOTSUPP;
+		dev_err(ctrl->ctrl.device,
+			"I/O controller doesn't allow zero I/O queues!\n");
+		goto destroy_admin;
+	}
+
 	if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize + 1) {
 		dev_warn(ctrl->ctrl.device,
 			"queue_size %zu > ctrl sqsize %u, clamping down\n",
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 644f84284b6f..3fe2f617bfd5 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -2199,6 +2199,17 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
 		goto destroy_admin;
 	}
 
+	/* An admin or discovery controller has one admin queue, but no I/O queues */
+	if (nvme_admin_ctrl(ctrl) || nvmf_discovery_ctrl(ctrl)) {
+		ctrl->queue_count = 1;
+	} else if (ctrl->queue_count < 2) {
+		/* I/O controller with no I/O queues is not allowed */
+		ret = -EOPNOTSUPP;
+		dev_err(ctrl->device,
+			"I/O controller doesn't allow zero I/O queues!\n");
+		goto destroy_admin;
+	}
+
 	if (opts->queue_size > ctrl->sqsize + 1)
 		dev_warn(ctrl->device,
 			"queue_size %zu > ctrl sqsize %u, clamping down\n",
-- 
2.43.0

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Damien Le Moal]

On 2025/03/28 14:36, Kamaljit Singh wrote:
> Added capability to connect to an administrative controller by

s/Added/Add

> preventing ioq creation for admin-controllers. Also prevent ioq creation
> for discovery-controllers as the spec prohibits them.

This second part should be a different (preparatory) patch.

> 
> * Added nvme_admin_ctrl() to check for an administrative controller

s/Added/Add
And this should be a different preparatory patch.

> 
> * Renamed nvme_discovery_ctrl() to nvmf_discovery_ctrl() as discovery is
>   more relevant to fabrics

I do not think that is necessary since this is testing the controller type,
which may be limited to fabrics or not.

> * Similar to a discovery ctrl, prevent an admin-ctrl from getting a
>   smart log (LID=2). LID 2 is optional for admin controllers but in the
>   future when we add support for the newly added LID=0 (supported log
>   pages), we can make GLP accesses smarter by basing such calls on
>   response from LID=0 reads.
> 
> Signed-off-by: Kamaljit Singh <kamaljit.singh1@wdc.com>

[...]

> @@ -2863,13 +2858,19 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
>  	else
>  		subsys->subtype = NVME_NQN_NVME;
>  
> -	if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
> +	if (nvmf_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
>  		dev_err(ctrl->device,
>  			"Subsystem %s is not a discovery controller",
>  			subsys->subnqn);
>  		kfree(subsys);
>  		return -EINVAL;
>  	}
> +	if (nvme_admin_ctrl(ctrl)) {
> +		dev_info(ctrl->device,
> +			"Subsystem %s is an administrative controller",
> +			subsys->subnqn);
> +	}

Is this really necessary ? In any case, please remove the curly brackets.

[...]

> diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
> index 299e3c19df9d..0f3150411bd5 100644
> --- a/drivers/nvme/host/rdma.c
> +++ b/drivers/nvme/host/rdma.c
> @@ -1030,6 +1030,17 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
>  		goto destroy_admin;
>  	}
>  
> +	/* An admin or discovery controller has one admin queue, but no I/O queues */
> +	if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
> +		ctrl->ctrl.queue_count = 1;
> +	} else if (ctrl->ctrl.queue_count < 2) {
> +		/* I/O controller with no I/O queues is not allowed */
> +		ret = -EOPNOTSUPP;
> +		dev_err(ctrl->ctrl.device,
> +			"I/O controller doesn't allow zero I/O queues!\n");
> +		goto destroy_admin;
> +	}

This is identical to the change for tcp, so maybe make this a helper function ?

> +
>  	if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize + 1) {
>  		dev_warn(ctrl->ctrl.device,
>  			"queue_size %zu > ctrl sqsize %u, clamping down\n",
> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> index 644f84284b6f..3fe2f617bfd5 100644
> --- a/drivers/nvme/host/tcp.c
> +++ b/drivers/nvme/host/tcp.c
> @@ -2199,6 +2199,17 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
>  		goto destroy_admin;
>  	}
>  
> +	/* An admin or discovery controller has one admin queue, but no I/O queues */
> +	if (nvme_admin_ctrl(ctrl) || nvmf_discovery_ctrl(ctrl)) {
> +		ctrl->queue_count = 1;
> +	} else if (ctrl->queue_count < 2) {
> +		/* I/O controller with no I/O queues is not allowed */
> +		ret = -EOPNOTSUPP;
> +		dev_err(ctrl->device,
> +			"I/O controller doesn't allow zero I/O queues!\n");
> +		goto destroy_admin;
> +	}
> +
>  	if (opts->queue_size > ctrl->sqsize + 1)
>  		dev_warn(ctrl->device,
>  			"queue_size %zu > ctrl sqsize %u, clamping down\n",

-- 
Damien Le Moal
Western Digital Research

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Hi Damien,
 
On 2025/03/28 15:09, Damien Le Moal wrote:
>> Added capability to connect to an administrative controller by
> 
>s/Added/Add
Done
 
>> preventing ioq creation for admin-controllers. Also prevent ioq creation
>> for discovery-controllers as the spec prohibits them.
 
>This second part should be a different (preparatory) patch.
I've separated it as its own patch. Sorry took a while to procure the setup
But I’ve tested it now. Will share updated patch-set.
 
You were right, it is a pain using Outlook for kernel work :(
Vim to the rescue for now.
 
>> 
>> * Added nvme_admin_ctrl() to check for an administrative controller
 
>s/Added/Add
>And this should be a different preparatory patch.
Sounds good.
 
>> 
>> * Renamed nvme_discovery_ctrl() to nvmf_discovery_ctrl() as discovery is
>>   more relevant to fabrics
 
>I do not think that is necessary since this is testing the controller type,
>which may be limited to fabrics or not.
I undid that rename and moved the function as Niklas suggested, to core.c.
I tried moving nvme_discovery_ctrl(), nvme_admin_ctrl &
nvme_update_ctrl_queue_count() to host/nvme.h but couldn't since target/
passthru.c includes that host header & fails compilation.
 
To make it work that way we would have to include host/fabrics.h in target/
passthru.c but that's causing a lot of unresolved symbols and requires a
different/larger commit. Also, it may not even be desirable as it will further
pollute the target code with host specific header.  Please comment.
 
 
>> @@ -2863,13 +2858,19 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
>>       else
>>               subsys->subtype = NVME_NQN_NVME;
>> 
>> -     if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
>> +     if (nvmf_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
>>               dev_err(ctrl->device,
>>                       "Subsystem %s is not a discovery controller",
>>                       subsys->subnqn);
>>               kfree(subsys);
>>               return -EINVAL;
>>       }
>> +     if (nvme_admin_ctrl(ctrl)) {
>> +             dev_info(ctrl->device,
>> +                     "Subsystem %s is an administrative controller",
>> +                     subsys->subnqn);
>> +     }
 
>Is this really necessary ? In any case, please remove the curly brackets.
I've removed it. I had started off by adding a dev_err check similar to
the one for discovery ctrl above it but since there is no unique identifier
(like the SUBNQN or unique port) for an admin controller, I turned it into
a dev_info to at least announce the presence of an admin controller. Any
suggestions for error validation at init time?
 
 
>> diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
>> index 299e3c19df9d..0f3150411bd5 100644
>> --- a/drivers/nvme/host/rdma.c
>> +++ b/drivers/nvme/host/rdma.c
>> @@ -1030,6 +1030,17 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
>>               goto destroy_admin;
>>       }
>> 
>> +     /* An admin or discovery controller has one admin queue, but no I/O queues */
>> +     if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
>> +             ctrl->ctrl.queue_count = 1;
>> +     } else if (ctrl->ctrl.queue_count < 2) {
>> +             /* I/O controller with no I/O queues is not allowed */
>> +             ret = -EOPNOTSUPP;
>> +             dev_err(ctrl->ctrl.device,
>> +                     "I/O controller doesn't allow zero I/O queues!\n");
>> +             goto destroy_admin;
>> +     }
 
>This is identical to the change for tcp, so maybe make this a helper function ?
I've converted it into nvme_update_ctrl_queue_count() and added it to host/nvme.h.
Will be in the next patch-set.
 

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Niklas Cassel]

Hello Kamaljit,

On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:

Like Damien said,

> Added capability to connect to an administrative controller by
> preventing ioq creation for admin-controllers.

patch 2/4,
I think I deserve:
Suggested-by: Niklas Cassel <cassel@kernel.org>

on this patch.


> Also prevent ioq creation for discovery-controllers as the spec
> prohibits them.

patch 3/4


> 
> * Added nvme_admin_ctrl() to check for an administrative controller

This could be part of patch 2/4.


> 
> * Renamed nvme_discovery_ctrl() to nvmf_discovery_ctrl() as discovery is
>   more relevant to fabrics

patch 1/4, you need to move the function, but can keep the name.


> 
> * Similar to a discovery ctrl, prevent an admin-ctrl from getting a
>   smart log (LID=2). LID 2 is optional for admin controllers but in the
>   future when we add support for the newly added LID=0 (supported log
>   pages), we can make GLP accesses smarter by basing such calls on
>   response from LID=0 reads.

patch 4/4


> 
> Signed-off-by: Kamaljit Singh <kamaljit.singh1@wdc.com>


Kind regards,
Niklas

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Hi Niklas,

On 2025/03/31 00:25, Niklas Cassel wrote:

>Like Damien said,
>
>> Added capability to connect to an administrative controller by
>> preventing ioq creation for admin-controllers.
>
>patch 2/4,
>I think I deserve:
>Suggested-by: Niklas Cassel <cassel@kernel.org>
>
>on this patch.
You sure do. I wasn't aware of this designation. I'll add it. Thanks for
all your help.


>> * Renamed nvme_discovery_ctrl() to nvmf_discovery_ctrl() as discovery is
>>   more relevant to fabrics
>
>patch 1/4, you need to move the function, but can keep the name.
Like I replied to Damien, I wasn't able to easily move these to host/nvme.h
so they're in core.c.

I'll separate into patches as you suggested.

Thanks,
Kamaljit

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Keith Busch]

On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:
> -static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
> -{
> -	return ctrl->opts && ctrl->opts->discovery_nqn;
> -}
> -

I suppose it's fine to rename this function with an nvmf_ prefix, but
it's not really related to the rest of the patch and makes the diff
larger than necessary.

> +	/* An admin or discovery controller has one admin queue, but no I/O queues */
> +	if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
> +		ctrl->ctrl.queue_count = 1;
> +	} else if (ctrl->ctrl.queue_count < 2) {
> +		/* I/O controller with no I/O queues is not allowed */
> +		ret = -EOPNOTSUPP;
> +		dev_err(ctrl->ctrl.device,
> +			"I/O controller doesn't allow zero I/O queues!\n");
> +		goto destroy_admin;
> +	}

The queue_count comes from the user. If the user provides a bad value
for an IO controller, you're erroring. If they provide a bad value for a
discovery or admin controller, you override the value. Why the different
behavior?

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Chaitanya Kulkarni]

On 3/31/25 08:03, Keith Busch wrote:
> On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:
>> -static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
>> -{
>> -	return ctrl->opts && ctrl->opts->discovery_nqn;
>> -}
>> -
> I suppose it's fine to rename this function with an nvmf_ prefix, but
> it's not really related to the rest of the patch and makes the diff
> larger than necessary.

perhaps consider making a separate patch ...

-ck

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Niklas Cassel]

On Mon, Mar 31, 2025 at 09:03:11AM -0600, Keith Busch wrote:
> On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:
> > -static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
> > -{
> > -	return ctrl->opts && ctrl->opts->discovery_nqn;
> > -}
> > -
> 
> I suppose it's fine to rename this function with an nvmf_ prefix, but
> it's not really related to the rest of the patch and makes the diff
> larger than necessary.
> 
> > +	/* An admin or discovery controller has one admin queue, but no I/O queues */
> > +	if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
> > +		ctrl->ctrl.queue_count = 1;
> > +	} else if (ctrl->ctrl.queue_count < 2) {
> > +		/* I/O controller with no I/O queues is not allowed */
> > +		ret = -EOPNOTSUPP;
> > +		dev_err(ctrl->ctrl.device,
> > +			"I/O controller doesn't allow zero I/O queues!\n");
> > +		goto destroy_admin;
> > +	}
> 
> The queue_count comes from the user. If the user provides a bad value
> for an IO controller, you're erroring. If they provide a bad value for a
> discovery or admin controller, you override the value. Why the different
> behavior?
> 

Good question.

My initial proposal was simply to override the user provided value
to 1 (admin queue only) in case of admin (or discovery) controller.

The check for queue_count < 2 should be in a separate patch, if we
want that check at all. But to be honest, the code did previously
allow an I/O controller with just the admin queue and no I/O queues.
Thus, without a commit message explaining clearly why we should start
to disallow an I/O controller with just the admin queue, I think that
additional check is wrong.


Kind regards,
Niklas

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Hi Niklas & Keith,
 
On 2025/04/01 01:04, Niklas Cassel wrote:
 
>> > +   /* An admin or discovery controller has one admin queue, but no I/O queues */
>> > +   if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
>> > +           ctrl->ctrl.queue_count = 1;
>> > +   } else if (ctrl->ctrl.queue_count < 2) {
>> > +           /* I/O controller with no I/O queues is not allowed */
>> > +           ret = -EOPNOTSUPP;
>> > +           dev_err(ctrl->ctrl.device,
>> > +                   "I/O controller doesn't allow zero I/O queues!\n");
>> > +           goto destroy_admin;
>> > +   }
>> 
>> The queue_count comes from the user. If the user provides a bad value
>> for an IO controller, you're erroring. If they provide a bad value for a
>> discovery or admin controller, you override the value. Why the different
>> behavior?
>> 
> 
> Good question.
Keith,
Yeah, that was me trying to plug the driver hole when I was hacking the
nvme-cli to pass a zero I/O queue value for an admin-controller. While
doing that allowed us to connect to an admin-controller, it didn't prevent
us from connecting to an I/O controller with zero I/O queues. Per the
spec that shouldn't be allowed. Hope that clarifies the reason for that
2nd change. I'll make it into its own patch.
 
 
> 
> My initial proposal was simply to override the user provided value
> to 1 (admin queue only) in case of admin (or discovery) controller.
>
> The check for queue_count < 2 should be in a separate patch, if we
> want that check at all.
Yes, I'll make that into a separate patch.
 
> But to be honest, the code did previously
> allow an I/O controller with just the admin queue and no I/O queues.
Agree. Initially, I was able to use that hole by forcing nvme-cli to
allow zero IOQs. But based on your suggested driver change we don't
need to patch nvme-cli anymore. That's slick!
 
However, from the nvme-cli's perspective it does feel awkward to be forced
by "nvme connect" to use -i <non-zero> for an admin-controller, even though
its now being overridden with this patch. We will have to come up with a
cleaner and standardized way to connect to an admin controller.  A standard
port number for an admin controller might be the way to go but it's not in
the spec yet.
 
 
> Thus, without a commit message explaining clearly why we should start
> to disallow an I/O controller with just the admin queue, I think that
> additional check is wrong.
For the separate patch you suggested, I'll add comments to make it clearer.
 
Thanks,
Kamaljit
 

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Niklas Cassel]

Hello Kamaljit,

On Tue, Apr 01, 2025 at 10:57:36PM +0000, Kamaljit Singh wrote:
> On 2025/04/01 01:04, Niklas Cassel wrote:
>  
> > But to be honest, the code did previously
> > allow an I/O controller with just the admin queue and no I/O queues.
> Agree. Initially, I was able to use that hole by forcing nvme-cli to
> allow zero IOQs. But based on your suggested driver change we don't
> need to patch nvme-cli anymore. That's slick!
>  
> However, from the nvme-cli's perspective it does feel awkward to be forced
> by "nvme connect" to use -i <non-zero> for an admin-controller, even though
> its now being overridden with this patch. We will have to come up with a
> cleaner and standardized way to connect to an admin controller.  A standard
> port number for an admin controller might be the way to go but it's not in
> the spec yet.

So, with this patch which overrides the user provided value,
if the controller is an admin controller, you need to use:

$ nvme connect -i <non-zero> ?

Can't you simply omit the -i parameter?


Kind regards,
Niklas

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Kamaljit Singh]

Hi Niklas,

On Tue, Apr 01, 2025 at 11:37 PM, Niklas Cassel wrote:

>> > But to be honest, the code did previously
>> > allow an I/O controller with just the admin queue and no I/O queues.
>> Agree. Initially, I was able to use that hole by forcing nvme-cli to
>> allow zero IOQs. But based on your suggested driver change we don't
>> need to patch nvme-cli anymore. That's slick!
>>
>> However, from the nvme-cli's perspective it does feel awkward to be forced
>> by "nvme connect" to use -i <non-zero> for an admin-controller, even though
>> its now being overridden with this patch. We will have to come up with a
>> cleaner and standardized way to connect to an admin controller.  A standard
>> port number for an admin controller might be the way to go but it's not in
>> the spec yet.

>So, with this patch which overrides the user provided value,
>if the controller is an admin controller, you need to use:

>$ nvme connect -i <non-zero> ?

>Can't you simply omit the -i parameter?
That worked. But under the covers nvme-cli uses a default --nr-io-queues
based on the core count, which of course will be overridden by this patch to zero
in the kernel. I'm not sure if nvme-cli may hit any internal state/cache issues related 
to an inconsistency between requested vs actual nr-io-queues. This method works 
for now.

Thanks,
Kamaljit

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Christoph Hellwig]

On Mon, Mar 31, 2025 at 09:03:11AM -0600, Keith Busch wrote:
> On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:
> > -static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
> > -{
> > -	return ctrl->opts && ctrl->opts->discovery_nqn;
> > -}
> > -
> 
> I suppose it's fine to rename this function with an nvmf_ prefix, but
> it's not really related to the rest of the patch and makes the diff
> larger than necessary.

It isn't, nvmf_ is really for code in the fabrics library and not used
by core code.

> > +	/* An admin or discovery controller has one admin queue, but no I/O queues */
> > +	if (nvme_admin_ctrl(&ctrl->ctrl) || nvmf_discovery_ctrl(&ctrl->ctrl)) {
> > +		ctrl->ctrl.queue_count = 1;
> > +	} else if (ctrl->ctrl.queue_count < 2) {
> > +		/* I/O controller with no I/O queues is not allowed */
> > +		ret = -EOPNOTSUPP;
> > +		dev_err(ctrl->ctrl.device,
> > +			"I/O controller doesn't allow zero I/O queues!\n");
> > +		goto destroy_admin;
> > +	}
> 
> The queue_count comes from the user. If the user provides a bad value
> for an IO controller, you're erroring. If they provide a bad value for a
> discovery or admin controller, you override the value. Why the different
> behavior?

Also we historically do allow I/O controllers without I/O queues to
allow for various kinds of recovery action.  So I don't think we
should change that.

Re: [PATCH v1 1/1] nvme: add admin controller support. prohibit ioq creation for admin & disco ctrlrs

[Keith Busch]

On Thu, Apr 03, 2025 at 06:49:01AM +0200, Christoph Hellwig wrote:
> On Mon, Mar 31, 2025 at 09:03:11AM -0600, Keith Busch wrote:
> > On Fri, Mar 28, 2025 at 02:36:40PM -0700, Kamaljit Singh wrote:
> > > -static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
> > > -{
> > > -	return ctrl->opts && ctrl->opts->discovery_nqn;
> > > -}
> > > -
> > 
> > I suppose it's fine to rename this function with an nvmf_ prefix, but
> > it's not really related to the rest of the patch and makes the diff
> > larger than necessary.
> 
> It isn't, nvmf_ is really for code in the fabrics library and not used
> by core code.

Yes, I'm not suggesting to use "nvmf_" in the core. This function checks
for ctrl->ops, which only applies to fabrics, so this is a fabrics
specific function that could belong there instead of the core, is all
I'm saying. Not that I'd recommend making such a move in this patch...