[NDCTL PATCH 1/2] cxl: Save the number of decoders committed to a port

[NDCTL PATCH 1/2] cxl: Save the number of decoders committed to a port

[Dave Jiang]

Save the number of decoders committed to a port exposed by the kernel to the
libcxl cxl_port context. The attribute is helpful for determing if a region is
active.  Add libcxl API to retrieve the number of decoders committed.
Add the decoders_committed attribute to the port for cxl list command.

Link: https://lore.kernel.org/linux-cxl/169645700414.623072.3893376765415910289.stgit@djiang5-mobl3/T/#t
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 Documentation/cxl/lib/libcxl.txt |    1 +
 cxl/json.c                       |    4 ++++
 cxl/lib/libcxl.c                 |    9 +++++++++
 cxl/lib/libcxl.sym               |    5 +++++
 cxl/lib/private.h                |    1 +
 cxl/libcxl.h                     |    1 +
 6 files changed, 21 insertions(+)

diff --git a/Documentation/cxl/lib/libcxl.txt b/Documentation/cxl/lib/libcxl.txt
index 31bc85511270..0b51d8b0012c 100644
--- a/Documentation/cxl/lib/libcxl.txt
+++ b/Documentation/cxl/lib/libcxl.txt
@@ -294,6 +294,7 @@ bool cxl_port_is_endpoint(struct cxl_port *port);
 int cxl_port_get_depth(struct cxl_port *port);
 bool cxl_port_hosts_memdev(struct cxl_port *port, struct cxl_memdev *memdev);
 int cxl_port_get_nr_dports(struct cxl_port *port);
+struct cxl_port *cxl_port_decoders_committed(struct cxl_port *port);
 ----
 The port type is communicated via cxl_port_is_<type>(). An 'enabled' port
 is one that has succeeded in discovering the CXL component registers in
diff --git a/cxl/json.c b/cxl/json.c
index 7678d02020b6..33800c6f9024 100644
--- a/cxl/json.c
+++ b/cxl/json.c
@@ -1120,6 +1120,10 @@ static struct json_object *__util_cxl_port_to_json(struct cxl_port *port,
 			json_object_object_add(jport, "state", jobj);
 	}
 
+	jobj = json_object_new_int(cxl_port_decoders_committed(port));
+	if (jobj)
+		json_object_object_add(jport, "decoders_committed", jobj);
+
 	json_object_set_userdata(jport, port, NULL);
 	return jport;
 }
diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index af4ca44eae19..759713110cd1 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -1833,6 +1833,10 @@ static int cxl_port_init(struct cxl_port *port, struct cxl_port *parent_port,
 	if (sysfs_read_attr(ctx, path, buf) == 0)
 		port->module = util_modalias_to_module(ctx, buf);
 
+	sprintf(path, "%s/decoders_committed", cxlport_base);
+	if (sysfs_read_attr(ctx, path, buf) == 0)
+		port->decoders_committed = strtoul(buf, NULL, 0);
+
 	free(path);
 	return 0;
 err:
@@ -3071,6 +3075,11 @@ cxl_port_get_dport_by_memdev(struct cxl_port *port, struct cxl_memdev *memdev)
 	return NULL;
 }
 
+CXL_EXPORT int cxl_port_decoders_committed(struct cxl_port *port)
+{
+	return port->decoders_committed;
+}
+
 static void *add_cxl_bus(void *parent, int id, const char *cxlbus_base)
 {
 	const char *devname = devpath_to_devname(cxlbus_base);
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 8fa1cca3d0d7..17b43ac39056 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -264,3 +264,8 @@ global:
 	cxl_memdev_update_fw;
 	cxl_memdev_cancel_fw_update;
 } LIBCXL_5;
+
+LIBCXL_7 {
+global:
+	cxl_port_decoders_committed;
+} LIBCXL_6;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index a641727000f1..aaaecba74096 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -87,6 +87,7 @@ struct cxl_port {
 	int dports_init;
 	int nr_dports;
 	int depth;
+	int decoders_committed;
 	struct cxl_ctx *ctx;
 	struct cxl_bus *bus;
 	enum cxl_port_type type;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 0f4f4b2648fb..9d9b4cc57769 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -145,6 +145,7 @@ bool cxl_port_hosts_memdev(struct cxl_port *port, struct cxl_memdev *memdev);
 int cxl_port_get_nr_dports(struct cxl_port *port);
 int cxl_port_disable_invalidate(struct cxl_port *port);
 int cxl_port_enable(struct cxl_port *port);
+int cxl_port_decoders_committed(struct cxl_port *port);
 struct cxl_port *cxl_port_get_next_all(struct cxl_port *port,
 				       const struct cxl_port *top);
 

[NDCTL PATCH 2/2] cxl: Add check for regions before disabling memdev

[Dave Jiang]

Add a check for memdev disable to see if there are active regions present
before disabling the device. This is necessary now regions are present to
fulfill the TODO that was left there. The best way to determine if a
region is active is to see if there are decoders enabled for the mem
device. This is also best effort as the state is only a snapshot the
kernel provides and is not atomic WRT the memdev disable operation. The
expectation is the admin issuing the command has full control of the mem
device and there are no other agents also attempt to control the device.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 cxl/memdev.c |   14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/cxl/memdev.c b/cxl/memdev.c
index f6a2d3f1fdca..314bac082719 100644
--- a/cxl/memdev.c
+++ b/cxl/memdev.c
@@ -373,11 +373,21 @@ static int action_free_dpa(struct cxl_memdev *memdev,
 
 static int action_disable(struct cxl_memdev *memdev, struct action_context *actx)
 {
+	struct cxl_endpoint *ep;
+	struct cxl_port *port;
+
 	if (!cxl_memdev_is_enabled(memdev))
 		return 0;
 
-	if (!param.force) {
-		/* TODO: actually detect rather than assume active */
+	ep = cxl_memdev_get_endpoint(memdev);
+	if (!ep)
+		return -ENODEV;
+
+	port = cxl_endpoint_get_port(ep);
+	if (!port)
+		return -ENODEV;
+
+	if (cxl_port_decoders_committed(port) && !param.force) {
 		log_err(&ml, "%s is part of an active region\n",
 			cxl_memdev_get_devname(memdev));
 		return -EBUSY;

Re: [NDCTL PATCH 2/2] cxl: Add check for regions before disabling memdev

[Cao, Quanquan/曹 全全]

> Add a check for memdev disable to see if there are active regions present
> before disabling the device. This is necessary now regions are present to
> fulfill the TODO that was left there. The best way to determine if a
> region is active is to see if there are decoders enabled for the mem
> device. This is also best effort as the state is only a snapshot the
> kernel provides and is not atomic WRT the memdev disable operation. The
> expectation is the admin issuing the command has full control of the mem
> device and there are no other agents also attempt to control the device.
> 
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
>   cxl/memdev.c |   14 ++++++++++++--
>   1 file changed, 12 insertions(+), 2 deletions(-)
> 
> diff --git a/cxl/memdev.c b/cxl/memdev.c
> index f6a2d3f1fdca..314bac082719 100644
> --- a/cxl/memdev.c
> +++ b/cxl/memdev.c
> @@ -373,11 +373,21 @@ static int action_free_dpa(struct cxl_memdev *memdev,
>   
>   static int action_disable(struct cxl_memdev *memdev, struct action_context *actx)
>   {
> +	struct cxl_endpoint *ep;
> +	struct cxl_port *port;
> +
>   	if (!cxl_memdev_is_enabled(memdev))
>   		return 0;
>   
> -	if (!param.force) {
> -		/* TODO: actually detect rather than assume active */
> +	ep = cxl_memdev_get_endpoint(memdev);
> +	if (!ep)
> +		return -ENODEV;
> +
> +	port = cxl_endpoint_get_port(ep);
> +	if (!port)
> +		return -ENODEV;
> +
> +	if (cxl_port_decoders_committed(port) && !param.force) {
>   		log_err(&ml, "%s is part of an active region\n",
>   			cxl_memdev_get_devname(memdev));
>   		return -EBUSY;
> 
> 
Hi Dave,

Based on my understanding of the logic in the "disable_region" and 
"destroy_region" code, in the code logic of 'disable-region -f,' after 
the check, it proceeds with the offline operation. In the code logic of 
'destroy-region -f,' after the check, it performs a disable operation on 
the region. For the 'disable-memdev -f' operation, after completing the 
check, is it also necessary to perform corresponding operations on the 
region(such as disabling region/destroying region) before disabling memdev?

Re: [NDCTL PATCH 2/2] cxl: Add check for regions before disabling memdev

[Verma, Vishal L]

On Tue, 2023-11-28 at 14:35 +0800, Cao, Quanquan/曹 全全 wrote:
> 
> > Add a check for memdev disable to see if there are active regions present
> > before disabling the device. This is necessary now regions are present to
> > fulfill the TODO that was left there. The best way to determine if a
> > region is active is to see if there are decoders enabled for the mem
> > device. This is also best effort as the state is only a snapshot the
> > kernel provides and is not atomic WRT the memdev disable operation. The
> > expectation is the admin issuing the command has full control of the mem
> > device and there are no other agents also attempt to control the device.
> > 
> > Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> > ---
> >   cxl/memdev.c |   14 ++++++++++++--
> >   1 file changed, 12 insertions(+), 2 deletions(-)
> > 
> > diff --git a/cxl/memdev.c b/cxl/memdev.c
> > index f6a2d3f1fdca..314bac082719 100644
> > --- a/cxl/memdev.c
> > +++ b/cxl/memdev.c
> > @@ -373,11 +373,21 @@ static int action_free_dpa(struct cxl_memdev *memdev,
> >   
> >   static int action_disable(struct cxl_memdev *memdev, struct action_context *actx)
> >   {
> > +       struct cxl_endpoint *ep;
> > +       struct cxl_port *port;
> > +
> >         if (!cxl_memdev_is_enabled(memdev))
> >                 return 0;
> >   
> > -       if (!param.force) {
> > -               /* TODO: actually detect rather than assume active */
> > +       ep = cxl_memdev_get_endpoint(memdev);
> > +       if (!ep)
> > +               return -ENODEV;
> > +
> > +       port = cxl_endpoint_get_port(ep);
> > +       if (!port)
> > +               return -ENODEV;
> > +
> > +       if (cxl_port_decoders_committed(port) && !param.force) {
> >                 log_err(&ml, "%s is part of an active region\n",
> >                         cxl_memdev_get_devname(memdev));
> >                 return -EBUSY;
> > 
> > 
> Hi Dave,
> 
> Based on my understanding of the logic in the "disable_region" and 
> "destroy_region" code, in the code logic of 'disable-region -f,' after 
> the check, it proceeds with the offline operation. In the code logic of 
> 'destroy-region -f,' after the check, it performs a disable operation on 
> the region. For the 'disable-memdev -f' operation, after completing the 
> check, is it also necessary to perform corresponding operations on the 
> region(such as disabling region/destroying region) before disabling memdev?
> 
I think it is better for a disable-memdev to just perform the check and
complain, instead of trying to unwind a _lot_ of things - i.e. offline
memory, disable region, and then disable memdev. If there is an error
in one of the intermediate steps, it can make it hard to trace down
what happened, and what state the system is in.

I do think that the current -f behavior of just unbind the driver
without the safety checks should come with a big loud warning in the
man page at least about what -f will do, and how it will potentially
break existing regions.

Re: [NDCTL PATCH 2/2] cxl: Add check for regions before disabling memdev

[Alison Schofield]

On Wed, Oct 04, 2023 at 03:08:30PM -0700, Dave Jiang wrote:
> Add a check for memdev disable to see if there are active regions present
> before disabling the device. This is necessary now regions are present to
> fulfill the TODO that was left there. The best way to determine if a
> region is active is to see if there are decoders enabled for the mem
> device. This is also best effort as the state is only a snapshot the
> kernel provides and is not atomic WRT the memdev disable operation. The
> expectation is the admin issuing the command has full control of the mem
> device and there are no other agents also attempt to control the device.
> 

snip

> +
> +	if (cxl_port_decoders_committed(port) && !param.force) {
>  		log_err(&ml, "%s is part of an active region\n",
>  			cxl_memdev_get_devname(memdev));
>  		return -EBUSY;

Can you emit the message either way, that is, even if it is forced,
let the user know what they just trampled on.

How easy is it to add the region names in the message?

> 
> 
> 

Re: [NDCTL PATCH 2/2] cxl: Add check for regions before disabling memdev

[Dave Jiang]

On 11/28/23 11:31, Alison Schofield wrote:
> On Wed, Oct 04, 2023 at 03:08:30PM -0700, Dave Jiang wrote:
>> Add a check for memdev disable to see if there are active regions present
>> before disabling the device. This is necessary now regions are present to
>> fulfill the TODO that was left there. The best way to determine if a
>> region is active is to see if there are decoders enabled for the mem
>> device. This is also best effort as the state is only a snapshot the
>> kernel provides and is not atomic WRT the memdev disable operation. The
>> expectation is the admin issuing the command has full control of the mem
>> device and there are no other agents also attempt to control the device.
>>
> 
> snip
> 
>> +
>> +	if (cxl_port_decoders_committed(port) && !param.force) {
>>  		log_err(&ml, "%s is part of an active region\n",
>>  			cxl_memdev_get_devname(memdev));
>>  		return -EBUSY;
> 
> Can you emit the message either way, that is, even if it is forced,
> let the user know what they just trampled on.

Ok I'll change that.

> 
> How easy is it to add the region names in the message?

I don't think we can get to a region easily. The way we currently check is to see if there are active decoders and not region existence.  

> 
>>
>>
>>