* [NDCTL PATCH v5 1/4] ndctl: cxl: Add QoS class retrieval for the root decoder
2024-02-01 23:05 ndctl: Add support of qos_class for CXL CLI Dave Jiang
@ 2024-02-01 23:05 ` Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 2/4] ndctl: cxl: Add QoS class support for the memory device Dave Jiang
` (2 subsequent siblings)
3 siblings, 0 replies; 8+ messages in thread
From: Dave Jiang @ 2024-02-01 23:05 UTC (permalink / raw)
To: linux-cxl, nvdimm; +Cc: vishal.l.verma, Alison Schofield
Add libcxl API to retrieve the QoS class for the root decoder. Also add
support to display the QoS class for the root decoder through the 'cxl
list' command. The qos_class is the QTG ID of the CFMWS window that
represents the root decoder.
Reviewed-by: Alison Schofield <alison.schofield@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
cxl/filter.h | 4 ++++
cxl/json.c | 10 ++++++++++
cxl/lib/libcxl.c | 14 ++++++++++++++
cxl/lib/libcxl.sym | 1 +
cxl/lib/private.h | 1 +
cxl/libcxl.h | 3 +++
cxl/list.c | 1 +
util/json.h | 1 +
8 files changed, 35 insertions(+)
diff --git a/cxl/filter.h b/cxl/filter.h
index 1241f72ccf62..3c5f9e8a0452 100644
--- a/cxl/filter.h
+++ b/cxl/filter.h
@@ -31,6 +31,7 @@ struct cxl_filter_params {
bool alert_config;
bool dax;
bool poison;
+ bool qos;
int verbose;
struct log_ctx ctx;
};
@@ -91,6 +92,9 @@ static inline unsigned long cxl_filter_to_flags(struct cxl_filter_params *param)
flags |= UTIL_JSON_DAX | UTIL_JSON_DAX_DEVS;
if (param->poison)
flags |= UTIL_JSON_MEDIA_ERRORS;
+ if (param->qos)
+ flags |= UTIL_JSON_QOS_CLASS;
+
return flags;
}
diff --git a/cxl/json.c b/cxl/json.c
index 6fb17582a1cb..48a43ddf14b0 100644
--- a/cxl/json.c
+++ b/cxl/json.c
@@ -1062,6 +1062,16 @@ struct json_object *util_cxl_decoder_to_json(struct cxl_decoder *decoder,
jobj);
}
+ if ((flags & UTIL_JSON_QOS_CLASS) && cxl_port_is_root(port)) {
+ int qos_class = cxl_root_decoder_get_qos_class(decoder);
+
+ if (qos_class != CXL_QOS_CLASS_NONE) {
+ jobj = json_object_new_int(qos_class);
+ if (jobj)
+ json_object_object_add(jdecoder, "qos_class", jobj);
+ }
+ }
+
json_object_set_userdata(jdecoder, decoder, NULL);
return jdecoder;
}
diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 1537a33d370e..9a1ac7001803 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -2229,6 +2229,12 @@ static void *add_cxl_decoder(void *parent, int id, const char *cxldecoder_base)
else
decoder->interleave_ways = strtoul(buf, NULL, 0);
+ sprintf(path, "%s/qos_class", cxldecoder_base);
+ if (sysfs_read_attr(ctx, path, buf) < 0)
+ decoder->qos_class = CXL_QOS_CLASS_NONE;
+ else
+ decoder->qos_class = atoi(buf);
+
switch (port->type) {
case CXL_PORT_ENDPOINT:
sprintf(path, "%s/dpa_resource", cxldecoder_base);
@@ -2423,6 +2429,14 @@ CXL_EXPORT unsigned long long cxl_decoder_get_size(struct cxl_decoder *decoder)
return decoder->size;
}
+CXL_EXPORT int cxl_root_decoder_get_qos_class(struct cxl_decoder *decoder)
+{
+ if (!cxl_port_is_root(decoder->port))
+ return -EINVAL;
+
+ return decoder->qos_class;
+}
+
CXL_EXPORT unsigned long long
cxl_decoder_get_dpa_resource(struct cxl_decoder *decoder)
{
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 2149f84d764e..384fea2c25e3 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -282,4 +282,5 @@ global:
LIBCXL_8 {
global:
cxl_memdev_wait_sanitize;
+ cxl_root_decoder_get_qos_class;
} LIBCXL_7;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index b26a8629e047..4847ff448f71 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -144,6 +144,7 @@ struct cxl_decoder {
struct list_head targets;
struct list_head regions;
struct list_head stale_regions;
+ int qos_class;
};
enum cxl_decode_state {
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 352b3a866f63..e5c08da77f77 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -173,6 +173,8 @@ struct cxl_dport *cxl_port_get_dport_by_memdev(struct cxl_port *port,
for (dport = cxl_dport_get_first(port); dport != NULL; \
dport = cxl_dport_get_next(dport))
+#define CXL_QOS_CLASS_NONE -1
+
struct cxl_decoder;
struct cxl_decoder *cxl_decoder_get_first(struct cxl_port *port);
struct cxl_decoder *cxl_decoder_get_next(struct cxl_decoder *decoder);
@@ -184,6 +186,7 @@ unsigned long long cxl_decoder_get_dpa_resource(struct cxl_decoder *decoder);
unsigned long long cxl_decoder_get_dpa_size(struct cxl_decoder *decoder);
unsigned long long
cxl_decoder_get_max_available_extent(struct cxl_decoder *decoder);
+int cxl_root_decoder_get_qos_class(struct cxl_decoder *decoder);
enum cxl_decoder_mode {
CXL_DECODER_MODE_NONE,
diff --git a/cxl/list.c b/cxl/list.c
index 13fef8569340..5402575b9937 100644
--- a/cxl/list.c
+++ b/cxl/list.c
@@ -136,6 +136,7 @@ int cmd_list(int argc, const char **argv, struct cxl_ctx *ctx)
param.regions = true;
/*fallthrough*/
case 0:
+ param.qos = true;
break;
}
diff --git a/util/json.h b/util/json.h
index ea370df4d1b7..b07055005084 100644
--- a/util/json.h
+++ b/util/json.h
@@ -21,6 +21,7 @@ enum util_json_flags {
UTIL_JSON_TARGETS = (1 << 11),
UTIL_JSON_PARTITION = (1 << 12),
UTIL_JSON_ALERT_CONFIG = (1 << 13),
+ UTIL_JSON_QOS_CLASS = (1 << 14),
};
void util_display_json_array(FILE *f_out, struct json_object *jarray,
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [NDCTL PATCH v5 2/4] ndctl: cxl: Add QoS class support for the memory device
2024-02-01 23:05 ndctl: Add support of qos_class for CXL CLI Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 1/4] ndctl: cxl: Add QoS class retrieval for the root decoder Dave Jiang
@ 2024-02-01 23:05 ` Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 4/4] ndctl: add test for qos_class in CXL test suite Dave Jiang
3 siblings, 0 replies; 8+ messages in thread
From: Dave Jiang @ 2024-02-01 23:05 UTC (permalink / raw)
To: linux-cxl, nvdimm; +Cc: vishal.l.verma, Alison Schofield
Add libcxl API to retrieve the QoS class tokens for the memory
devices. Two API calls are added. One for 'ram' or 'volatile'
mode and another for 'pmem' or 'persistent' mode. Support also added
for displaying the QoS class tokens through the 'cxl list' command.
Reviewed-by: Alison Schofield <alison.schofield@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
cxl/json.c | 14 ++++++++++++++
cxl/lib/libcxl.c | 22 ++++++++++++++++++++++
cxl/lib/libcxl.sym | 2 ++
cxl/lib/private.h | 2 ++
cxl/libcxl.h | 2 ++
5 files changed, 42 insertions(+)
diff --git a/cxl/json.c b/cxl/json.c
index 48a43ddf14b0..9d22bdd2188a 100644
--- a/cxl/json.c
+++ b/cxl/json.c
@@ -791,6 +791,13 @@ struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev,
jobj = util_json_object_size(size, flags);
if (jobj)
json_object_object_add(jdev, "pmem_size", jobj);
+
+ if (flags & UTIL_JSON_QOS_CLASS) {
+ jobj = json_object_new_int(
+ cxl_memdev_get_pmem_qos_class(memdev));
+ if (jobj)
+ json_object_object_add(jdev, "pmem_qos_class", jobj);
+ }
}
size = cxl_memdev_get_ram_size(memdev);
@@ -798,6 +805,13 @@ struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev,
jobj = util_json_object_size(size, flags);
if (jobj)
json_object_object_add(jdev, "ram_size", jobj);
+
+ if (flags & UTIL_JSON_QOS_CLASS) {
+ jobj = json_object_new_int(
+ cxl_memdev_get_ram_qos_class(memdev));
+ if (jobj)
+ json_object_object_add(jdev, "ram_qos_class", jobj);
+ }
}
if (flags & UTIL_JSON_HEALTH) {
diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 9a1ac7001803..6c293f1dfc91 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -1260,6 +1260,18 @@ static void *add_cxl_memdev(void *parent, int id, const char *cxlmem_base)
goto err_read;
memdev->ram_size = strtoull(buf, NULL, 0);
+ sprintf(path, "%s/pmem/qos_class", cxlmem_base);
+ if (sysfs_read_attr(ctx, path, buf) < 0)
+ memdev->pmem_qos_class = CXL_QOS_CLASS_NONE;
+ else
+ memdev->pmem_qos_class = atoi(buf);
+
+ sprintf(path, "%s/ram/qos_class", cxlmem_base);
+ if (sysfs_read_attr(ctx, path, buf) < 0)
+ memdev->ram_qos_class = CXL_QOS_CLASS_NONE;
+ else
+ memdev->ram_qos_class = atoi(buf);
+
sprintf(path, "%s/payload_max", cxlmem_base);
if (sysfs_read_attr(ctx, path, buf) < 0)
goto err_read;
@@ -1483,6 +1495,16 @@ CXL_EXPORT unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev)
return memdev->ram_size;
}
+CXL_EXPORT int cxl_memdev_get_pmem_qos_class(struct cxl_memdev *memdev)
+{
+ return memdev->pmem_qos_class;
+}
+
+CXL_EXPORT int cxl_memdev_get_ram_qos_class(struct cxl_memdev *memdev)
+{
+ return memdev->ram_qos_class;
+}
+
CXL_EXPORT const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev)
{
return memdev->firmware_version;
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 384fea2c25e3..465c78dc6c70 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -283,4 +283,6 @@ LIBCXL_8 {
global:
cxl_memdev_wait_sanitize;
cxl_root_decoder_get_qos_class;
+ cxl_memdev_get_pmem_qos_class;
+ cxl_memdev_get_ram_qos_class;
} LIBCXL_7;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index 4847ff448f71..07dc8c784f1d 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -47,6 +47,8 @@ struct cxl_memdev {
struct list_node list;
unsigned long long pmem_size;
unsigned long long ram_size;
+ int ram_qos_class;
+ int pmem_qos_class;
int payload_max;
size_t lsa_size;
struct kmod_module *module;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index e5c08da77f77..a180f01cb05e 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -72,6 +72,8 @@ int cxl_memdev_get_minor(struct cxl_memdev *memdev);
struct cxl_ctx *cxl_memdev_get_ctx(struct cxl_memdev *memdev);
unsigned long long cxl_memdev_get_pmem_size(struct cxl_memdev *memdev);
unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev);
+int cxl_memdev_get_pmem_qos_class(struct cxl_memdev *memdev);
+int cxl_memdev_get_ram_qos_class(struct cxl_memdev *memdev);
const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev);
bool cxl_memdev_fw_update_in_progress(struct cxl_memdev *memdev);
size_t cxl_memdev_fw_update_get_remaining(struct cxl_memdev *memdev);
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation
2024-02-01 23:05 ndctl: Add support of qos_class for CXL CLI Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 1/4] ndctl: cxl: Add QoS class retrieval for the root decoder Dave Jiang
2024-02-01 23:05 ` [NDCTL PATCH v5 2/4] ndctl: cxl: Add QoS class support for the memory device Dave Jiang
@ 2024-02-01 23:05 ` Dave Jiang
2024-02-06 6:00 ` Wonjae Lee
2024-02-01 23:05 ` [NDCTL PATCH v5 4/4] ndctl: add test for qos_class in CXL test suite Dave Jiang
3 siblings, 1 reply; 8+ messages in thread
From: Dave Jiang @ 2024-02-01 23:05 UTC (permalink / raw)
To: linux-cxl, nvdimm; +Cc: vishal.l.verma, Alison Schofield
The CFMWS provides a QTG ID. The kernel driver creates a root decoder that
represents the CFMWS. A qos_class attribute is exported via sysfs for the root
decoder.
One or more QoS class tokens are retrieved via QTG ID _DSM from the ACPI0017
device for a CXL memory device. The input for the _DSM is the read and write
latency and bandwidth for the path between the device and the CPU. The
numbers are constructed by the kernel driver for the _DSM input. When a
device is probed, QoS class tokens are retrieved. This is useful for a
hot-plugged CXL memory device that does not have regions created.
Add a QoS check during region creation. Emit a warning if the qos_class
token from the root decoder is different than the mem device qos_class
token. User parameter options are provided to fail instead of just
warning.
Reviewed-by: Alison Schofield <alison.schofield@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
Documentation/cxl/cxl-create-region.txt | 9 ++++
cxl/region.c | 56 ++++++++++++++++++++++++-
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/Documentation/cxl/cxl-create-region.txt b/Documentation/cxl/cxl-create-region.txt
index f11a412bddfe..d5e34cf38236 100644
--- a/Documentation/cxl/cxl-create-region.txt
+++ b/Documentation/cxl/cxl-create-region.txt
@@ -105,6 +105,15 @@ include::bus-option.txt[]
supplied, the first cross-host bridge (if available), decoder that
supports the largest interleave will be chosen.
+-e::
+--strict::
+ Enforce strict execution where any potential error will force failure.
+ For example, if qos_class mismatches region creation will fail.
+
+-q::
+--no-enforce-qos::
+ Parameter to bypass qos_class mismatch failure. Will only emit warning.
+
include::human-option.txt[]
include::debug-option.txt[]
diff --git a/cxl/region.c b/cxl/region.c
index 3a762db4800e..f9033fa0afbf 100644
--- a/cxl/region.c
+++ b/cxl/region.c
@@ -32,6 +32,8 @@ static struct region_params {
bool force;
bool human;
bool debug;
+ bool strict;
+ bool no_qos;
} param = {
.ways = INT_MAX,
.granularity = INT_MAX,
@@ -49,6 +51,8 @@ struct parsed_params {
const char **argv;
struct cxl_decoder *root_decoder;
enum cxl_decoder_mode mode;
+ bool strict;
+ bool no_qos;
};
enum region_actions {
@@ -81,7 +85,9 @@ OPT_STRING('U', "uuid", ¶m.uuid, \
"region uuid", "uuid for the new region (default: autogenerate)"), \
OPT_BOOLEAN('m', "memdevs", ¶m.memdevs, \
"non-option arguments are memdevs"), \
-OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats")
+OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats"), \
+OPT_BOOLEAN('e', "strict", ¶m.strict, "strict execution enforcement"), \
+OPT_BOOLEAN('q', "no-enforce-qos", ¶m.no_qos, "no enforce of qos_class")
static const struct option create_options[] = {
BASE_OPTIONS(),
@@ -360,6 +366,9 @@ static int parse_create_options(struct cxl_ctx *ctx, int count,
}
}
+ p->strict = param.strict;
+ p->no_qos = param.no_qos;
+
return 0;
err:
@@ -467,6 +476,49 @@ static void set_type_from_decoder(struct cxl_ctx *ctx, struct parsed_params *p)
p->mode = CXL_DECODER_MODE_PMEM;
}
+static int create_region_validate_qos_class(struct cxl_ctx *ctx,
+ struct parsed_params *p)
+{
+ int root_qos_class;
+ int qos_class;
+ int i;
+
+ root_qos_class = cxl_root_decoder_get_qos_class(p->root_decoder);
+ if (root_qos_class == CXL_QOS_CLASS_NONE)
+ return 0;
+
+ for (i = 0; i < p->ways; i++) {
+ struct json_object *jobj =
+ json_object_array_get_idx(p->memdevs, i);
+ struct cxl_memdev *memdev = json_object_get_userdata(jobj);
+
+ if (p->mode == CXL_DECODER_MODE_RAM)
+ qos_class = cxl_memdev_get_ram_qos_class(memdev);
+ else
+ qos_class = cxl_memdev_get_pmem_qos_class(memdev);
+
+ /* No qos_class entries. Possibly no kernel support */
+ if (qos_class == CXL_QOS_CLASS_NONE)
+ break;
+
+ if (qos_class != root_qos_class) {
+ if (p->strict && !p->no_qos) {
+ log_err(&rl, "%s QoS Class mismatches %s\n",
+ cxl_decoder_get_devname(p->root_decoder),
+ cxl_memdev_get_devname(memdev));
+
+ return -ENXIO;
+ }
+
+ log_notice(&rl, "%s QoS Class mismatches %s\n",
+ cxl_decoder_get_devname(p->root_decoder),
+ cxl_memdev_get_devname(memdev));
+ }
+ }
+
+ return 0;
+}
+
static int create_region_validate_config(struct cxl_ctx *ctx,
struct parsed_params *p)
{
@@ -507,6 +559,8 @@ found:
return rc;
collect_minsize(ctx, p);
+ create_region_validate_qos_class(ctx, p);
+
return 0;
}
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* RE: [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation
2024-02-01 23:05 ` [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation Dave Jiang
@ 2024-02-06 6:00 ` Wonjae Lee
2024-02-06 21:05 ` Dave Jiang
0 siblings, 1 reply; 8+ messages in thread
From: Wonjae Lee @ 2024-02-06 6:00 UTC (permalink / raw)
To: Dave Jiang, linux-cxl@vger.kernel.org, nvdimm@lists.linux.dev
Cc: vishal.l.verma@intel.com, Alison Schofield, KyungSan Kim,
Hojin Nam
On Thu, Feb 01, 2024 at 04:05:06PM -0700, Dave Jiang wrote:
> The CFMWS provides a QTG ID. The kernel driver creates a root decoder that
> represents the CFMWS. A qos_class attribute is exported via sysfs for the root
> decoder.
>
> One or more QoS class tokens are retrieved via QTG ID _DSM from the ACPI0017
> device for a CXL memory device. The input for the _DSM is the read and write
> latency and bandwidth for the path between the device and the CPU. The
> numbers are constructed by the kernel driver for the _DSM input. When a
> device is probed, QoS class tokens are retrieved. This is useful for a
> hot-plugged CXL memory device that does not have regions created.
>
> Add a QoS check during region creation. Emit a warning if the qos_class
> token from the root decoder is different than the mem device qos_class
> token. User parameter options are provided to fail instead of just
> warning.
>
> Reviewed-by: Alison Schofield <alison.schofield@intel.com>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
> Documentation/cxl/cxl-create-region.txt | 9 ++++
> cxl/region.c | 56 ++++++++++++++++++++++++-
> 2 files changed, 64 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/cxl/cxl-create-region.txt b/Documentation/cxl/cxl-create-region.txt
> index f11a412bddfe..d5e34cf38236 100644
> --- a/Documentation/cxl/cxl-create-region.txt
> +++ b/Documentation/cxl/cxl-create-region.txt
> @@ -105,6 +105,15 @@ include::bus-option.txt[]
> supplied, the first cross-host bridge (if available), decoder that
> supports the largest interleave will be chosen.
>
> +-e::
> +--strict::
> + Enforce strict execution where any potential error will force failure.
> + For example, if qos_class mismatches region creation will fail.
> +
> +-q::
> +--no-enforce-qos::
> + Parameter to bypass qos_class mismatch failure. Will only emit warning.
> +
> include::human-option.txt[]
>
> include::debug-option.txt[]
> diff --git a/cxl/region.c b/cxl/region.c
> index 3a762db4800e..f9033fa0afbf 100644
> --- a/cxl/region.c
> +++ b/cxl/region.c
> @@ -32,6 +32,8 @@ static struct region_params {
> bool force;
> bool human;
> bool debug;
> + bool strict;
> + bool no_qos;
> } param = {
> .ways = INT_MAX,
> .granularity = INT_MAX,
> @@ -49,6 +51,8 @@ struct parsed_params {
> const char **argv;
> struct cxl_decoder *root_decoder;
> enum cxl_decoder_mode mode;
> + bool strict;
> + bool no_qos;
> };
>
> enum region_actions {
> @@ -81,7 +85,9 @@ OPT_STRING('U', "uuid", ¶m.uuid, \
> "region uuid", "uuid for the new region (default: autogenerate)"), \
> OPT_BOOLEAN('m', "memdevs", ¶m.memdevs, \
> "non-option arguments are memdevs"), \
> -OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats")
> +OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats"), \
> +OPT_BOOLEAN('e', "strict", ¶m.strict, "strict execution enforcement"), \
> +OPT_BOOLEAN('q', "no-enforce-qos", ¶m.no_qos, "no enforce of qos_class")
>
> static const struct option create_options[] = {
> BASE_OPTIONS(),
> @@ -360,6 +366,9 @@ static int parse_create_options(struct cxl_ctx *ctx, int count,
> }
> }
>
> + p->strict = param.strict;
> + p->no_qos = param.no_qos;
> +
> return 0;
>
> err:
> @@ -467,6 +476,49 @@ static void set_type_from_decoder(struct cxl_ctx *ctx, struct parsed_params *p)
> p->mode = CXL_DECODER_MODE_PMEM;
> }
>
> +static int create_region_validate_qos_class(struct cxl_ctx *ctx,
> + struct parsed_params *p)
> +{
> + int root_qos_class;
> + int qos_class;
> + int i;
> +
> + root_qos_class = cxl_root_decoder_get_qos_class(p->root_decoder);
> + if (root_qos_class == CXL_QOS_CLASS_NONE)
> + return 0;
> +
> + for (i = 0; i < p->ways; i++) {
> + struct json_object *jobj =
> + json_object_array_get_idx(p->memdevs, i);
> + struct cxl_memdev *memdev = json_object_get_userdata(jobj);
> +
> + if (p->mode == CXL_DECODER_MODE_RAM)
> + qos_class = cxl_memdev_get_ram_qos_class(memdev);
> + else
> + qos_class = cxl_memdev_get_pmem_qos_class(memdev);
> +
> + /* No qos_class entries. Possibly no kernel support */
> + if (qos_class == CXL_QOS_CLASS_NONE)
> + break;
> +
> + if (qos_class != root_qos_class) {
> + if (p->strict && !p->no_qos) {
> + log_err(&rl, "%s QoS Class mismatches %s\n",
> + cxl_decoder_get_devname(p->root_decoder),
> + cxl_memdev_get_devname(memdev));
> +
> + return -ENXIO;
> + }
> +
> + log_notice(&rl, "%s QoS Class mismatches %s\n",
> + cxl_decoder_get_devname(p->root_decoder),
> + cxl_memdev_get_devname(memdev));
> + }
> + }
> +
> + return 0;
> +}
> +
> static int create_region_validate_config(struct cxl_ctx *ctx,
> struct parsed_params *p)
> {
> @@ -507,6 +559,8 @@ found:
> return rc;
>
> collect_minsize(ctx, p);
> + create_region_validate_qos_class(ctx, p);
Hello,
IIUC, if the strict option is given and a qos class mismatch occurs,
the region creation should fail. To do that, shouldn't the return
value of create_region_validate_qos_class() be handled like below?
diff --git a/cxl/region.c b/cxl/region.c
index f9033fa..0468f5f 100644
--- a/cxl/region.c
+++ b/cxl/region.c
@@ -559,7 +559,9 @@ found:
return rc;
collect_minsize(ctx, p);
- create_region_validate_qos_class(ctx, p);
+ rc = create_region_validate_qos_class(ctx, p);
+ if (rc)
+ return rc;
return 0;
}
Thanks,
Wonjae
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation
2024-02-06 6:00 ` Wonjae Lee
@ 2024-02-06 21:05 ` Dave Jiang
0 siblings, 0 replies; 8+ messages in thread
From: Dave Jiang @ 2024-02-06 21:05 UTC (permalink / raw)
To: wj28.lee, linux-cxl@vger.kernel.org, nvdimm@lists.linux.dev
Cc: vishal.l.verma@intel.com, Alison Schofield, KyungSan Kim,
Hojin Nam
On 2/5/24 11:00 PM, Wonjae Lee wrote:
> On Thu, Feb 01, 2024 at 04:05:06PM -0700, Dave Jiang wrote:
>> The CFMWS provides a QTG ID. The kernel driver creates a root decoder that
>> represents the CFMWS. A qos_class attribute is exported via sysfs for the root
>> decoder.
>>
>> One or more QoS class tokens are retrieved via QTG ID _DSM from the ACPI0017
>> device for a CXL memory device. The input for the _DSM is the read and write
>> latency and bandwidth for the path between the device and the CPU. The
>> numbers are constructed by the kernel driver for the _DSM input. When a
>> device is probed, QoS class tokens are retrieved. This is useful for a
>> hot-plugged CXL memory device that does not have regions created.
>>
>> Add a QoS check during region creation. Emit a warning if the qos_class
>> token from the root decoder is different than the mem device qos_class
>> token. User parameter options are provided to fail instead of just
>> warning.
>>
>> Reviewed-by: Alison Schofield <alison.schofield@intel.com>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>> ---
>> Documentation/cxl/cxl-create-region.txt | 9 ++++
>> cxl/region.c | 56 ++++++++++++++++++++++++-
>> 2 files changed, 64 insertions(+), 1 deletion(-)
>>
>> diff --git a/Documentation/cxl/cxl-create-region.txt b/Documentation/cxl/cxl-create-region.txt
>> index f11a412bddfe..d5e34cf38236 100644
>> --- a/Documentation/cxl/cxl-create-region.txt
>> +++ b/Documentation/cxl/cxl-create-region.txt
>> @@ -105,6 +105,15 @@ include::bus-option.txt[]
>> supplied, the first cross-host bridge (if available), decoder that
>> supports the largest interleave will be chosen.
>>
>> +-e::
>> +--strict::
>> + Enforce strict execution where any potential error will force failure.
>> + For example, if qos_class mismatches region creation will fail.
>> +
>> +-q::
>> +--no-enforce-qos::
>> + Parameter to bypass qos_class mismatch failure. Will only emit warning.
>> +
>> include::human-option.txt[]
>>
>> include::debug-option.txt[]
>> diff --git a/cxl/region.c b/cxl/region.c
>> index 3a762db4800e..f9033fa0afbf 100644
>> --- a/cxl/region.c
>> +++ b/cxl/region.c
>> @@ -32,6 +32,8 @@ static struct region_params {
>> bool force;
>> bool human;
>> bool debug;
>> + bool strict;
>> + bool no_qos;
>> } param = {
>> .ways = INT_MAX,
>> .granularity = INT_MAX,
>> @@ -49,6 +51,8 @@ struct parsed_params {
>> const char **argv;
>> struct cxl_decoder *root_decoder;
>> enum cxl_decoder_mode mode;
>> + bool strict;
>> + bool no_qos;
>> };
>>
>> enum region_actions {
>> @@ -81,7 +85,9 @@ OPT_STRING('U', "uuid", ¶m.uuid, \
>> "region uuid", "uuid for the new region (default: autogenerate)"), \
>> OPT_BOOLEAN('m', "memdevs", ¶m.memdevs, \
>> "non-option arguments are memdevs"), \
>> -OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats")
>> +OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats"), \
>> +OPT_BOOLEAN('e', "strict", ¶m.strict, "strict execution enforcement"), \
>> +OPT_BOOLEAN('q', "no-enforce-qos", ¶m.no_qos, "no enforce of qos_class")
>>
>> static const struct option create_options[] = {
>> BASE_OPTIONS(),
>> @@ -360,6 +366,9 @@ static int parse_create_options(struct cxl_ctx *ctx, int count,
>> }
>> }
>>
>> + p->strict = param.strict;
>> + p->no_qos = param.no_qos;
>> +
>> return 0;
>>
>> err:
>> @@ -467,6 +476,49 @@ static void set_type_from_decoder(struct cxl_ctx *ctx, struct parsed_params *p)
>> p->mode = CXL_DECODER_MODE_PMEM;
>> }
>>
>> +static int create_region_validate_qos_class(struct cxl_ctx *ctx,
>> + struct parsed_params *p)
>> +{
>> + int root_qos_class;
>> + int qos_class;
>> + int i;
>> +
>> + root_qos_class = cxl_root_decoder_get_qos_class(p->root_decoder);
>> + if (root_qos_class == CXL_QOS_CLASS_NONE)
>> + return 0;
>> +
>> + for (i = 0; i < p->ways; i++) {
>> + struct json_object *jobj =
>> + json_object_array_get_idx(p->memdevs, i);
>> + struct cxl_memdev *memdev = json_object_get_userdata(jobj);
>> +
>> + if (p->mode == CXL_DECODER_MODE_RAM)
>> + qos_class = cxl_memdev_get_ram_qos_class(memdev);
>> + else
>> + qos_class = cxl_memdev_get_pmem_qos_class(memdev);
>> +
>> + /* No qos_class entries. Possibly no kernel support */
>> + if (qos_class == CXL_QOS_CLASS_NONE)
>> + break;
>> +
>> + if (qos_class != root_qos_class) {
>> + if (p->strict && !p->no_qos) {
>> + log_err(&rl, "%s QoS Class mismatches %s\n",
>> + cxl_decoder_get_devname(p->root_decoder),
>> + cxl_memdev_get_devname(memdev));
>> +
>> + return -ENXIO;
>> + }
>> +
>> + log_notice(&rl, "%s QoS Class mismatches %s\n",
>> + cxl_decoder_get_devname(p->root_decoder),
>> + cxl_memdev_get_devname(memdev));
>> + }
>> + }
>> +
>> + return 0;
>> +}
>> +
>> static int create_region_validate_config(struct cxl_ctx *ctx,
>> struct parsed_params *p)
>> {
>> @@ -507,6 +559,8 @@ found:
>> return rc;
>>
>> collect_minsize(ctx, p);
>> + create_region_validate_qos_class(ctx, p);
>
> Hello,
>
> IIUC, if the strict option is given and a qos class mismatch occurs,
> the region creation should fail. To do that, shouldn't the return
> value of create_region_validate_qos_class() be handled like below?
You are correct. I'll fix.
>
> diff --git a/cxl/region.c b/cxl/region.c
> index f9033fa..0468f5f 100644
> --- a/cxl/region.c
> +++ b/cxl/region.c
> @@ -559,7 +559,9 @@ found:
> return rc;
>
> collect_minsize(ctx, p);
> - create_region_validate_qos_class(ctx, p);
> + rc = create_region_validate_qos_class(ctx, p);
> + if (rc)
> + return rc;
>
> return 0;
> }
>
> Thanks,
> Wonjae
^ permalink raw reply [flat|nested] 8+ messages in thread
* [NDCTL PATCH v5 4/4] ndctl: add test for qos_class in CXL test suite
2024-02-01 23:05 ndctl: Add support of qos_class for CXL CLI Dave Jiang
` (2 preceding siblings ...)
2024-02-01 23:05 ` [NDCTL PATCH v5 3/4] ndctl: cxl: add QoS class check for CXL region creation Dave Jiang
@ 2024-02-01 23:05 ` Dave Jiang
2024-02-02 16:17 ` Alison Schofield
3 siblings, 1 reply; 8+ messages in thread
From: Dave Jiang @ 2024-02-01 23:05 UTC (permalink / raw)
To: linux-cxl, nvdimm; +Cc: vishal.l.verma
Add tests in cxl-qos-class.sh to verify qos_class are set with the fake
qos_class create by the kernel. Root decoders should have qos_class
attribute set. Memory devices should have ram_qos_class or pmem_qos_class
set depending on which partitions are valid.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
v5:
- Split out from cxl-topology.sh (Vishal)
---
test/common | 4 +++
test/cxl-qos-class.sh | 65 +++++++++++++++++++++++++++++++++++++++++++
test/meson.build | 2 ++
3 files changed, 71 insertions(+)
create mode 100755 test/cxl-qos-class.sh
diff --git a/test/common b/test/common
index f1023ef20f7e..5694820c7adc 100644
--- a/test/common
+++ b/test/common
@@ -150,3 +150,7 @@ check_dmesg()
grep -q "Call Trace" <<< $log && err $1
true
}
+
+
+# CXL COMMON
+TEST_QOS_CLASS=42
diff --git a/test/cxl-qos-class.sh b/test/cxl-qos-class.sh
new file mode 100755
index 000000000000..365a7df9c1e4
--- /dev/null
+++ b/test/cxl-qos-class.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2024 Intel Corporation. All rights reserved.
+
+check_qos_decoders () {
+ # check root decoders have expected fake qos_class
+ # also make sure the number of root decoders equal to the number
+ # with qos_class found
+ json=$($CXL list -b cxl_test -D -d root)
+ decoders=$(echo "$json" | jq length)
+ count=0
+ while read -r qos_class
+ do
+ ((qos_class == TEST_QOS_CLASS)) || err "$LINENO"
+ count=$((count+1))
+ done <<< "$(echo "$json" | jq -r '.[] | .qos_class')"
+
+ ((count == decoders)) || err "$LINENO";
+}
+
+check_qos_memdevs () {
+ # Check that memdevs that expose ram_qos_class or pmem_qos_class have
+ # expected fake value programmed.
+ json=$(cxl list -b cxl_test -M)
+ readarray -t lines < <(jq ".[] | .ram_size, .pmem_size, .ram_qos_class, .pmem_qos_class" <<<"$json")
+ for (( i = 0; i < ${#lines[@]}; i += 4 ))
+ do
+ ram_size=${lines[i]}
+ pmem_size=${lines[i+1]}
+ ram_qos_class=${lines[i+2]}
+ pmem_qos_class=${lines[i+3]}
+
+ if [[ "$ram_size" != null ]]
+ then
+ ((ram_qos_class == TEST_QOS_CLASS)) || err "$LINENO"
+ fi
+ if [[ "$pmem_size" != null ]]
+ then
+ ((pmem_qos_class == TEST_QOS_CLASS)) || err "$LINENO"
+ fi
+ done
+}
+
+
+. $(dirname $0)/common
+
+rc=77
+
+set -ex
+
+trap 'err $LINENO' ERR
+
+check_prereq "jq"
+
+modprobe -r cxl_test
+modprobe cxl_test
+rc=1
+
+check_qos_decoders
+
+check_qos_memdevs
+
+check_dmesg "$LINEO"
+
+modprobe -r cxl_test
diff --git a/test/meson.build b/test/meson.build
index 5eb35749a95b..4892df11119f 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -160,6 +160,7 @@ cxl_events = find_program('cxl-events.sh')
cxl_poison = find_program('cxl-poison.sh')
cxl_sanitize = find_program('cxl-sanitize.sh')
cxl_destroy_region = find_program('cxl-destroy-region.sh')
+cxl_qos_class = find_program('cxl-qos-class.sh')
tests = [
[ 'libndctl', libndctl, 'ndctl' ],
@@ -192,6 +193,7 @@ tests = [
[ 'cxl-poison.sh', cxl_poison, 'cxl' ],
[ 'cxl-sanitize.sh', cxl_sanitize, 'cxl' ],
[ 'cxl-destroy-region.sh', cxl_destroy_region, 'cxl' ],
+ [ 'cxl-qos-class.sh', cxl_qos_class, 'cxl' ],
]
if get_option('destructive').enabled()
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [NDCTL PATCH v5 4/4] ndctl: add test for qos_class in CXL test suite
2024-02-01 23:05 ` [NDCTL PATCH v5 4/4] ndctl: add test for qos_class in CXL test suite Dave Jiang
@ 2024-02-02 16:17 ` Alison Schofield
0 siblings, 0 replies; 8+ messages in thread
From: Alison Schofield @ 2024-02-02 16:17 UTC (permalink / raw)
To: Dave Jiang; +Cc: linux-cxl, nvdimm, vishal.l.verma
On Thu, Feb 01, 2024 at 04:05:07PM -0700, Dave Jiang wrote:
> Add tests in cxl-qos-class.sh to verify qos_class are set with the fake
> qos_class create by the kernel. Root decoders should have qos_class
> attribute set. Memory devices should have ram_qos_class or pmem_qos_class
> set depending on which partitions are valid.
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
> v5:
> - Split out from cxl-topology.sh (Vishal)
> ---
> test/common | 4 +++
> test/cxl-qos-class.sh | 65 +++++++++++++++++++++++++++++++++++++++++++
> test/meson.build | 2 ++
> 3 files changed, 71 insertions(+)
> create mode 100755 test/cxl-qos-class.sh
>
> diff --git a/test/common b/test/common
> index f1023ef20f7e..5694820c7adc 100644
> --- a/test/common
> +++ b/test/common
> @@ -150,3 +150,7 @@ check_dmesg()
> grep -q "Call Trace" <<< $log && err $1
> true
> }
> +
> +
> +# CXL COMMON
> +TEST_QOS_CLASS=42
> diff --git a/test/cxl-qos-class.sh b/test/cxl-qos-class.sh
> new file mode 100755
> index 000000000000..365a7df9c1e4
> --- /dev/null
> +++ b/test/cxl-qos-class.sh
> @@ -0,0 +1,65 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (C) 2024 Intel Corporation. All rights reserved.
> +
> +check_qos_decoders () {
> + # check root decoders have expected fake qos_class
> + # also make sure the number of root decoders equal to the number
> + # with qos_class found
> + json=$($CXL list -b cxl_test -D -d root)
> + decoders=$(echo "$json" | jq length)
> + count=0
> + while read -r qos_class
> + do
> + ((qos_class == TEST_QOS_CLASS)) || err "$LINENO"
> + count=$((count+1))
> + done <<< "$(echo "$json" | jq -r '.[] | .qos_class')"
> +
> + ((count == decoders)) || err "$LINENO";
> +}
> +
> +check_qos_memdevs () {
> + # Check that memdevs that expose ram_qos_class or pmem_qos_class have
> + # expected fake value programmed.
> + json=$(cxl list -b cxl_test -M)
> + readarray -t lines < <(jq ".[] | .ram_size, .pmem_size, .ram_qos_class, .pmem_qos_class" <<<"$json")
> + for (( i = 0; i < ${#lines[@]}; i += 4 ))
> + do
> + ram_size=${lines[i]}
> + pmem_size=${lines[i+1]}
> + ram_qos_class=${lines[i+2]}
> + pmem_qos_class=${lines[i+3]}
> +
> + if [[ "$ram_size" != null ]]
> + then
> + ((ram_qos_class == TEST_QOS_CLASS)) || err "$LINENO"
> + fi
> + if [[ "$pmem_size" != null ]]
> + then
> + ((pmem_qos_class == TEST_QOS_CLASS)) || err "$LINENO"
> + fi
> + done
> +}
> +
> +
> +. $(dirname $0)/common
> +
> +rc=77
> +
> +set -ex
> +
> +trap 'err $LINENO' ERR
> +
> +check_prereq "jq"
> +
> +modprobe -r cxl_test
> +modprobe cxl_test
> +rc=1
This different style, boiler plate not first in file, caught my eye.
Functionally no difference but stopped me for a second.
Reviewed-by: Alison Schofield <alison.schofield@intel.com>
> +
> +check_qos_decoders
> +
> +check_qos_memdevs
> +
> +check_dmesg "$LINEO"
> +
> +modprobe -r cxl_test
> diff --git a/test/meson.build b/test/meson.build
> index 5eb35749a95b..4892df11119f 100644
> --- a/test/meson.build
> +++ b/test/meson.build
> @@ -160,6 +160,7 @@ cxl_events = find_program('cxl-events.sh')
> cxl_poison = find_program('cxl-poison.sh')
> cxl_sanitize = find_program('cxl-sanitize.sh')
> cxl_destroy_region = find_program('cxl-destroy-region.sh')
> +cxl_qos_class = find_program('cxl-qos-class.sh')
>
> tests = [
> [ 'libndctl', libndctl, 'ndctl' ],
> @@ -192,6 +193,7 @@ tests = [
> [ 'cxl-poison.sh', cxl_poison, 'cxl' ],
> [ 'cxl-sanitize.sh', cxl_sanitize, 'cxl' ],
> [ 'cxl-destroy-region.sh', cxl_destroy_region, 'cxl' ],
> + [ 'cxl-qos-class.sh', cxl_qos_class, 'cxl' ],
> ]
>
> if get_option('destructive').enabled()
> --
> 2.43.0
>
>
^ permalink raw reply [flat|nested] 8+ messages in thread