[NDCTL PATCH resend 0/5] ndctl: Add support and test for CXL features driver

[NDCTL PATCH resend 0/5] ndctl: Add support and test for CXL features driver

[Dave Jiang]

resend due to missing parts in last patch. Forgot to do stg refresh.

The series provides support of libcxl enumerating the 'features' device
exported by the CXL kernel driver. It discovers the char device major
and minor numbers for the CXL features device in order to allow issuing
of ioctls to the device. libcxl will also associate the 'features' device
with an 'endpoint' port device in order to allow finding the appropriate
char dev from the endpoint device. It feels appropriate to associate
with the endpoint device rather than the memdev since the features device
comes out of the mailbox of a CLX device that may or may not be a memdev.

A unit test is added to locate a feature device exported by the cxl_test
kernel module and issue all the supported ioctls to verify that all the
ioctl paths are working as expected.

Dave Jiang (5):
      cxl: Add cxl_bus_get_by_provider()
      cxl: Enumerate features 'devices'
      cxl: Add get major and minor for cxl features device
      cxl: Associate CXL features device with CXL endpoint
      cxl/test: Add test for cxl features device

 cxl/lib/libcxl.c     | 308 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 cxl/lib/libcxl.sym   |  13 +++
 cxl/lib/private.h    |  16 ++++
 cxl/libcxl.h         |  17 ++++
 test/cxl-features.sh |  17 ++++
 test/fwctl.c         | 362 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 test/meson.build     |  16 ++++
 7 files changed, 749 insertions(+)

[NDCTL PATCH resend 1/5] cxl: Add cxl_bus_get_by_provider()

[Dave Jiang]

Add helper function cxl_bus_get_by_provider() in order to support unit
test that will utilize the API call.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 cxl/lib/libcxl.c   | 11 +++++++++++
 cxl/lib/libcxl.sym |  5 +++++
 cxl/libcxl.h       |  2 ++
 3 files changed, 18 insertions(+)

diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 63aa4ef3acdc..bab7343e8a4a 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -3358,6 +3358,17 @@ CXL_EXPORT struct cxl_ctx *cxl_bus_get_ctx(struct cxl_bus *bus)
 	return cxl_port_get_ctx(&bus->port);
 }
 
+CXL_EXPORT struct cxl_bus *cxl_bus_get_by_provider(struct cxl_ctx *ctx,
+						   const char *provider)
+{
+	struct cxl_bus *bus;
+
+	cxl_bus_foreach(ctx, bus)
+		if (strcmp(provider, cxl_bus_get_provider(bus)) == 0)
+			return bus;
+	return NULL;
+}
+
 CXL_EXPORT void cxl_cmd_unref(struct cxl_cmd *cmd)
 {
 	if (!cmd)
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 0c155a40ad47..1fc33cc6e1a4 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -288,3 +288,8 @@ global:
 	cxl_memdev_trigger_poison_list;
 	cxl_region_trigger_poison_list;
 } LIBCXL_7;
+
+LIBCXL_9 {
+global:
+	cxl_bus_get_by_provider;
+} LIBECXL_8;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 0a5fd0e13cc2..3b309968a808 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -122,6 +122,8 @@ int cxl_bus_get_id(struct cxl_bus *bus);
 struct cxl_port *cxl_bus_get_port(struct cxl_bus *bus);
 struct cxl_ctx *cxl_bus_get_ctx(struct cxl_bus *bus);
 int cxl_bus_disable_invalidate(struct cxl_bus *bus);
+struct cxl_bus *cxl_bus_get_by_provider(struct cxl_ctx *ctx,
+					const char *provider);
 
 #define cxl_bus_foreach(ctx, bus)                                              \
 	for (bus = cxl_bus_get_first(ctx); bus != NULL;                        \
-- 
2.47.1

[NDCTL PATCH resend 2/5] cxl: Enumerate features 'devices'

[Dave Jiang]

Add support to libcxl to enumerate the 'features' device exported by the
kernel. 'features' is a new 'struct device' exported by the kernel CLX
subsystem in order to support CXL mailbox feature commands. libcxl will
walk the sysfs and pick up the featuresN devices. The char device via
the FWCTL subsystem registered by the features driver is used to send
feature commands via ioctl to the kernel driver. The discovery and
initialization will identify the char device.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 cxl/lib/libcxl.c   | 180 +++++++++++++++++++++++++++++++++++++++++++++
 cxl/lib/libcxl.sym |   2 +
 cxl/lib/private.h  |  13 ++++
 cxl/libcxl.h       |   9 +++
 4 files changed, 204 insertions(+)

diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index bab7343e8a4a..8bc0394543dc 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -46,11 +46,13 @@ struct cxl_ctx {
 	void *userdata;
 	int memdevs_init;
 	int buses_init;
+	int features_init;
 	unsigned long timeout;
 	struct udev *udev;
 	struct udev_queue *udev_queue;
 	struct list_head memdevs;
 	struct list_head buses;
+	struct list_head features;
 	struct kmod_ctx *kmod_ctx;
 	struct daxctl_ctx *daxctl_ctx;
 	void *private_data;
@@ -91,6 +93,23 @@ static void free_memdev(struct cxl_memdev *memdev, struct list_head *head)
 	free(memdev);
 }
 
+static void __free_features(struct cxl_features *feat)
+{
+	free(feat->dev_buf);
+	free(feat->dev_path);
+	free(feat->host_path);
+	free(feat->hostname);
+	free(feat);
+}
+
+static void free_features(struct cxl_features *feat, struct list_head *head)
+{
+	if (head)
+		list_del_from(head, &feat->list);
+	kmod_module_unref(feat->module);
+	__free_features(feat);
+}
+
 static void free_target(struct cxl_target *target, struct list_head *head)
 {
 	if (head)
@@ -289,6 +308,7 @@ CXL_EXPORT int cxl_new(struct cxl_ctx **ctx)
 	*ctx = c;
 	list_head_init(&c->memdevs);
 	list_head_init(&c->buses);
+	list_head_init(&c->features);
 	c->kmod_ctx = kmod_ctx;
 	c->daxctl_ctx = daxctl_ctx;
 	c->udev = udev;
@@ -329,6 +349,7 @@ CXL_EXPORT struct cxl_ctx *cxl_ref(struct cxl_ctx *ctx)
  */
 CXL_EXPORT void cxl_unref(struct cxl_ctx *ctx)
 {
+	struct cxl_features *feat, *_f;
 	struct cxl_memdev *memdev, *_d;
 	struct cxl_bus *bus, *_b;
 
@@ -344,6 +365,9 @@ CXL_EXPORT void cxl_unref(struct cxl_ctx *ctx)
 	list_for_each_safe(&ctx->buses, bus, _b, port.list)
 		free_bus(bus, &ctx->buses);
 
+	list_for_each_safe(&ctx->features, feat, _f, list)
+		free_features(feat, &ctx->features);
+
 	udev_queue_unref(ctx->udev_queue);
 	udev_unref(ctx->udev);
 	kmod_unref(ctx->kmod_ctx);
@@ -4694,3 +4718,159 @@ CXL_EXPORT struct cxl_cmd *cxl_cmd_new_set_alert_config(struct cxl_memdev *memde
 {
 	return cxl_cmd_new_generic(memdev, CXL_MEM_COMMAND_ID_SET_ALERT_CONFIG);
 }
+
+static const char fwctl_prefix[] = "fwctl";
+static int get_feature_chardev(const char *base, char *chardev_path)
+{
+	char *path = calloc(1, strlen(base) + 100);
+	struct dirent *entry;
+	DIR *dir;
+	int rc;
+
+	if (!path)
+		return -ENOMEM;
+
+	sprintf(path, "%s/fwctl/", base);
+	dir = opendir(path);
+	if (!dir) {
+		rc = -errno;
+		goto err;
+	}
+
+	while ((entry = readdir(dir)) != NULL)
+		if (strncmp(entry->d_name, fwctl_prefix, strlen(fwctl_prefix)) == 0)
+			break;
+
+	if (!entry) {
+		rc = -ENOENT;
+		goto read_err;
+	}
+
+	sprintf(chardev_path, "/dev/fwctl/%s", entry->d_name);
+	closedir(dir);
+
+	return 0;
+
+read_err:
+	closedir(dir);
+err:
+	free(path);
+	return rc;
+}
+
+static void *add_cxl_features(void *parent, int id, const char *cxlfeat_base)
+{
+	const char *devname = devpath_to_devname(cxlfeat_base);
+	char *path = calloc(1, strlen(cxlfeat_base) + 100);
+	struct cxl_features *feat, *feat_dupe;
+	char *host, *host_path, *hostname;
+	struct cxl_ctx *ctx = parent;
+	char buf[SYSFS_ATTR_SIZE];
+	struct stat st;
+	int rc;
+
+	if (!path)
+		return NULL;
+
+	dbg(ctx, "%s: base: \'%s\'\n", devname, cxlfeat_base);
+
+	feat = calloc(1, sizeof(*feat));
+	if (!feat)
+		goto err_dev;
+
+	feat->id = id;
+	feat->ctx = ctx;
+	sprintf(path, "/dev/cxl/%s", devname);
+
+	rc = get_feature_chardev(cxlfeat_base, path);
+	if (rc) {
+		/* No fwctl entry, no need to enumerate the "device" */
+		__free_features(feat);
+		free(path);
+		return NULL;
+	}
+
+	if (stat(path, &st) < 0)
+		goto err_read;
+
+	feat->major = major(st.st_rdev);
+	feat->minor = minor(st.st_rdev);
+
+	feat->dev_path = strdup(cxlfeat_base);
+	if (!feat->dev_path)
+		goto err_read;
+
+	feat->host_path = realpath(cxlfeat_base, NULL);
+	if (!feat->host_path)
+		goto err_read;
+	host = strrchr(feat->host_path, '/');
+	if (!host)
+		goto err_read;
+	host[0] = '\0';
+
+	feat->dev_buf = calloc(1, strlen(cxlfeat_base) + 50);
+	if (!feat->dev_buf)
+		goto err_read;
+	feat->buf_len = strlen(cxlfeat_base) + 50;
+
+	sprintf(path, "%s/features", cxlfeat_base);
+	if (sysfs_read_attr(ctx, path, buf) < 0)
+		goto err_read;
+	feat->features = atoi(buf);
+
+	/*
+	 * Extract host device name for the features "device". This is used for
+	 * matching up with an endpoint uport.
+	 */
+	host_path = strdup(feat->host_path);
+	if (!host_path)
+		goto err_read;
+
+	hostname = strdup(devpath_to_devname(host_path));
+	if (!hostname)
+		goto err_read;
+
+	free(host_path);
+	feat->hostname = hostname;
+
+	cxl_features_foreach(ctx, feat_dupe)
+		if (feat_dupe->id == feat->id) {
+			__free_features(feat);
+			free(path);
+			return feat_dupe;
+		}
+
+	list_add(&ctx->features, &feat->list);
+	free(path);
+	return feat;
+
+ err_read:
+	__free_features(feat);
+ err_dev:
+	free(path);
+	return NULL;
+}
+
+static void cxl_features_init(struct cxl_ctx *ctx)
+{
+	if (ctx->features_init)
+		return;
+
+	ctx->features_init = 1;
+
+	device_parse(ctx, "/sys/bus/cxl/devices", "features", ctx, add_cxl_features);
+}
+
+CXL_EXPORT struct cxl_features *cxl_features_get_first(struct cxl_ctx *ctx)
+{
+	cxl_features_init(ctx);
+
+	return list_top(&ctx->features, struct cxl_features, list);
+}
+
+CXL_EXPORT struct cxl_features *cxl_features_get_next(struct cxl_features *features)
+{
+	struct cxl_ctx *ctx = features->ctx;
+
+	return list_next(&ctx->features, features, list);
+}
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 1fc33cc6e1a4..9b1708d8e86a 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -292,4 +292,6 @@ global:
 LIBCXL_9 {
 global:
 	cxl_bus_get_by_provider;
+	cxl_features_get_first;
+	cxl_features_get_next;
 } LIBECXL_8;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index b6cd910e9335..34785af4e64f 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -34,6 +34,19 @@ enum cxl_fwl_loading {
 	CXL_FWL_LOADING_START,
 };
 
+struct cxl_features {
+	int id, major, minor;
+	struct cxl_ctx *ctx;
+	void *dev_buf;
+	size_t buf_len;
+	char *host_path;
+	char *dev_path;
+	char *hostname;
+	struct kmod_module *module;
+	struct list_node list;
+	int features;
+};
+
 struct cxl_endpoint;
 struct cxl_memdev {
 	int id, major, minor;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 3b309968a808..7e94eb8bce24 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -489,6 +489,15 @@ int cxl_cmd_alert_config_set_enable_alert_actions(struct cxl_cmd *cmd,
 						  int enable);
 struct cxl_cmd *cxl_cmd_new_set_alert_config(struct cxl_memdev *memdev);
 
+struct cxl_features;
+#define cxl_features_foreach(ctx, features) \
+        for (features = cxl_features_get_first(ctx); \
+             features != NULL; \
+             features = cxl_features_get_next(features))
+
+struct cxl_features *cxl_features_get_first(struct cxl_ctx *ctx);
+struct cxl_features *cxl_features_get_next(struct cxl_features *features);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
-- 
2.47.1

[NDCTL PATCH resend 3/5] cxl: Add get major and minor for cxl features device

[Dave Jiang]

Add helper functions to retrieve the major and minor of the associated
char device for the cxl features device. The char device provides
access to issue ioctl via the FWCTL subsystem to the CXL features
driver for CXL mailbox feature commands.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 cxl/lib/libcxl.c   | 10 ++++++++++
 cxl/lib/libcxl.sym |  2 ++
 cxl/libcxl.h       |  2 ++
 3 files changed, 14 insertions(+)

diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 8bc0394543dc..f2da48278cb0 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -4874,3 +4874,13 @@ CXL_EXPORT struct cxl_features *cxl_features_get_next(struct cxl_features *featu
 
 	return list_next(&ctx->features, features, list);
 }
+
+CXL_EXPORT int cxl_features_get_major(struct cxl_features *features)
+{
+	return features->major;
+}
+
+CXL_EXPORT int cxl_features_get_minor(struct cxl_features *features)
+{
+	return features->minor;
+}
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 9b1708d8e86a..6d70d5b90377 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -294,4 +294,6 @@ global:
 	cxl_bus_get_by_provider;
 	cxl_features_get_first;
 	cxl_features_get_next;
+	cxl_features_get_major;
+	cxl_features_get_minor;
 } LIBECXL_8;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 7e94eb8bce24..5dcc60c8bf1a 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -497,6 +497,8 @@ struct cxl_features;
 
 struct cxl_features *cxl_features_get_first(struct cxl_ctx *ctx);
 struct cxl_features *cxl_features_get_next(struct cxl_features *features);
+int cxl_features_get_major(struct cxl_features *features);
+int cxl_features_get_minor(struct cxl_features *features);
 
 #ifdef __cplusplus
 } /* extern "C" */
-- 
2.47.1

[NDCTL PATCH resend 4/5] cxl: Associate CXL features device with CXL endpoint

[Dave Jiang]

Add support of cxl_features_get_bus() by setting up the association
between 'cxl endpoint' and 'cxl features'. Add related helper functions
for assisting the setup of the relationship with 'features' and 'endpoint'.
The 'endpoint' is considered as the parent device of 'features' given a
'features' device is exported per CXL mailbox. The association is
established by matching the feature's parent device with the endpoint's
uport deivce. For example, on cxl_test bus provider, both should match
to cxl_memN device.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 cxl/lib/libcxl.c   | 107 +++++++++++++++++++++++++++++++++++++++++++++
 cxl/lib/libcxl.sym |   4 ++
 cxl/lib/private.h  |   3 ++
 cxl/libcxl.h       |   4 ++
 4 files changed, 118 insertions(+)

diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index f2da48278cb0..f13b0055f332 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -213,6 +213,7 @@ static void free_port(struct cxl_port *port, struct list_head *head)
 static void free_endpoint(struct cxl_endpoint *endpoint, struct list_head *head)
 {
 	__free_port(&endpoint->port, head);
+	free(endpoint->uport_host);
 	free(endpoint);
 }
 
@@ -2044,6 +2045,7 @@ static void *add_cxl_endpoint(void *parent, int id, const char *cxlep_base)
 	struct cxl_endpoint *endpoint, *endpoint_dup;
 	struct cxl_port *port = parent;
 	struct cxl_ctx *ctx = cxl_port_get_ctx(port);
+	char *uport_path, *host_path;
 	int rc;
 
 	dbg(ctx, "%s: base: \'%s\'\n", devname, cxlep_base);
@@ -2057,6 +2059,17 @@ static void *add_cxl_endpoint(void *parent, int id, const char *cxlep_base)
 	if (rc)
 		goto err;
 
+
+	uport_path = strdup(endpoint->port.uport);
+	if (!uport_path)
+		goto err;
+
+	host_path = dirname(uport_path);
+	endpoint->uport_host = strdup(devpath_to_devname(host_path));
+	free(uport_path);
+	if (!endpoint->uport_host)
+		goto err;
+
 	cxl_endpoint_foreach(port, endpoint_dup)
 		if (endpoint_dup->port.id == endpoint->port.id) {
 			free_endpoint(endpoint, NULL);
@@ -2168,6 +2181,11 @@ cxl_endpoint_get_memdev(struct cxl_endpoint *endpoint)
 	return NULL;
 }
 
+CXL_EXPORT const char *cxl_endpoint_get_uport_hostname(struct cxl_endpoint *endpoint)
+{
+	return endpoint->uport_host;
+}
+
 static bool cxl_region_is_configured(struct cxl_region *region)
 {
 	return region->size && (region->decode_state != CXL_DECODE_RESET);
@@ -4884,3 +4902,92 @@ CXL_EXPORT int cxl_features_get_minor(struct cxl_features *features)
 {
 	return features->minor;
 }
+
+CXL_EXPORT const char *cxl_features_get_hostname(struct cxl_features *features)
+{
+	return features->hostname;
+}
+
+CXL_EXPORT const char *cxl_features_get_devname(struct cxl_features *features)
+{
+	return devpath_to_devname(features->dev_path);
+}
+
+static struct cxl_endpoint *
+cxl_port_recurse_endpoint_by_features(struct cxl_port *parent_port,
+				      struct cxl_features *features)
+{
+	struct cxl_endpoint *endpoint;
+	struct cxl_port *port;
+
+	cxl_port_foreach(parent_port, port) {
+		cxl_endpoint_foreach(port, endpoint)
+			if (strcmp(cxl_endpoint_get_uport_hostname(endpoint),
+				   cxl_features_get_hostname(features)) == 0)
+				return endpoint;
+		endpoint = cxl_port_recurse_endpoint_by_features(port, features);
+		if (endpoint)
+			return endpoint;
+	}
+
+	return NULL;
+}
+
+static struct cxl_endpoint *
+cxl_port_find_endpoint_by_features(struct cxl_port *parent_port,
+				   struct cxl_features *features)
+{
+	struct cxl_endpoint *endpoint;
+
+	cxl_endpoint_foreach(parent_port, endpoint)
+		if (strcmp(cxl_endpoint_get_uport_hostname(endpoint),
+			   cxl_features_get_hostname(features)) == 0)
+			return endpoint;
+	return cxl_port_recurse_endpoint_by_features(parent_port, features);
+}
+
+CXL_EXPORT struct cxl_endpoint *
+cxl_features_get_endpoint(struct cxl_features *features)
+{
+	struct cxl_ctx *ctx = cxl_features_get_ctx(features);
+	struct cxl_endpoint *endpoint = NULL;
+	struct cxl_bus *bus;
+
+	if (features->endpoint)
+		return features->endpoint;
+
+	cxl_bus_foreach(ctx, bus) {
+		struct cxl_port *port = cxl_bus_get_port(bus);
+
+		endpoint = cxl_port_find_endpoint_by_features(port, features);
+		if (endpoint)
+			break;
+	}
+
+	if (!endpoint)
+		return NULL;
+
+	if (endpoint->features && endpoint->features != features)
+		err(ctx, "%s assigned to %s not %s\n",
+		    cxl_endpoint_get_devname(endpoint),
+		    cxl_features_get_devname(endpoint->features),
+		    cxl_features_get_devname(features));
+	features->endpoint = endpoint;
+	endpoint->features = features;
+
+	return endpoint;
+}
+
+CXL_EXPORT struct cxl_bus *cxl_features_get_bus(struct cxl_features *features)
+{
+	struct cxl_endpoint *endpoint = cxl_features_get_endpoint(features);
+
+	if (!endpoint)
+		return NULL;
+	return cxl_endpoint_get_bus(endpoint);
+}
+
+CXL_EXPORT struct cxl_ctx *cxl_features_get_ctx(struct cxl_features *features)
+{
+	return features->ctx;
+}
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 6d70d5b90377..a29d42e1414b 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -296,4 +296,8 @@ global:
 	cxl_features_get_next;
 	cxl_features_get_major;
 	cxl_features_get_minor;
+	cxl_features_get_hostname;
+	cxl_features_get_bus;
+	cxl_features_get_ctx;
+	cxl_features_get_devname;
 } LIBECXL_8;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index 34785af4e64f..b38ee7d33a40 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -37,6 +37,7 @@ enum cxl_fwl_loading {
 struct cxl_features {
 	int id, major, minor;
 	struct cxl_ctx *ctx;
+	struct cxl_endpoint *endpoint;
 	void *dev_buf;
 	size_t buf_len;
 	char *host_path;
@@ -122,6 +123,8 @@ struct cxl_bus {
 struct cxl_endpoint {
 	struct cxl_port port;
 	struct cxl_memdev *memdev;
+	struct cxl_features *features;
+	char *uport_host;
 };
 
 struct cxl_target {
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index 5dcc60c8bf1a..610c13051b6d 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -499,6 +499,10 @@ struct cxl_features *cxl_features_get_first(struct cxl_ctx *ctx);
 struct cxl_features *cxl_features_get_next(struct cxl_features *features);
 int cxl_features_get_major(struct cxl_features *features);
 int cxl_features_get_minor(struct cxl_features *features);
+const char *cxl_features_get_devname(struct cxl_features *features);
+const char *cxl_features_get_hostname(struct cxl_features *features);
+struct cxl_bus *cxl_features_get_bus(struct cxl_features *features);
+struct cxl_ctx *cxl_features_get_ctx(struct cxl_features *features);
 
 #ifdef __cplusplus
 } /* extern "C" */
-- 
2.47.1

[NDCTL PATCH resend 5/5] cxl/test: Add test for cxl features device

[Dave Jiang]

Add a unit test to verify the features ioctl commands. Test support added
for locating a features device, retrieve and verify the supported features
commands, retrieve specific feature command data, retrieve test feature
data, and write and verify test feature data.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 test/cxl-features.sh |  19 +++
 test/fwctl.c         | 362 +++++++++++++++++++++++++++++++++++++++++++
 test/meson.build     |  17 ++
 3 files changed, 398 insertions(+)
 create mode 100755 test/cxl-features.sh
 create mode 100644 test/fwctl.c

diff --git a/test/cxl-features.sh b/test/cxl-features.sh
new file mode 100755
index 000000000000..e0351028e8fb
--- /dev/null
+++ b/test/cxl-features.sh
@@ -0,0 +1,19 @@
+#!/bin/bash -Ex
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2025 Intel Corporation. All rights reserved.
+
+rc=77
+
+. $(dirname $0)/common
+FWCTL="$TEST_PATH"/fwctl
+
+trap 'err $LINENO' ERR
+
+#check_min_kver "6.15" || do_skip "may lack CXL features support"
+
+modprobe cxl_test
+
+test -x "$FWCTL" || do_skip "no fwctl"
+"$FWCTL"
+
+_cxl_cleanup
diff --git a/test/fwctl.c b/test/fwctl.c
new file mode 100644
index 000000000000..897c9351750a
--- /dev/null
+++ b/test/fwctl.c
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2024-2025 Intel Corporation. All rights reserved.
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <endian.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <syslog.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <cxl/libcxl.h>
+#include <cxl/features.h>
+#include <fwctl/fwctl.h>
+#include <fwctl/cxl.h>
+#include <linux/uuid.h>
+#include <uuid/uuid.h>
+#include <util/bitmap.h>
+
+static const char provider[] = "cxl_test";
+
+UUID_DEFINE(test_uuid,
+	    0xff, 0xff, 0xff, 0xff,
+	    0xff, 0xff,
+	    0xff, 0xff,
+	    0xff, 0xff,
+	    0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+);
+
+#define GET_FEAT_SIZE	4
+#define SET_FEAT_SIZE	4
+#define EFFECTS_MASK	(BIT(0) | BIT(9))
+
+#define CMD_TEST_MASK (BIT(CXL_FEATURE_ID_GET_SUPPORTED_FEATURES) | \
+		       BIT(CXL_FEATURE_ID_GET_FEATURE) | \
+		       BIT(CXL_FEATURE_ID_SET_FEATURE))
+
+#define MAX_TEST_FEATURES	1
+#define DEFAULT_TEST_DATA	0xdeadbeef
+#define DEFAULT_TEST_DATA2	0xabcdabcd
+
+struct test_feature {
+	uuid_t uuid;
+	size_t get_size;
+	size_t set_size;
+};
+
+static int send_command(int fd, struct fwctl_rpc *rpc, struct fwctl_rpc_cxl_out *out)
+{
+	if (ioctl(fd, FWCTL_RPC, rpc) == -1) {
+		fprintf(stderr, "RPC ioctl error: %s\n", strerror(errno));
+		return -errno;
+	}
+
+	if (out->retval) {
+		fprintf(stderr, "operation returned failure: %d\n", out->retval);
+		return -ENXIO;
+	}
+
+	return 0;
+}
+
+static int cxl_fwctl_rpc_get_test_feature(int fd, struct test_feature *feat_ctx,
+					  const uint32_t expected_data)
+{
+	struct cxl_mbox_get_feat_in feat_in __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc_cxl in __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc_cxl_out *out;
+	struct fwctl_rpc rpc = {0};
+	size_t out_size;
+	uint32_t val;
+	void *data;
+	int rc;
+
+	uuid_copy(feat_in.uuid, feat_ctx->uuid);
+	feat_in.count = feat_ctx->get_size;
+
+	out_size = sizeof(*out) + feat_in.count;
+	out = aligned_alloc(16, out_size);
+	if (!out)
+		return -ENOMEM;
+
+	memset(out, 0, out_size);
+	data = out->payload;
+
+	in.command_id = CXL_FEATURE_ID_GET_FEATURE;
+	in.op_size = sizeof(feat_in);
+	in.in_payload = (uint64_t)(uint64_t *)&feat_in;
+
+	rpc.size = sizeof(rpc);
+	rpc.scope = FWCTL_RPC_CONFIGURATION;
+	rpc.in_len = sizeof(in);
+	rpc.out_len = out_size;
+	rpc.in = (uint64_t)(uint64_t *)&in;
+	rpc.out = (uint64_t)(uint64_t *)out;
+
+	rc = send_command(fd, &rpc, out);
+	if (rc)
+		goto out;
+
+	val = le32toh(*(__le32 *)data);
+	if (memcmp(&val, &expected_data, sizeof(val)) != 0) {
+		rc = -ENXIO;
+		goto out;
+	}
+
+out:
+	free(out);
+	return rc;
+}
+
+static int cxl_fwctl_rpc_set_test_feature(int fd, struct test_feature *feat_ctx)
+{
+	struct cxl_mbox_set_feat_in *feat_in;
+	struct fwctl_rpc_cxl in __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc_cxl_out out __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc rpc = {0};
+	size_t in_size;
+	uint32_t val;
+	void *data;
+	int rc;
+
+	in_size = sizeof(*feat_in) + sizeof(val);
+	feat_in = aligned_alloc(16, in_size);
+	if (!feat_in)
+		return -ENOMEM;
+
+	memset(feat_in, 0, in_size);
+	uuid_copy(feat_in->hdr.uuid, feat_ctx->uuid);
+	data = feat_in->data;
+	val = DEFAULT_TEST_DATA2;
+	*(uint32_t *)data = htole32(val);
+	feat_in->hdr.flags = CXL_SET_FEAT_FLAG_FULL_DATA_TRANSFER;
+
+	in.command_id = CXL_FEATURE_ID_SET_FEATURE;
+	in.op_size = in_size;
+	in.in_payload = (uint64_t)(uint64_t *)feat_in;
+
+	rpc.size = sizeof(rpc);
+	rpc.scope = FWCTL_RPC_DEBUG_WRITE_FULL;
+	rpc.in_len = in_size;
+	rpc.out_len = sizeof(out);
+	rpc.in = (uint64_t)(uint64_t *)&in;
+	rpc.out = (uint64_t)(uint64_t *)&out;
+
+	rc = send_command(fd, &rpc, &out);
+	if (rc)
+		goto out;
+
+	rc = cxl_fwctl_rpc_get_test_feature(fd, feat_ctx, DEFAULT_TEST_DATA2);
+	if (rc) {
+		fprintf(stderr, "Failed ioctl to get feature verify: %d\n", rc);
+		goto out;
+	}
+
+out:
+	free(feat_in);
+	return rc;
+}
+
+static int cxl_fwctl_rpc_get_supported_features(int fd, struct test_feature *feat_ctx)
+{
+	struct cxl_mbox_get_sup_feats_in feat_in __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc_cxl_out out __attribute__((aligned(16))) = {0};
+	struct fwctl_rpc_cxl in __attribute__((aligned(16))) = {0};
+	struct cxl_mbox_get_sup_feats_out *feat_out;
+	struct fwctl_rpc_cxl_out *out2;
+	struct cxl_feat_entry *entry;
+	struct fwctl_rpc rpc = {0};
+	size_t out_size;
+	int feats, rc;
+
+	/* First query, to get number of features w/o per feature data */
+	in.command_id = CXL_FEATURE_ID_GET_SUPPORTED_FEATURES;
+	in.op_size = sizeof(feat_in);
+	in.in_payload = (uint64_t)(uint64_t *)&feat_in;
+
+	rpc.size = sizeof(rpc);
+	rpc.scope = FWCTL_RPC_CONFIGURATION;
+	rpc.in_len = sizeof(in);
+	rpc.out_len = sizeof(out) + sizeof(*feat_out);
+	rpc.in = (uint64_t)(uint64_t *)&in;
+	rpc.out = (uint64_t)(uint64_t *)&out;
+
+	rc = send_command(fd, &rpc, &out);
+	if (rc)
+		return rc;
+
+	feat_out = (struct cxl_mbox_get_sup_feats_out *)&out.payload[0];
+	feats = le16toh(feat_out->supported_feats);
+	if (feats != MAX_TEST_FEATURES) {
+		fprintf(stderr, "Test device has greater than %d test features.\n",
+			MAX_TEST_FEATURES);
+		return -ENXIO;
+	}
+
+	/* Going second round to retrieve each feature details */
+	out_size = sizeof(*out2) + sizeof(*feat_out) + feats * sizeof(*entry);
+	rpc.out_len = out_size;
+	out2 = aligned_alloc(16, out_size);
+	if (!out2)
+		return -ENOMEM;
+
+	memset(out2, 0, out_size);
+	rpc.out = (uint64_t)(uint64_t *)out2;
+	feat_in.count = htole32(feats * sizeof(*entry));
+
+	rc = send_command(fd, &rpc, out2);
+	if (rc)
+		goto out;
+
+	feat_out = (struct cxl_mbox_get_sup_feats_out *)&out2->payload[0];
+	feats = le16toh(feat_out->supported_feats);
+	if (feats != MAX_TEST_FEATURES) {
+		fprintf(stderr, "Test device has greater than %u test features.\n",
+			MAX_TEST_FEATURES);
+		rc = -ENXIO;
+		goto out;
+	}
+
+	if (le16toh(feat_out->num_entries) != MAX_TEST_FEATURES) {
+		fprintf(stderr, "Test device did not return expected entries. %u\n",
+			le16toh(feat_out->num_entries));
+		rc = -ENXIO;
+		goto out;
+	}
+
+	entry = &feat_out->ents[0];
+	if (uuid_compare(test_uuid, entry->uuid) != 0) {
+		fprintf(stderr, "Test device did not export expected test feature.\n");
+		rc = -ENXIO;
+		goto out;
+	}
+
+	if (le16toh(entry->get_feat_size) != GET_FEAT_SIZE ||
+	    le16toh(entry->set_feat_size) != SET_FEAT_SIZE) {
+		fprintf(stderr, "Test device feature in/out size incorrect.\n");
+		rc = -ENXIO;
+		goto out;
+	}
+
+	if (le16toh(entry->effects) != EFFECTS_MASK) {
+		fprintf(stderr, "Test device set effects incorrect\n");
+		rc = -ENXIO;
+		goto out;
+	}
+
+	uuid_copy(feat_ctx->uuid, entry->uuid);
+	feat_ctx->get_size = le16toh(entry->get_feat_size);
+	feat_ctx->set_size = le16toh(entry->set_feat_size);
+
+out:
+	free(out2);
+	return rc;
+}
+
+static int cxl_fwctl_retrieve_info(int fd)
+{
+	struct fwctl_info info = {0};
+	struct fwctl_info_cxl cxl_info __attribute__((aligned(16))) = {0};
+
+	info.size = sizeof(info);
+	info.out_device_type = FWCTL_DEVICE_TYPE_CXL;
+	info.device_data_len = sizeof(cxl_info);
+	info.out_device_data = (uint64_t)(uint64_t *)&cxl_info;
+
+	if (ioctl(fd, FWCTL_INFO, &info) == -1) {
+		fprintf(stderr, "INFO ioctl error: %s\n", strerror(errno));
+		return -errno;
+	}
+
+	if (cxl_info.cmd_mask != CMD_TEST_MASK)
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+static int test_fwctl_feat(struct cxl_features *feat)
+{
+	struct test_feature feat_ctx;
+	int major, minor, fd, rc;
+	char path[256];
+
+	major = cxl_features_get_major(feat);
+	minor = cxl_features_get_minor(feat);
+	sprintf(path, "/dev/char/%d:%d", major, minor);
+
+	fd = open(path, O_RDONLY, 0644);
+	if (!fd) {
+		fprintf(stderr, "Failed to open: %d\n", -errno);
+		return -errno;
+	}
+
+	rc = cxl_fwctl_retrieve_info(fd);
+	if (rc) {
+		fprintf(stderr, "Failed ioctl to retrieve drv info: %d\n", rc);
+		goto out;
+	}
+
+	rc = cxl_fwctl_rpc_get_supported_features(fd, &feat_ctx);
+	if (rc) {
+		fprintf(stderr, "Failed ioctl to get supported features: %d\n", rc);
+		goto out;
+	}
+
+	rc = cxl_fwctl_rpc_get_test_feature(fd, &feat_ctx, DEFAULT_TEST_DATA);
+	if (rc) {
+		fprintf(stderr, "Failed ioctl to get feature: %d\n", rc);
+		goto out;
+	}
+
+	rc = cxl_fwctl_rpc_set_test_feature(fd, &feat_ctx);
+	if (rc) {
+		fprintf(stderr, "Failed ioctl to set feature: %d\n", rc);
+		goto out;
+	}
+
+out:
+	close(fd);
+	return rc;
+}
+
+static int test_fwctl(struct cxl_ctx *ctx, struct cxl_bus *bus)
+{
+	struct cxl_features *feat;
+
+	cxl_features_foreach(ctx, feat) {
+		if (cxl_features_get_bus(feat) != bus)
+			continue;
+		return test_fwctl_feat(feat);
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct cxl_ctx *ctx;
+	struct cxl_bus *bus;
+	int rc;
+
+	rc = cxl_new(&ctx);
+	if (rc < 0)
+		return rc;
+
+	cxl_set_log_priority(ctx, LOG_DEBUG);
+
+	bus = cxl_bus_get_by_provider(ctx, provider);
+	if (!bus) {
+		fprintf(stderr, "%s: unable to find bus (%s)\n",
+			argv[0], provider);
+		rc = -ENODEV;
+		goto out;
+	}
+
+	rc = test_fwctl(ctx, bus);
+
+out:
+	cxl_unref(ctx);
+	return rc;
+}
diff --git a/test/meson.build b/test/meson.build
index d871e28e17ce..5704bc4659b5 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -17,6 +17,13 @@ ndctl_deps = libndctl_deps + [
   versiondep,
 ]
 
+libcxl_deps = [
+  cxl_dep,
+  ndctl_dep,
+  uuid,
+  kmod,
+]
+
 libndctl = executable('libndctl', testcore + [ 'libndctl.c'],
   dependencies : libndctl_deps,
   include_directories : root_inc,
@@ -130,6 +137,13 @@ revoke_devmem = executable('revoke_devmem', testcore + [
   include_directories : root_inc,
 )
 
+fwctl = executable('fwctl', testcore + [
+    'fwctl.c',
+  ],
+  dependencies : libcxl_deps,
+  include_directories : root_inc,
+)
+
 mmap = executable('mmap', 'mmap.c',)
 
 create = find_program('create.sh')
@@ -161,6 +175,7 @@ cxl_sanitize = find_program('cxl-sanitize.sh')
 cxl_destroy_region = find_program('cxl-destroy-region.sh')
 cxl_qos_class = find_program('cxl-qos-class.sh')
 cxl_poison = find_program('cxl-poison.sh')
+cxl_features = find_program('cxl-features.sh')
 
 tests = [
   [ 'libndctl',               libndctl,		  'ndctl' ],
@@ -194,6 +209,7 @@ tests = [
   [ 'cxl-destroy-region.sh',  cxl_destroy_region, 'cxl'   ],
   [ 'cxl-qos-class.sh',       cxl_qos_class,      'cxl'   ],
   [ 'cxl-poison.sh',          cxl_poison,         'cxl'   ],
+  [ 'cxl-features.sh',        cxl_features,       'cxl'   ],
 ]
 
 if get_option('destructive').enabled()
@@ -249,6 +265,7 @@ foreach t : tests
       daxdev_errors,
       dax_dev,
       mmap,
+      fwctl,
     ],
     suite: t[2],
     timeout : 600,
-- 
2.47.1