* [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-19 6:31 ` Alejandro Lucero Palau
2024-07-26 17:28 ` Jonathan Cameron
2024-07-18 21:32 ` [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data Dave Jiang
` (14 subsequent siblings)
15 siblings, 2 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Create a new 'struct cxl_mailbox' and move all mailbox related bits to
it. This allows isolation of all CXL mailbox data in order to export
some of the calls to external caller (fwctl) and avoid exporting of
CXL driver specific bits such has device states.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
MAINTAINERS | 1 +
drivers/cxl/core/mbox.c | 48 ++++++++++++++++++---------
drivers/cxl/core/memdev.c | 18 +++++++----
drivers/cxl/cxlmem.h | 49 ++--------------------------
drivers/cxl/pci.c | 58 +++++++++++++++++++--------------
drivers/cxl/pmem.c | 4 ++-
include/linux/cxl/mailbox.h | 63 ++++++++++++++++++++++++++++++++++++
tools/testing/cxl/test/mem.c | 27 ++++++++++------
8 files changed, 163 insertions(+), 105 deletions(-)
create mode 100644 include/linux/cxl/mailbox.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 9a8ca772b186..3e1934b7e044 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -5466,6 +5466,7 @@ S: Maintained
F: drivers/cxl/
F: include/linux/einj-cxl.h
F: include/linux/cxl-event.h
+F: include/linux/cxl/
F: include/uapi/linux/cxl_mem.h
F: tools/testing/cxl/
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 2626f3fff201..783cb5ed823f 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -244,16 +244,17 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
struct cxl_mbox_cmd *mbox_cmd)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
size_t out_size, min_out;
int rc;
- if (mbox_cmd->size_in > mds->payload_size ||
- mbox_cmd->size_out > mds->payload_size)
+ if (mbox_cmd->size_in > cxl_mbox->payload_size ||
+ mbox_cmd->size_out > cxl_mbox->payload_size)
return -E2BIG;
out_size = mbox_cmd->size_out;
min_out = mbox_cmd->min_out;
- rc = mds->mbox_send(mds, mbox_cmd);
+ rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
/*
* EIO is reserved for a payload size mismatch and mbox_send()
* may not return this error.
@@ -353,6 +354,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
struct cxl_memdev_state *mds, u16 opcode,
size_t in_size, size_t out_size, u64 in_payload)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
*mbox = (struct cxl_mbox_cmd) {
.opcode = opcode,
.size_in = in_size,
@@ -374,7 +376,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
/* Prepare to handle a full payload for variable sized output */
if (out_size == CXL_VARIABLE_PAYLOAD)
- mbox->size_out = mds->payload_size;
+ mbox->size_out = cxl_mbox->payload_size;
else
mbox->size_out = out_size;
@@ -398,6 +400,8 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
if (send_cmd->raw.rsvd)
return -EINVAL;
@@ -406,7 +410,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
* gets passed along without further checking, so it must be
* validated here.
*/
- if (send_cmd->out.size > mds->payload_size)
+ if (send_cmd->out.size > cxl_mbox->payload_size)
return -EINVAL;
if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
@@ -494,6 +498,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
struct cxl_memdev_state *mds,
const struct cxl_send_command *send_cmd)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mem_command mem_cmd;
int rc;
@@ -505,7 +510,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
* supports, but output can be arbitrarily large (simply write out as
* much data as the hardware provides).
*/
- if (send_cmd->in.size > mds->payload_size)
+ if (send_cmd->in.size > cxl_mbox->payload_size)
return -EINVAL;
/* Sanitize and construct a cxl_mem_command */
@@ -591,6 +596,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
u64 out_payload, s32 *size_out,
u32 *retval)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct device *dev = mds->cxlds.dev;
int rc;
@@ -601,7 +607,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
cxl_mem_opcode_to_name(mbox_cmd->opcode),
mbox_cmd->opcode, mbox_cmd->size_in);
- rc = mds->mbox_send(mds, mbox_cmd);
+ rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
if (rc)
goto out;
@@ -659,11 +665,12 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
u32 *size, u8 *out)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
u32 remaining = *size;
u32 offset = 0;
while (remaining) {
- u32 xfer_size = min_t(u32, remaining, mds->payload_size);
+ u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size);
struct cxl_mbox_cmd mbox_cmd;
struct cxl_mbox_get_log log;
int rc;
@@ -752,17 +759,18 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_supported_logs *ret;
struct cxl_mbox_cmd mbox_cmd;
int rc;
- ret = kvmalloc(mds->payload_size, GFP_KERNEL);
+ ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
if (!ret)
return ERR_PTR(-ENOMEM);
mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
- .size_out = mds->payload_size,
+ .size_out = cxl_mbox->payload_size,
.payload_out = ret,
/* At least the record number field must be valid */
.min_out = 2,
@@ -910,6 +918,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
enum cxl_event_log_type log,
struct cxl_get_event_payload *get_pl)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_clear_event_payload *payload;
u16 total = le16_to_cpu(get_pl->record_count);
u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
@@ -920,8 +929,8 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
int i;
/* Payload size may limit the max handles */
- if (pl_size > mds->payload_size) {
- max_handles = (mds->payload_size - sizeof(*payload)) /
+ if (pl_size > cxl_mbox->payload_size) {
+ max_handles = (cxl_mbox->payload_size - sizeof(*payload)) /
sizeof(__le16);
pl_size = struct_size(payload, handles, max_handles);
}
@@ -979,6 +988,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
enum cxl_event_log_type type)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
struct device *dev = mds->cxlds.dev;
struct cxl_get_event_payload *payload;
@@ -995,7 +1005,7 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
.payload_in = &log_type,
.size_in = sizeof(log_type),
.payload_out = payload,
- .size_out = mds->payload_size,
+ .size_out = cxl_mbox->payload_size,
.min_out = struct_size(payload, records, 0),
};
@@ -1328,6 +1338,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
struct cxl_region *cxlr)
{
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_poison_out *po;
struct cxl_mbox_poison_in pi;
int nr_records = 0;
@@ -1346,7 +1357,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
.opcode = CXL_MBOX_OP_GET_POISON,
.size_in = sizeof(pi),
.payload_in = &pi,
- .size_out = mds->payload_size,
+ .size_out = cxl_mbox->payload_size,
.payload_out = po,
.min_out = struct_size(po, record, 0),
};
@@ -1382,7 +1393,9 @@ static void free_poison_buf(void *buf)
/* Get Poison List output buffer is protected by mds->poison.lock */
static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
{
- mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
+ mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
if (!mds->poison.list_out)
return -ENOMEM;
@@ -1411,6 +1424,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
{
struct cxl_memdev_state *mds;
+ struct cxl_mailbox *cxl_mbox;
mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
if (!mds) {
@@ -1418,7 +1432,9 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
return ERR_PTR(-ENOMEM);
}
- mutex_init(&mds->mbox_mutex);
+ cxl_mbox = &mds->cxlds.cxl_mbox;
+ mutex_init(&cxl_mbox->mbox_mutex);
+
mutex_init(&mds->event.log_lock);
mds->cxlds.dev = dev;
mds->cxlds.reg_map.host = dev;
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 0277726afd04..05bb84cb1274 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -58,7 +58,7 @@ static ssize_t payload_max_show(struct device *dev,
if (!mds)
return sysfs_emit(buf, "\n");
- return sysfs_emit(buf, "%zu\n", mds->payload_size);
+ return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
}
static DEVICE_ATTR_RO(payload_max);
@@ -124,15 +124,16 @@ static ssize_t security_state_show(struct device *dev,
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
unsigned long state = mds->security.state;
int rc = 0;
/* sync with latest submission state */
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
if (mds->security.sanitize_active)
rc = sysfs_emit(buf, "sanitize\n");
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
if (rc)
return rc;
@@ -829,12 +830,13 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
{
struct cxl_memdev_state *mds = fwl->dd_handle;
struct cxl_mbox_transfer_fw *transfer;
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
if (!size)
return FW_UPLOAD_ERR_INVALID_SIZE;
mds->fw.oneshot = struct_size(transfer, data, size) <
- mds->payload_size;
+ cxl_mbox->payload_size;
if (cxl_mem_get_fw_info(mds))
return FW_UPLOAD_ERR_HW_ERROR;
@@ -854,6 +856,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
{
struct cxl_memdev_state *mds = fwl->dd_handle;
struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
struct cxl_memdev *cxlmd = cxlds->cxlmd;
struct cxl_mbox_transfer_fw *transfer;
struct cxl_mbox_cmd mbox_cmd;
@@ -877,7 +880,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
* sizeof(*transfer) is 128. These constraints imply that @cur_size
* will always be 128b aligned.
*/
- cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
+ cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
remaining = size - cur_size;
size_in = struct_size(transfer, data, cur_size);
@@ -1059,16 +1062,17 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
static void sanitize_teardown_notifier(void *data)
{
struct cxl_memdev_state *mds = data;
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct kernfs_node *state;
/*
* Prevent new irq triggered invocations of the workqueue and
* flush inflight invocations.
*/
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
state = mds->security.sanitize_node;
mds->security.sanitize_node = NULL;
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
cancel_delayed_work_sync(&mds->security.poll_dwork);
sysfs_put(state);
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index af8169ccdbc0..8f683fe08410 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -8,6 +8,7 @@
#include <linux/rcuwait.h>
#include <linux/cxl-event.h>
#include <linux/node.h>
+#include <linux/cxl/mailbox.h>
#include "cxl.h"
/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */
@@ -105,42 +106,6 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
return xa_load(&port->endpoints, (unsigned long)&cxlmd->dev);
}
-/**
- * struct cxl_mbox_cmd - A command to be submitted to hardware.
- * @opcode: (input) The command set and command submitted to hardware.
- * @payload_in: (input) Pointer to the input payload.
- * @payload_out: (output) Pointer to the output payload. Must be allocated by
- * the caller.
- * @size_in: (input) Number of bytes to load from @payload_in.
- * @size_out: (input) Max number of bytes loaded into @payload_out.
- * (output) Number of bytes generated by the device. For fixed size
- * outputs commands this is always expected to be deterministic. For
- * variable sized output commands, it tells the exact number of bytes
- * written.
- * @min_out: (input) internal command output payload size validation
- * @poll_count: (input) Number of timeouts to attempt.
- * @poll_interval_ms: (input) Time between mailbox background command polling
- * interval timeouts.
- * @return_code: (output) Error code returned from hardware.
- *
- * This is the primary mechanism used to send commands to the hardware.
- * All the fields except @payload_* correspond exactly to the fields described in
- * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
- * @payload_out are written to, and read from the Command Payload Registers
- * defined in CXL 2.0 8.2.8.4.8.
- */
-struct cxl_mbox_cmd {
- u16 opcode;
- void *payload_in;
- void *payload_out;
- size_t size_in;
- size_t size_out;
- size_t min_out;
- int poll_count;
- int poll_interval_ms;
- u16 return_code;
-};
-
/*
* Per CXL 3.0 Section 8.2.8.4.5.1
*/
@@ -438,6 +403,7 @@ struct cxl_dev_state {
struct resource ram_res;
u64 serial;
enum cxl_devtype type;
+ struct cxl_mailbox cxl_mbox;
};
/**
@@ -448,11 +414,8 @@ struct cxl_dev_state {
* the functionality related to that like Identify Memory Device and Get
* Partition Info
* @cxlds: Core driver state common across Type-2 and Type-3 devices
- * @payload_size: Size of space for payload
- * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register)
* @lsa_size: Size of Label Storage Area
* (CXL 2.0 8.2.9.5.1.1 Identify Memory Device)
- * @mbox_mutex: Mutex to synchronize mailbox access.
* @firmware_version: Firmware version for the memory device.
* @enabled_cmds: Hardware commands found enabled in CEL.
* @exclusive_cmds: Commands that are kernel-internal only
@@ -470,17 +433,13 @@ struct cxl_dev_state {
* @poison: poison driver state info
* @security: security driver state info
* @fw: firmware upload / activation state
- * @mbox_wait: RCU wait for mbox send completely
- * @mbox_send: @dev specific transport for transmitting mailbox commands
*
* See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for
* details on capacity parameters.
*/
struct cxl_memdev_state {
struct cxl_dev_state cxlds;
- size_t payload_size;
size_t lsa_size;
- struct mutex mbox_mutex; /* Protects device mailbox and firmware */
char firmware_version[0x10];
DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
@@ -500,10 +459,6 @@ struct cxl_memdev_state {
struct cxl_poison_state poison;
struct cxl_security_state security;
struct cxl_fw_state fw;
-
- struct rcuwait mbox_wait;
- int (*mbox_send)(struct cxl_memdev_state *mds,
- struct cxl_mbox_cmd *cmd);
};
static inline struct cxl_memdev_state *
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index e53646e9f2fb..45b184b6ef97 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -124,6 +124,7 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
u16 opcode;
struct cxl_dev_id *dev_id = id;
struct cxl_dev_state *cxlds = dev_id->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
if (!cxl_mbox_background_complete(cxlds))
@@ -132,13 +133,13 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
if (opcode == CXL_MBOX_OP_SANITIZE) {
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
if (mds->security.sanitize_node)
mod_delayed_work(system_wq, &mds->security.poll_dwork, 0);
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
} else {
/* short-circuit the wait in __cxl_pci_mbox_send_cmd() */
- rcuwait_wake_up(&mds->mbox_wait);
+ rcuwait_wake_up(&cxl_mbox->mbox_wait);
}
return IRQ_HANDLED;
@@ -152,8 +153,9 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
struct cxl_memdev_state *mds =
container_of(work, typeof(*mds), security.poll_dwork.work);
struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
if (cxl_mbox_background_complete(cxlds)) {
mds->security.poll_tmo_secs = 0;
if (mds->security.sanitize_node)
@@ -167,7 +169,7 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
mds->security.poll_tmo_secs = min(15 * 60, timeout);
schedule_delayed_work(&mds->security.poll_dwork, timeout * HZ);
}
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
}
/**
@@ -192,17 +194,20 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
* not need to coordinate with each other. The driver only uses the primary
* mailbox.
*/
-static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
+static int __cxl_pci_mbox_send_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *mbox_cmd)
{
- struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_dev_state *cxlds = container_of(cxl_mbox,
+ struct cxl_dev_state,
+ cxl_mbox);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
struct device *dev = cxlds->dev;
u64 cmd_reg, status_reg;
size_t out_len;
int rc;
- lockdep_assert_held(&mds->mbox_mutex);
+ lockdep_assert_held(&cxl_mbox->mbox_mutex);
/*
* Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
@@ -315,10 +320,10 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
timeout = mbox_cmd->poll_interval_ms;
for (i = 0; i < mbox_cmd->poll_count; i++) {
- if (rcuwait_wait_event_timeout(&mds->mbox_wait,
- cxl_mbox_background_complete(cxlds),
- TASK_UNINTERRUPTIBLE,
- msecs_to_jiffies(timeout)) > 0)
+ if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait,
+ cxl_mbox_background_complete(cxlds),
+ TASK_UNINTERRUPTIBLE,
+ msecs_to_jiffies(timeout)) > 0)
break;
}
@@ -360,7 +365,7 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
*/
size_t n;
- n = min3(mbox_cmd->size_out, mds->payload_size, out_len);
+ n = min3(mbox_cmd->size_out, cxl_mbox->payload_size, out_len);
memcpy_fromio(mbox_cmd->payload_out, payload, n);
mbox_cmd->size_out = n;
} else {
@@ -370,14 +375,14 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
return 0;
}
-static int cxl_pci_mbox_send(struct cxl_memdev_state *mds,
+static int cxl_pci_mbox_send(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd)
{
int rc;
- mutex_lock_io(&mds->mbox_mutex);
- rc = __cxl_pci_mbox_send_cmd(mds, cmd);
- mutex_unlock(&mds->mbox_mutex);
+ mutex_lock_io(&cxl_mbox->mbox_mutex);
+ rc = __cxl_pci_mbox_send_cmd(cxl_mbox, cmd);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
return rc;
}
@@ -385,6 +390,7 @@ static int cxl_pci_mbox_send(struct cxl_memdev_state *mds,
static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
{
struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
struct device *dev = cxlds->dev;
unsigned long timeout;
@@ -392,6 +398,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
u64 md_status;
u32 ctrl;
+ cxl_mbox->host = dev;
timeout = jiffies + mbox_ready_timeout * HZ;
do {
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
@@ -417,8 +424,8 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
return -ETIMEDOUT;
}
- mds->mbox_send = cxl_pci_mbox_send;
- mds->payload_size =
+ cxl_mbox->mbox_send = cxl_pci_mbox_send;
+ cxl_mbox->payload_size =
1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
/*
@@ -428,16 +435,16 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
* there's no point in going forward. If the size is too large, there's
* no harm is soft limiting it.
*/
- mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M);
- if (mds->payload_size < 256) {
+ cxl_mbox->payload_size = min_t(size_t, cxl_mbox->payload_size, SZ_1M);
+ if (cxl_mbox->payload_size < 256) {
dev_err(dev, "Mailbox is too small (%zub)",
- mds->payload_size);
+ cxl_mbox->payload_size);
return -ENXIO;
}
- dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size);
+ dev_dbg(dev, "Mailbox payload sized %zu", cxl_mbox->payload_size);
- rcuwait_init(&mds->mbox_wait);
+ rcuwait_init(&cxl_mbox->mbox_wait);
INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work);
/* background command interrupts are optional */
@@ -578,9 +585,10 @@ static void free_event_buf(void *buf)
*/
static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_get_event_payload *buf;
- buf = kvmalloc(mds->payload_size, GFP_KERNEL);
+ buf = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
if (!buf)
return -ENOMEM;
mds->event.buf = buf;
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 2ecdaee63021..9f62dc03c883 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -102,13 +102,15 @@ static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
struct nd_cmd_get_config_size *cmd,
unsigned int buf_len)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
if (sizeof(*cmd) > buf_len)
return -EINVAL;
*cmd = (struct nd_cmd_get_config_size){
.config_size = mds->lsa_size,
.max_xfer =
- mds->payload_size - sizeof(struct cxl_mbox_set_lsa),
+ cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),
};
return 0;
diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
new file mode 100644
index 000000000000..654df6175828
--- /dev/null
+++ b/include/linux/cxl/mailbox.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2024 Intel Corporation. */
+#ifndef __CXL_MBOX_H__
+#define __CXL_MBOX_H__
+
+#include <linux/auxiliary_bus.h>
+
+/**
+ * struct cxl_mbox_cmd - A command to be submitted to hardware.
+ * @opcode: (input) The command set and command submitted to hardware.
+ * @payload_in: (input) Pointer to the input payload.
+ * @payload_out: (output) Pointer to the output payload. Must be allocated by
+ * the caller.
+ * @size_in: (input) Number of bytes to load from @payload_in.
+ * @size_out: (input) Max number of bytes loaded into @payload_out.
+ * (output) Number of bytes generated by the device. For fixed size
+ * outputs commands this is always expected to be deterministic. For
+ * variable sized output commands, it tells the exact number of bytes
+ * written.
+ * @min_out: (input) internal command output payload size validation
+ * @poll_count: (input) Number of timeouts to attempt.
+ * @poll_interval_ms: (input) Time between mailbox background command polling
+ * interval timeouts.
+ * @return_code: (output) Error code returned from hardware.
+ *
+ * This is the primary mechanism used to send commands to the hardware.
+ * All the fields except @payload_* correspond exactly to the fields described in
+ * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
+ * @payload_out are written to, and read from the Command Payload Registers
+ * defined in CXL 2.0 8.2.8.4.8.
+ */
+struct cxl_mbox_cmd {
+ u16 opcode;
+ void *payload_in;
+ void *payload_out;
+ size_t size_in;
+ size_t size_out;
+ size_t min_out;
+ int poll_count;
+ int poll_interval_ms;
+ u16 return_code;
+};
+
+/**
+ * struct cxl_mailbox - context for CXL mailbox operations
+ * @host: device that hosts the mailbox
+ * @adev: auxiliary device for fw-ctl
+ * @payload_size: Size of space for payload
+ * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register)
+ * @mbox_mutex: mutex protects device mailbox and firmware
+ * @mbox_wait: rcuwait for mailbox
+ * @mbox_send: @dev specific transport for transmitting mailbox commands
+ */
+struct cxl_mailbox {
+ struct device *host;
+ struct auxiliary_device adev; /* For fw-ctl */
+ size_t payload_size;
+ struct mutex mbox_mutex; /* lock to protect mailbox context */
+ struct rcuwait mbox_wait;
+ int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
+};
+
+#endif
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index eaf091a3d331..20d58f883ca2 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -8,6 +8,7 @@
#include <linux/delay.h>
#include <linux/sizes.h>
#include <linux/bits.h>
+#include <linux/cxl/mailbox.h>
#include <asm/unaligned.h>
#include <crypto/sha2.h>
#include <cxlmem.h>
@@ -530,6 +531,7 @@ static int mock_gsl(struct cxl_mbox_cmd *cmd)
static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_log *gl = cmd->payload_in;
u32 offset = le32_to_cpu(gl->offset);
u32 length = le32_to_cpu(gl->length);
@@ -538,7 +540,7 @@ static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd)
if (cmd->size_in < sizeof(*gl))
return -EINVAL;
- if (length > mds->payload_size)
+ if (length > cxl_mbox->payload_size)
return -EINVAL;
if (offset + length > sizeof(mock_cel))
return -EINVAL;
@@ -613,12 +615,13 @@ void cxl_mockmem_sanitize_work(struct work_struct *work)
{
struct cxl_memdev_state *mds =
container_of(work, typeof(*mds), security.poll_dwork.work);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
if (mds->security.sanitize_node)
sysfs_notify_dirent(mds->security.sanitize_node);
mds->security.sanitize_active = false;
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
dev_dbg(mds->cxlds.dev, "sanitize complete\n");
}
@@ -627,6 +630,7 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata,
struct cxl_mbox_cmd *cmd)
{
struct cxl_memdev_state *mds = mdata->mds;
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
int rc = 0;
if (cmd->size_in != 0)
@@ -644,14 +648,14 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata,
return -ENXIO;
}
- mutex_lock(&mds->mbox_mutex);
+ mutex_lock(&cxl_mbox->mbox_mutex);
if (schedule_delayed_work(&mds->security.poll_dwork,
msecs_to_jiffies(mdata->sanitize_timeout))) {
mds->security.sanitize_active = true;
dev_dbg(mds->cxlds.dev, "sanitize issued\n");
} else
rc = -EBUSY;
- mutex_unlock(&mds->mbox_mutex);
+ mutex_unlock(&cxl_mbox->mbox_mutex);
return rc;
}
@@ -1330,10 +1334,13 @@ static int mock_activate_fw(struct cxl_mockmem_data *mdata,
return -EINVAL;
}
-static int cxl_mock_mbox_send(struct cxl_memdev_state *mds,
+static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd)
{
- struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_dev_state *cxlds = container_of(cxl_mbox,
+ struct cxl_dev_state,
+ cxl_mbox);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
struct device *dev = cxlds->dev;
struct cxl_mockmem_data *mdata = dev_get_drvdata(dev);
int rc = -EIO;
@@ -1457,6 +1464,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
struct cxl_memdev_state *mds;
struct cxl_dev_state *cxlds;
struct cxl_mockmem_data *mdata;
+ struct cxl_mailbox *cxl_mbox;
int rc;
mdata = devm_kzalloc(dev, sizeof(*mdata), GFP_KERNEL);
@@ -1484,9 +1492,10 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
if (IS_ERR(mds))
return PTR_ERR(mds);
+ cxl_mbox = &mds->cxlds.cxl_mbox;
mdata->mds = mds;
- mds->mbox_send = cxl_mock_mbox_send;
- mds->payload_size = SZ_4K;
+ cxl_mbox->mbox_send = cxl_mock_mbox_send;
+ cxl_mbox->payload_size = SZ_4K;
mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf;
INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mockmem_sanitize_work);
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context
2024-07-18 21:32 ` [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context Dave Jiang
@ 2024-07-19 6:31 ` Alejandro Lucero Palau
2024-07-19 15:47 ` Dave Jiang
2024-07-26 17:28 ` Jonathan Cameron
1 sibling, 1 reply; 47+ messages in thread
From: Alejandro Lucero Palau @ 2024-07-19 6:31 UTC (permalink / raw)
To: Dave Jiang, linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
On 7/18/24 22:32, Dave Jiang wrote:
> Create a new 'struct cxl_mailbox' and move all mailbox related bits to
> it. This allows isolation of all CXL mailbox data in order to export
> some of the calls to external caller (fwctl) and avoid exporting of
> CXL driver specific bits such has device states.
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
> MAINTAINERS | 1 +
> drivers/cxl/core/mbox.c | 48 ++++++++++++++++++---------
> drivers/cxl/core/memdev.c | 18 +++++++----
> drivers/cxl/cxlmem.h | 49 ++--------------------------
> drivers/cxl/pci.c | 58 +++++++++++++++++++--------------
> drivers/cxl/pmem.c | 4 ++-
> include/linux/cxl/mailbox.h | 63 ++++++++++++++++++++++++++++++++++++
> tools/testing/cxl/test/mem.c | 27 ++++++++++------
> 8 files changed, 163 insertions(+), 105 deletions(-)
> create mode 100644 include/linux/cxl/mailbox.h
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 9a8ca772b186..3e1934b7e044 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -5466,6 +5466,7 @@ S: Maintained
> F: drivers/cxl/
> F: include/linux/einj-cxl.h
> F: include/linux/cxl-event.h
> +F: include/linux/cxl/
> F: include/uapi/linux/cxl_mem.h
> F: tools/testing/cxl/
>
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 2626f3fff201..783cb5ed823f 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -244,16 +244,17 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
> int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
> struct cxl_mbox_cmd *mbox_cmd)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> size_t out_size, min_out;
> int rc;
>
> - if (mbox_cmd->size_in > mds->payload_size ||
> - mbox_cmd->size_out > mds->payload_size)
> + if (mbox_cmd->size_in > cxl_mbox->payload_size ||
> + mbox_cmd->size_out > cxl_mbox->payload_size)
> return -E2BIG;
>
> out_size = mbox_cmd->size_out;
> min_out = mbox_cmd->min_out;
> - rc = mds->mbox_send(mds, mbox_cmd);
> + rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
> /*
> * EIO is reserved for a payload size mismatch and mbox_send()
> * may not return this error.
> @@ -353,6 +354,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
> struct cxl_memdev_state *mds, u16 opcode,
> size_t in_size, size_t out_size, u64 in_payload)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> *mbox = (struct cxl_mbox_cmd) {
> .opcode = opcode,
> .size_in = in_size,
> @@ -374,7 +376,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
>
> /* Prepare to handle a full payload for variable sized output */
> if (out_size == CXL_VARIABLE_PAYLOAD)
> - mbox->size_out = mds->payload_size;
> + mbox->size_out = cxl_mbox->payload_size;
> else
> mbox->size_out = out_size;
>
> @@ -398,6 +400,8 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
> const struct cxl_send_command *send_cmd,
> struct cxl_memdev_state *mds)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> +
> if (send_cmd->raw.rsvd)
> return -EINVAL;
>
> @@ -406,7 +410,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
> * gets passed along without further checking, so it must be
> * validated here.
> */
> - if (send_cmd->out.size > mds->payload_size)
> + if (send_cmd->out.size > cxl_mbox->payload_size)
> return -EINVAL;
>
> if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
> @@ -494,6 +498,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
> struct cxl_memdev_state *mds,
> const struct cxl_send_command *send_cmd)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mem_command mem_cmd;
> int rc;
>
> @@ -505,7 +510,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
> * supports, but output can be arbitrarily large (simply write out as
> * much data as the hardware provides).
> */
> - if (send_cmd->in.size > mds->payload_size)
> + if (send_cmd->in.size > cxl_mbox->payload_size)
> return -EINVAL;
>
> /* Sanitize and construct a cxl_mem_command */
> @@ -591,6 +596,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
> u64 out_payload, s32 *size_out,
> u32 *retval)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct device *dev = mds->cxlds.dev;
> int rc;
>
> @@ -601,7 +607,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
> cxl_mem_opcode_to_name(mbox_cmd->opcode),
> mbox_cmd->opcode, mbox_cmd->size_in);
>
> - rc = mds->mbox_send(mds, mbox_cmd);
> + rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
> if (rc)
> goto out;
>
> @@ -659,11 +665,12 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
> static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
> u32 *size, u8 *out)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> u32 remaining = *size;
> u32 offset = 0;
>
> while (remaining) {
> - u32 xfer_size = min_t(u32, remaining, mds->payload_size);
> + u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size);
> struct cxl_mbox_cmd mbox_cmd;
> struct cxl_mbox_get_log log;
> int rc;
> @@ -752,17 +759,18 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
>
> static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_get_supported_logs *ret;
> struct cxl_mbox_cmd mbox_cmd;
> int rc;
>
> - ret = kvmalloc(mds->payload_size, GFP_KERNEL);
> + ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
> if (!ret)
> return ERR_PTR(-ENOMEM);
>
> mbox_cmd = (struct cxl_mbox_cmd) {
> .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
> - .size_out = mds->payload_size,
> + .size_out = cxl_mbox->payload_size,
> .payload_out = ret,
> /* At least the record number field must be valid */
> .min_out = 2,
> @@ -910,6 +918,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
> enum cxl_event_log_type log,
> struct cxl_get_event_payload *get_pl)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_clear_event_payload *payload;
> u16 total = le16_to_cpu(get_pl->record_count);
> u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
> @@ -920,8 +929,8 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
> int i;
>
> /* Payload size may limit the max handles */
> - if (pl_size > mds->payload_size) {
> - max_handles = (mds->payload_size - sizeof(*payload)) /
> + if (pl_size > cxl_mbox->payload_size) {
> + max_handles = (cxl_mbox->payload_size - sizeof(*payload)) /
> sizeof(__le16);
> pl_size = struct_size(payload, handles, max_handles);
> }
> @@ -979,6 +988,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
> static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
> enum cxl_event_log_type type)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
> struct device *dev = mds->cxlds.dev;
> struct cxl_get_event_payload *payload;
> @@ -995,7 +1005,7 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
> .payload_in = &log_type,
> .size_in = sizeof(log_type),
> .payload_out = payload,
> - .size_out = mds->payload_size,
> + .size_out = cxl_mbox->payload_size,
> .min_out = struct_size(payload, records, 0),
> };
>
> @@ -1328,6 +1338,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
> struct cxl_region *cxlr)
> {
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_poison_out *po;
> struct cxl_mbox_poison_in pi;
> int nr_records = 0;
> @@ -1346,7 +1357,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
> .opcode = CXL_MBOX_OP_GET_POISON,
> .size_in = sizeof(pi),
> .payload_in = &pi,
> - .size_out = mds->payload_size,
> + .size_out = cxl_mbox->payload_size,
> .payload_out = po,
> .min_out = struct_size(po, record, 0),
> };
> @@ -1382,7 +1393,9 @@ static void free_poison_buf(void *buf)
> /* Get Poison List output buffer is protected by mds->poison.lock */
> static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
> {
> - mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> +
> + mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
> if (!mds->poison.list_out)
> return -ENOMEM;
>
> @@ -1411,6 +1424,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
> struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
> {
> struct cxl_memdev_state *mds;
> + struct cxl_mailbox *cxl_mbox;
>
> mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
> if (!mds) {
> @@ -1418,7 +1432,9 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
> return ERR_PTR(-ENOMEM);
> }
>
> - mutex_init(&mds->mbox_mutex);
> + cxl_mbox = &mds->cxlds.cxl_mbox;
> + mutex_init(&cxl_mbox->mbox_mutex);
> +
> mutex_init(&mds->event.log_lock);
> mds->cxlds.dev = dev;
> mds->cxlds.reg_map.host = dev;
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 0277726afd04..05bb84cb1274 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -58,7 +58,7 @@ static ssize_t payload_max_show(struct device *dev,
>
> if (!mds)
> return sysfs_emit(buf, "\n");
> - return sysfs_emit(buf, "%zu\n", mds->payload_size);
> + return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
> }
> static DEVICE_ATTR_RO(payload_max);
>
> @@ -124,15 +124,16 @@ static ssize_t security_state_show(struct device *dev,
> {
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct cxl_dev_state *cxlds = cxlmd->cxlds;
> + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
> unsigned long state = mds->security.state;
> int rc = 0;
>
> /* sync with latest submission state */
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> if (mds->security.sanitize_active)
> rc = sysfs_emit(buf, "sanitize\n");
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
> if (rc)
> return rc;
>
> @@ -829,12 +830,13 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
> {
> struct cxl_memdev_state *mds = fwl->dd_handle;
> struct cxl_mbox_transfer_fw *transfer;
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
>
> if (!size)
> return FW_UPLOAD_ERR_INVALID_SIZE;
>
> mds->fw.oneshot = struct_size(transfer, data, size) <
> - mds->payload_size;
> + cxl_mbox->payload_size;
>
> if (cxl_mem_get_fw_info(mds))
> return FW_UPLOAD_ERR_HW_ERROR;
> @@ -854,6 +856,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
> {
> struct cxl_memdev_state *mds = fwl->dd_handle;
> struct cxl_dev_state *cxlds = &mds->cxlds;
> + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
> struct cxl_memdev *cxlmd = cxlds->cxlmd;
> struct cxl_mbox_transfer_fw *transfer;
> struct cxl_mbox_cmd mbox_cmd;
> @@ -877,7 +880,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
> * sizeof(*transfer) is 128. These constraints imply that @cur_size
> * will always be 128b aligned.
> */
> - cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
> + cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
>
> remaining = size - cur_size;
> size_in = struct_size(transfer, data, cur_size);
> @@ -1059,16 +1062,17 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
> static void sanitize_teardown_notifier(void *data)
> {
> struct cxl_memdev_state *mds = data;
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct kernfs_node *state;
>
> /*
> * Prevent new irq triggered invocations of the workqueue and
> * flush inflight invocations.
> */
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> state = mds->security.sanitize_node;
> mds->security.sanitize_node = NULL;
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
>
> cancel_delayed_work_sync(&mds->security.poll_dwork);
> sysfs_put(state);
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index af8169ccdbc0..8f683fe08410 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -8,6 +8,7 @@
> #include <linux/rcuwait.h>
> #include <linux/cxl-event.h>
> #include <linux/node.h>
> +#include <linux/cxl/mailbox.h>
> #include "cxl.h"
>
> /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */
> @@ -105,42 +106,6 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
> return xa_load(&port->endpoints, (unsigned long)&cxlmd->dev);
> }
>
> -/**
> - * struct cxl_mbox_cmd - A command to be submitted to hardware.
> - * @opcode: (input) The command set and command submitted to hardware.
> - * @payload_in: (input) Pointer to the input payload.
> - * @payload_out: (output) Pointer to the output payload. Must be allocated by
> - * the caller.
> - * @size_in: (input) Number of bytes to load from @payload_in.
> - * @size_out: (input) Max number of bytes loaded into @payload_out.
> - * (output) Number of bytes generated by the device. For fixed size
> - * outputs commands this is always expected to be deterministic. For
> - * variable sized output commands, it tells the exact number of bytes
> - * written.
> - * @min_out: (input) internal command output payload size validation
> - * @poll_count: (input) Number of timeouts to attempt.
> - * @poll_interval_ms: (input) Time between mailbox background command polling
> - * interval timeouts.
> - * @return_code: (output) Error code returned from hardware.
> - *
> - * This is the primary mechanism used to send commands to the hardware.
> - * All the fields except @payload_* correspond exactly to the fields described in
> - * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
> - * @payload_out are written to, and read from the Command Payload Registers
> - * defined in CXL 2.0 8.2.8.4.8.
> - */
> -struct cxl_mbox_cmd {
> - u16 opcode;
> - void *payload_in;
> - void *payload_out;
> - size_t size_in;
> - size_t size_out;
> - size_t min_out;
> - int poll_count;
> - int poll_interval_ms;
> - u16 return_code;
> -};
> -
> /*
> * Per CXL 3.0 Section 8.2.8.4.5.1
> */
> @@ -438,6 +403,7 @@ struct cxl_dev_state {
> struct resource ram_res;
> u64 serial;
> enum cxl_devtype type;
> + struct cxl_mailbox cxl_mbox;
> };
>
Being a mailbox an optional capability for Type2, not sure it should be
here. Maybe it should but as a pointer for dynamically creating the
struct if needed.
> /**
> @@ -448,11 +414,8 @@ struct cxl_dev_state {
> * the functionality related to that like Identify Memory Device and Get
> * Partition Info
> * @cxlds: Core driver state common across Type-2 and Type-3 devices
> - * @payload_size: Size of space for payload
> - * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register)
> * @lsa_size: Size of Label Storage Area
> * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device)
> - * @mbox_mutex: Mutex to synchronize mailbox access.
> * @firmware_version: Firmware version for the memory device.
> * @enabled_cmds: Hardware commands found enabled in CEL.
> * @exclusive_cmds: Commands that are kernel-internal only
> @@ -470,17 +433,13 @@ struct cxl_dev_state {
> * @poison: poison driver state info
> * @security: security driver state info
> * @fw: firmware upload / activation state
> - * @mbox_wait: RCU wait for mbox send completely
> - * @mbox_send: @dev specific transport for transmitting mailbox commands
> *
> * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for
> * details on capacity parameters.
> */
> struct cxl_memdev_state {
> struct cxl_dev_state cxlds;
> - size_t payload_size;
> size_t lsa_size;
> - struct mutex mbox_mutex; /* Protects device mailbox and firmware */
> char firmware_version[0x10];
> DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
> DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
> @@ -500,10 +459,6 @@ struct cxl_memdev_state {
> struct cxl_poison_state poison;
> struct cxl_security_state security;
> struct cxl_fw_state fw;
> -
> - struct rcuwait mbox_wait;
> - int (*mbox_send)(struct cxl_memdev_state *mds,
> - struct cxl_mbox_cmd *cmd);
> };
>
> static inline struct cxl_memdev_state *
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index e53646e9f2fb..45b184b6ef97 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -124,6 +124,7 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
> u16 opcode;
> struct cxl_dev_id *dev_id = id;
> struct cxl_dev_state *cxlds = dev_id->cxlds;
> + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
>
> if (!cxl_mbox_background_complete(cxlds))
> @@ -132,13 +133,13 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
> reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
> opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
> if (opcode == CXL_MBOX_OP_SANITIZE) {
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> if (mds->security.sanitize_node)
> mod_delayed_work(system_wq, &mds->security.poll_dwork, 0);
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
> } else {
> /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */
> - rcuwait_wake_up(&mds->mbox_wait);
> + rcuwait_wake_up(&cxl_mbox->mbox_wait);
> }
>
> return IRQ_HANDLED;
> @@ -152,8 +153,9 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
> struct cxl_memdev_state *mds =
> container_of(work, typeof(*mds), security.poll_dwork.work);
> struct cxl_dev_state *cxlds = &mds->cxlds;
> + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
>
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> if (cxl_mbox_background_complete(cxlds)) {
> mds->security.poll_tmo_secs = 0;
> if (mds->security.sanitize_node)
> @@ -167,7 +169,7 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
> mds->security.poll_tmo_secs = min(15 * 60, timeout);
> schedule_delayed_work(&mds->security.poll_dwork, timeout * HZ);
> }
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
> }
>
> /**
> @@ -192,17 +194,20 @@ static void cxl_mbox_sanitize_work(struct work_struct *work)
> * not need to coordinate with each other. The driver only uses the primary
> * mailbox.
> */
> -static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
> +static int __cxl_pci_mbox_send_cmd(struct cxl_mailbox *cxl_mbox,
> struct cxl_mbox_cmd *mbox_cmd)
> {
> - struct cxl_dev_state *cxlds = &mds->cxlds;
> + struct cxl_dev_state *cxlds = container_of(cxl_mbox,
> + struct cxl_dev_state,
> + cxl_mbox);
> + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
> void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
> struct device *dev = cxlds->dev;
> u64 cmd_reg, status_reg;
> size_t out_len;
> int rc;
>
> - lockdep_assert_held(&mds->mbox_mutex);
> + lockdep_assert_held(&cxl_mbox->mbox_mutex);
>
> /*
> * Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
> @@ -315,10 +320,10 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
>
> timeout = mbox_cmd->poll_interval_ms;
> for (i = 0; i < mbox_cmd->poll_count; i++) {
> - if (rcuwait_wait_event_timeout(&mds->mbox_wait,
> - cxl_mbox_background_complete(cxlds),
> - TASK_UNINTERRUPTIBLE,
> - msecs_to_jiffies(timeout)) > 0)
> + if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait,
> + cxl_mbox_background_complete(cxlds),
> + TASK_UNINTERRUPTIBLE,
> + msecs_to_jiffies(timeout)) > 0)
> break;
> }
>
> @@ -360,7 +365,7 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
> */
> size_t n;
>
> - n = min3(mbox_cmd->size_out, mds->payload_size, out_len);
> + n = min3(mbox_cmd->size_out, cxl_mbox->payload_size, out_len);
> memcpy_fromio(mbox_cmd->payload_out, payload, n);
> mbox_cmd->size_out = n;
> } else {
> @@ -370,14 +375,14 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
> return 0;
> }
>
> -static int cxl_pci_mbox_send(struct cxl_memdev_state *mds,
> +static int cxl_pci_mbox_send(struct cxl_mailbox *cxl_mbox,
> struct cxl_mbox_cmd *cmd)
> {
> int rc;
>
> - mutex_lock_io(&mds->mbox_mutex);
> - rc = __cxl_pci_mbox_send_cmd(mds, cmd);
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_lock_io(&cxl_mbox->mbox_mutex);
> + rc = __cxl_pci_mbox_send_cmd(cxl_mbox, cmd);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
>
> return rc;
> }
> @@ -385,6 +390,7 @@ static int cxl_pci_mbox_send(struct cxl_memdev_state *mds,
> static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
> {
> struct cxl_dev_state *cxlds = &mds->cxlds;
> + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
> const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
> struct device *dev = cxlds->dev;
> unsigned long timeout;
> @@ -392,6 +398,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
> u64 md_status;
> u32 ctrl;
>
> + cxl_mbox->host = dev;
> timeout = jiffies + mbox_ready_timeout * HZ;
> do {
> md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
> @@ -417,8 +424,8 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
> return -ETIMEDOUT;
> }
>
> - mds->mbox_send = cxl_pci_mbox_send;
> - mds->payload_size =
> + cxl_mbox->mbox_send = cxl_pci_mbox_send;
> + cxl_mbox->payload_size =
> 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
>
> /*
> @@ -428,16 +435,16 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
> * there's no point in going forward. If the size is too large, there's
> * no harm is soft limiting it.
> */
> - mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M);
> - if (mds->payload_size < 256) {
> + cxl_mbox->payload_size = min_t(size_t, cxl_mbox->payload_size, SZ_1M);
> + if (cxl_mbox->payload_size < 256) {
> dev_err(dev, "Mailbox is too small (%zub)",
> - mds->payload_size);
> + cxl_mbox->payload_size);
> return -ENXIO;
> }
>
> - dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size);
> + dev_dbg(dev, "Mailbox payload sized %zu", cxl_mbox->payload_size);
>
> - rcuwait_init(&mds->mbox_wait);
> + rcuwait_init(&cxl_mbox->mbox_wait);
> INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work);
>
> /* background command interrupts are optional */
> @@ -578,9 +585,10 @@ static void free_event_buf(void *buf)
> */
> static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_get_event_payload *buf;
>
> - buf = kvmalloc(mds->payload_size, GFP_KERNEL);
> + buf = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
> if (!buf)
> return -ENOMEM;
> mds->event.buf = buf;
> diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
> index 2ecdaee63021..9f62dc03c883 100644
> --- a/drivers/cxl/pmem.c
> +++ b/drivers/cxl/pmem.c
> @@ -102,13 +102,15 @@ static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
> struct nd_cmd_get_config_size *cmd,
> unsigned int buf_len)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> +
> if (sizeof(*cmd) > buf_len)
> return -EINVAL;
>
> *cmd = (struct nd_cmd_get_config_size){
> .config_size = mds->lsa_size,
> .max_xfer =
> - mds->payload_size - sizeof(struct cxl_mbox_set_lsa),
> + cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),
> };
>
> return 0;
> diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
> new file mode 100644
> index 000000000000..654df6175828
> --- /dev/null
> +++ b/include/linux/cxl/mailbox.h
> @@ -0,0 +1,63 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/* Copyright(c) 2024 Intel Corporation. */
> +#ifndef __CXL_MBOX_H__
> +#define __CXL_MBOX_H__
> +
> +#include <linux/auxiliary_bus.h>
> +
> +/**
> + * struct cxl_mbox_cmd - A command to be submitted to hardware.
> + * @opcode: (input) The command set and command submitted to hardware.
> + * @payload_in: (input) Pointer to the input payload.
> + * @payload_out: (output) Pointer to the output payload. Must be allocated by
> + * the caller.
> + * @size_in: (input) Number of bytes to load from @payload_in.
> + * @size_out: (input) Max number of bytes loaded into @payload_out.
> + * (output) Number of bytes generated by the device. For fixed size
> + * outputs commands this is always expected to be deterministic. For
> + * variable sized output commands, it tells the exact number of bytes
> + * written.
> + * @min_out: (input) internal command output payload size validation
> + * @poll_count: (input) Number of timeouts to attempt.
> + * @poll_interval_ms: (input) Time between mailbox background command polling
> + * interval timeouts.
> + * @return_code: (output) Error code returned from hardware.
> + *
> + * This is the primary mechanism used to send commands to the hardware.
> + * All the fields except @payload_* correspond exactly to the fields described in
> + * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
> + * @payload_out are written to, and read from the Command Payload Registers
> + * defined in CXL 2.0 8.2.8.4.8.
> + */
> +struct cxl_mbox_cmd {
> + u16 opcode;
> + void *payload_in;
> + void *payload_out;
> + size_t size_in;
> + size_t size_out;
> + size_t min_out;
> + int poll_count;
> + int poll_interval_ms;
> + u16 return_code;
> +};
> +
> +/**
> + * struct cxl_mailbox - context for CXL mailbox operations
> + * @host: device that hosts the mailbox
> + * @adev: auxiliary device for fw-ctl
> + * @payload_size: Size of space for payload
> + * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register)
> + * @mbox_mutex: mutex protects device mailbox and firmware
> + * @mbox_wait: rcuwait for mailbox
> + * @mbox_send: @dev specific transport for transmitting mailbox commands
> + */
> +struct cxl_mailbox {
> + struct device *host;
> + struct auxiliary_device adev; /* For fw-ctl */
> + size_t payload_size;
> + struct mutex mbox_mutex; /* lock to protect mailbox context */
> + struct rcuwait mbox_wait;
> + int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
> +};
> +
> +#endif
> diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
> index eaf091a3d331..20d58f883ca2 100644
> --- a/tools/testing/cxl/test/mem.c
> +++ b/tools/testing/cxl/test/mem.c
> @@ -8,6 +8,7 @@
> #include <linux/delay.h>
> #include <linux/sizes.h>
> #include <linux/bits.h>
> +#include <linux/cxl/mailbox.h>
> #include <asm/unaligned.h>
> #include <crypto/sha2.h>
> #include <cxlmem.h>
> @@ -530,6 +531,7 @@ static int mock_gsl(struct cxl_mbox_cmd *cmd)
>
> static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd)
> {
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_get_log *gl = cmd->payload_in;
> u32 offset = le32_to_cpu(gl->offset);
> u32 length = le32_to_cpu(gl->length);
> @@ -538,7 +540,7 @@ static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd)
>
> if (cmd->size_in < sizeof(*gl))
> return -EINVAL;
> - if (length > mds->payload_size)
> + if (length > cxl_mbox->payload_size)
> return -EINVAL;
> if (offset + length > sizeof(mock_cel))
> return -EINVAL;
> @@ -613,12 +615,13 @@ void cxl_mockmem_sanitize_work(struct work_struct *work)
> {
> struct cxl_memdev_state *mds =
> container_of(work, typeof(*mds), security.poll_dwork.work);
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
>
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> if (mds->security.sanitize_node)
> sysfs_notify_dirent(mds->security.sanitize_node);
> mds->security.sanitize_active = false;
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
>
> dev_dbg(mds->cxlds.dev, "sanitize complete\n");
> }
> @@ -627,6 +630,7 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata,
> struct cxl_mbox_cmd *cmd)
> {
> struct cxl_memdev_state *mds = mdata->mds;
> + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> int rc = 0;
>
> if (cmd->size_in != 0)
> @@ -644,14 +648,14 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata,
> return -ENXIO;
> }
>
> - mutex_lock(&mds->mbox_mutex);
> + mutex_lock(&cxl_mbox->mbox_mutex);
> if (schedule_delayed_work(&mds->security.poll_dwork,
> msecs_to_jiffies(mdata->sanitize_timeout))) {
> mds->security.sanitize_active = true;
> dev_dbg(mds->cxlds.dev, "sanitize issued\n");
> } else
> rc = -EBUSY;
> - mutex_unlock(&mds->mbox_mutex);
> + mutex_unlock(&cxl_mbox->mbox_mutex);
>
> return rc;
> }
> @@ -1330,10 +1334,13 @@ static int mock_activate_fw(struct cxl_mockmem_data *mdata,
> return -EINVAL;
> }
>
> -static int cxl_mock_mbox_send(struct cxl_memdev_state *mds,
> +static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
> struct cxl_mbox_cmd *cmd)
> {
> - struct cxl_dev_state *cxlds = &mds->cxlds;
> + struct cxl_dev_state *cxlds = container_of(cxl_mbox,
> + struct cxl_dev_state,
> + cxl_mbox);
> + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
> struct device *dev = cxlds->dev;
> struct cxl_mockmem_data *mdata = dev_get_drvdata(dev);
> int rc = -EIO;
> @@ -1457,6 +1464,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
> struct cxl_memdev_state *mds;
> struct cxl_dev_state *cxlds;
> struct cxl_mockmem_data *mdata;
> + struct cxl_mailbox *cxl_mbox;
> int rc;
>
> mdata = devm_kzalloc(dev, sizeof(*mdata), GFP_KERNEL);
> @@ -1484,9 +1492,10 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
> if (IS_ERR(mds))
> return PTR_ERR(mds);
>
> + cxl_mbox = &mds->cxlds.cxl_mbox;
> mdata->mds = mds;
> - mds->mbox_send = cxl_mock_mbox_send;
> - mds->payload_size = SZ_4K;
> + cxl_mbox->mbox_send = cxl_mock_mbox_send;
> + cxl_mbox->payload_size = SZ_4K;
> mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf;
> INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mockmem_sanitize_work);
>
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context
2024-07-19 6:31 ` Alejandro Lucero Palau
@ 2024-07-19 15:47 ` Dave Jiang
2024-07-19 16:07 ` Alejandro Lucero Palau
0 siblings, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-07-19 15:47 UTC (permalink / raw)
To: Alejandro Lucero Palau, linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
On 7/18/24 11:31 PM, Alejandro Lucero Palau wrote:
>
> On 7/18/24 22:32, Dave Jiang wrote:
<snip>
>> /*
>> * Per CXL 3.0 Section 8.2.8.4.5.1
>> */
>> @@ -438,6 +403,7 @@ struct cxl_dev_state {
>> struct resource ram_res;
>> u64 serial;
>> enum cxl_devtype type;
>> + struct cxl_mailbox cxl_mbox;
>> };
>>
>
>
> Being a mailbox an optional capability for Type2, not sure it should be here. Maybe it should but as a pointer for dynamically creating the struct if needed.
>
>
Yes.... Which actually also ties in with your Type2 work. I talked to Dan last night while looking at your patch where the a software cap mask gets passed in for mailbox and HDM enumeration. The conclusion was that we need to refactor the code to deal with the optional component enumeration. For mailbox, it needs to be independent of memdev device state and the enumeration needs to be split out. I'm going to try to take a crack at it, unless you want to do it. :)
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context
2024-07-19 15:47 ` Dave Jiang
@ 2024-07-19 16:07 ` Alejandro Lucero Palau
0 siblings, 0 replies; 47+ messages in thread
From: Alejandro Lucero Palau @ 2024-07-19 16:07 UTC (permalink / raw)
To: Dave Jiang, linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
On 7/19/24 16:47, Dave Jiang wrote:
> On 7/18/24 11:31 PM, Alejandro Lucero Palau wrote:
>> On 7/18/24 22:32, Dave Jiang wrote:
> <snip>
>
>>> /*
>>> * Per CXL 3.0 Section 8.2.8.4.5.1
>>> */
>>> @@ -438,6 +403,7 @@ struct cxl_dev_state {
>>> struct resource ram_res;
>>> u64 serial;
>>> enum cxl_devtype type;
>>> + struct cxl_mailbox cxl_mbox;
>>> };
>>>
>>
>> Being a mailbox an optional capability for Type2, not sure it should be here. Maybe it should but as a pointer for dynamically creating the struct if needed.
>>
>>
> Yes.... Which actually also ties in with your Type2 work. I talked to Dan last night while looking at your patch where the a software cap mask gets passed in for mailbox and HDM enumeration. The conclusion was that we need to refactor the code to deal with the optional component enumeration. For mailbox, it needs to be independent of memdev device state and the enumeration needs to be split out. I'm going to try to take a crack at it, unless you want to do it. :)
>
That sounds good. I think your work with current mailbox needs gives you
a better position than mine, but happy to help/discuss/review. An happy
to know my concerns about current core struct not unfounded.
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context
2024-07-18 21:32 ` [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context Dave Jiang
2024-07-19 6:31 ` Alejandro Lucero Palau
@ 2024-07-26 17:28 ` Jonathan Cameron
1 sibling, 0 replies; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-26 17:28 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:19 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> Create a new 'struct cxl_mailbox' and move all mailbox related bits to
> it. This allows isolation of all CXL mailbox data in order to export
> some of the calls to external caller (fwctl) and avoid exporting of
> CXL driver specific bits such has device states.
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Some minor comments inline.
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 2626f3fff201..783cb5ed823f 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -1411,6 +1424,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
> struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
> {
> struct cxl_memdev_state *mds;
> + struct cxl_mailbox *cxl_mbox;
>
> mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
> if (!mds) {
> @@ -1418,7 +1432,9 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
> return ERR_PTR(-ENOMEM);
> }
>
> - mutex_init(&mds->mbox_mutex);
> + cxl_mbox = &mds->cxlds.cxl_mbox;
> + mutex_init(&cxl_mbox->mbox_mutex);
Probably factor out a cxl_mbox_init()
Maybe you do that in later patches...
> /*
> * Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
> @@ -315,10 +320,10 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
>
> timeout = mbox_cmd->poll_interval_ms;
> for (i = 0; i < mbox_cmd->poll_count; i++) {
> - if (rcuwait_wait_event_timeout(&mds->mbox_wait,
> - cxl_mbox_background_complete(cxlds),
> - TASK_UNINTERRUPTIBLE,
> - msecs_to_jiffies(timeout)) > 0)
> + if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait,
> + cxl_mbox_background_complete(cxlds),
> + TASK_UNINTERRUPTIBLE,
> + msecs_to_jiffies(timeout)) > 0)
Keep original indent - will reduce diff.
> diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
> new file mode 100644
> index 000000000000..654df6175828
> --- /dev/null
> +++ b/include/linux/cxl/mailbox.h
> @@ -0,0 +1,63 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/* Copyright(c) 2024 Intel Corporation. */
> +#ifndef __CXL_MBOX_H__
> +#define __CXL_MBOX_H__
> +
> +#include <linux/auxiliary_bus.h>
> +/**
> + * struct cxl_mailbox - context for CXL mailbox operations
> + * @host: device that hosts the mailbox
> + * @adev: auxiliary device for fw-ctl
> + * @payload_size: Size of space for payload
> + * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register)
> + * @mbox_mutex: mutex protects device mailbox and firmware
> + * @mbox_wait: rcuwait for mailbox
> + * @mbox_send: @dev specific transport for transmitting mailbox commands
> + */
> +struct cxl_mailbox {
> + struct device *host;
> + struct auxiliary_device adev; /* For fw-ctl */
I'm not sure the embedding makes sense here.
Also not used yet, so don't do it in this patch.
> + size_t payload_size;
> + struct mutex mbox_mutex; /* lock to protect mailbox context */
Maybe drop the mbox_prefix on this.
> + struct rcuwait mbox_wait;
and this
> + int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
and this
as now they are inside struct cxl_mailbox the mbox bit is pretty obvious
and we all love shorter line lengths.
> +};
> +
> +#endif
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-26 17:29 ` Jonathan Cameron
2024-07-18 21:32 ` [RFC PATCH 03/13] cxl: Refactor user ioctl command path from mds to mailbox Dave Jiang
` (13 subsequent siblings)
15 siblings, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
The code indicates that the min of n_commands and total commands
is returned. The comment incorrectly says it's the max(). Correct
comment to min().
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/mbox.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 783cb5ed823f..433819ac36b7 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -547,7 +547,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
/*
- * otherwise, return max(n_commands, total commands) cxl_command_info
+ * otherwise, return min(n_commands, total commands) cxl_command_info
* structures.
*/
cxl_for_each_cmd(cmd) {
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data
2024-07-18 21:32 ` [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data Dave Jiang
@ 2024-07-26 17:29 ` Jonathan Cameron
2024-09-24 23:41 ` Dave Jiang
0 siblings, 1 reply; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-26 17:29 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:20 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> The code indicates that the min of n_commands and total commands
> is returned. The comment incorrectly says it's the max(). Correct
> comment to min().
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Send this one separately.
> ---
> drivers/cxl/core/mbox.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 783cb5ed823f..433819ac36b7 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -547,7 +547,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
> return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
>
> /*
> - * otherwise, return max(n_commands, total commands) cxl_command_info
> + * otherwise, return min(n_commands, total commands) cxl_command_info
> * structures.
> */
> cxl_for_each_cmd(cmd) {
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data
2024-07-26 17:29 ` Jonathan Cameron
@ 2024-09-24 23:41 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-09-24 23:41 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On 7/26/24 10:29 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:20 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> The code indicates that the min of n_commands and total commands
>> is returned. The comment incorrectly says it's the max(). Correct
>> comment to min().
>>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> Send this one separately.
Yup. This one is now queued for 6.12.
>> ---
>> drivers/cxl/core/mbox.c | 2 +-
>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
>> index 783cb5ed823f..433819ac36b7 100644
>> --- a/drivers/cxl/core/mbox.c
>> +++ b/drivers/cxl/core/mbox.c
>> @@ -547,7 +547,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
>> return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
>>
>> /*
>> - * otherwise, return max(n_commands, total commands) cxl_command_info
>> + * otherwise, return min(n_commands, total commands) cxl_command_info
>> * structures.
>> */
>> cxl_for_each_cmd(cmd) {
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 03/13] cxl: Refactor user ioctl command path from mds to mailbox
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 01/13] cxl: Move mailbox related bits to the same context Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 02/13] cxl: Fix comment regarding cxl_query_cmd() return data Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage Dave Jiang
` (12 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
With 'struct cxl_mailbox' context introduced, the helper functions
cxl_query_cmd() and cxl_send_cmd() can take a cxl_mailbox directly
rather than a cxl_memdev parameter. Refactor to use cxl_mailbox
directly.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/core.h | 6 +-
drivers/cxl/core/mbox.c | 118 ++++++++++++++++++------------------
drivers/cxl/core/memdev.c | 39 ++++++++----
drivers/cxl/cxlmem.h | 6 +-
drivers/cxl/pci.c | 6 +-
drivers/cxl/pmem.c | 6 +-
drivers/cxl/security.c | 18 ++++--
include/linux/cxl/mailbox.h | 5 ++
8 files changed, 115 insertions(+), 89 deletions(-)
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 625394486459..7eb33ea07848 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -4,6 +4,8 @@
#ifndef __CXL_CORE_H__
#define __CXL_CORE_H__
+#include <linux/cxl/mailbox.h>
+
extern const struct device_type cxl_nvdimm_bridge_type;
extern const struct device_type cxl_nvdimm_type;
extern const struct device_type cxl_pmu_type;
@@ -65,9 +67,9 @@ static inline void cxl_region_exit(void)
struct cxl_send_command;
struct cxl_mem_query_commands;
-int cxl_query_cmd(struct cxl_memdev *cxlmd,
+int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mem_query_commands __user *q);
-int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s);
+int cxl_send_cmd(struct cxl_mailbox *cxl_mailbox, struct cxl_send_command __user *s);
void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
resource_size_t length);
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 433819ac36b7..b9c64f1837a8 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -225,7 +225,7 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
/**
* cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
- * @mds: The driver data for the operation
+ * @cxl_mbox: CXL mailbox context for the operation
* @mbox_cmd: initialized command to execute
*
* Context: Any context.
@@ -241,10 +241,9 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
* error. While this distinction can be useful for commands from userspace, the
* kernel will only be able to use results when both are successful.
*/
-int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
+int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *mbox_cmd)
{
- struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
size_t out_size, min_out;
int rc;
@@ -350,40 +349,40 @@ static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
return true;
}
-static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
- struct cxl_memdev_state *mds, u16 opcode,
+static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd,
+ struct cxl_mailbox *cxl_mbox, u16 opcode,
size_t in_size, size_t out_size, u64 in_payload)
{
- struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
- *mbox = (struct cxl_mbox_cmd) {
+ *mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = opcode,
.size_in = in_size,
};
if (in_size) {
- mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
- in_size);
- if (IS_ERR(mbox->payload_in))
- return PTR_ERR(mbox->payload_in);
+ mbox_cmd->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
+ in_size);
+ if (IS_ERR(mbox_cmd->payload_in))
+ return PTR_ERR(mbox_cmd->payload_in);
- if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
- dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",
+ if (!cxl_payload_from_user_allowed(opcode,
+ mbox_cmd->payload_in)) {
+ dev_dbg(cxl_mbox->host, "%s: input payload not allowed\n",
cxl_mem_opcode_to_name(opcode));
- kvfree(mbox->payload_in);
+ kvfree(mbox_cmd->payload_in);
return -EBUSY;
}
}
/* Prepare to handle a full payload for variable sized output */
if (out_size == CXL_VARIABLE_PAYLOAD)
- mbox->size_out = cxl_mbox->payload_size;
+ mbox_cmd->size_out = cxl_mbox->payload_size;
else
- mbox->size_out = out_size;
+ mbox_cmd->size_out = out_size;
- if (mbox->size_out) {
- mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);
- if (!mbox->payload_out) {
- kvfree(mbox->payload_in);
+ if (mbox_cmd->size_out) {
+ mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL);
+ if (!mbox_cmd->payload_out) {
+ kvfree(mbox_cmd->payload_in);
return -ENOMEM;
}
}
@@ -398,10 +397,8 @@ static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
- struct cxl_memdev_state *mds)
+ struct cxl_mailbox *cxl_mbox)
{
- struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
-
if (send_cmd->raw.rsvd)
return -EINVAL;
@@ -416,7 +413,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
return -EPERM;
- dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");
+ dev_WARN_ONCE(cxl_mbox->host, true, "raw command path used\n");
*mem_cmd = (struct cxl_mem_command) {
.info = {
@@ -432,7 +429,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
- struct cxl_memdev_state *mds)
+ struct cxl_mailbox *cxl_mbox)
{
struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
const struct cxl_command_info *info = &c->info;
@@ -447,11 +444,11 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
return -EINVAL;
/* Check that the command is enabled for hardware */
- if (!test_bit(info->id, mds->enabled_cmds))
+ if (!test_bit(info->id, cxl_mbox->enabled_cmds))
return -ENOTTY;
/* Check that the command is not claimed for exclusive kernel use */
- if (test_bit(info->id, mds->exclusive_cmds))
+ if (test_bit(info->id, cxl_mbox->exclusive_cmds))
return -EBUSY;
/* Check the input buffer is the expected size */
@@ -495,10 +492,9 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
* safe to send to the hardware.
*/
static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
- struct cxl_memdev_state *mds,
+ struct cxl_mailbox *cxl_mbox,
const struct cxl_send_command *send_cmd)
{
- struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mem_command mem_cmd;
int rc;
@@ -515,24 +511,23 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
/* Sanitize and construct a cxl_mem_command */
if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
- rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);
+ rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxl_mbox);
else
- rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);
+ rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxl_mbox);
if (rc)
return rc;
/* Sanitize and construct a cxl_mbox_cmd */
- return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,
+ return cxl_mbox_cmd_ctor(mbox_cmd, cxl_mbox, mem_cmd.opcode,
mem_cmd.info.size_in, mem_cmd.info.size_out,
send_cmd->in.payload);
}
-int cxl_query_cmd(struct cxl_memdev *cxlmd,
+int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mem_query_commands __user *q)
{
- struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
- struct device *dev = &cxlmd->dev;
+ struct device *dev = cxl_mbox->host;
struct cxl_mem_command *cmd;
u32 n_commands;
int j = 0;
@@ -553,9 +548,9 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
cxl_for_each_cmd(cmd) {
struct cxl_command_info info = cmd->info;
- if (test_bit(info.id, mds->enabled_cmds))
+ if (test_bit(info.id, cxl_mbox->enabled_cmds))
info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
- if (test_bit(info.id, mds->exclusive_cmds))
+ if (test_bit(info.id, cxl_mbox->exclusive_cmds))
info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
@@ -570,7 +565,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
/**
* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
- * @mds: The driver data for the operation
+ * @mailbox: The mailbox context for the operation.
* @mbox_cmd: The validated mailbox command.
* @out_payload: Pointer to userspace's output payload.
* @size_out: (Input) Max payload size to copy out.
@@ -591,13 +586,12 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
*
* See cxl_send_cmd().
*/
-static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
+static int handle_mailbox_cmd_from_user(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *mbox_cmd,
u64 out_payload, s32 *size_out,
u32 *retval)
{
- struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
- struct device *dev = mds->cxlds.dev;
+ struct device *dev = cxl_mbox->host;
int rc;
dev_dbg(dev,
@@ -634,10 +628,9 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
return rc;
}
-int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
+int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command __user *s)
{
- struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
- struct device *dev = &cxlmd->dev;
+ struct device *dev = cxl_mbox->host;
struct cxl_send_command send;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -647,11 +640,11 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
if (copy_from_user(&send, s, sizeof(send)))
return -EFAULT;
- rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);
+ rc = cxl_validate_cmd_from_user(&mbox_cmd, cxl_mbox, &send);
if (rc)
return rc;
- rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,
+ rc = handle_mailbox_cmd_from_user(cxl_mbox, &mbox_cmd, send.out.payload,
&send.out.size, &send.retval);
if (rc)
return rc;
@@ -689,7 +682,7 @@ static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
.payload_out = out,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
/*
* The output payload length that indicates the number
@@ -725,6 +718,7 @@ static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
*/
static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_cel_entry *cel_entry;
const int cel_entries = size / sizeof(*cel_entry);
struct device *dev = mds->cxlds.dev;
@@ -738,7 +732,7 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
int enabled = 0;
if (cmd) {
- set_bit(cmd->info.id, mds->enabled_cmds);
+ set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
enabled++;
}
@@ -775,7 +769,7 @@ static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *
/* At least the record number field must be valid */
.min_out = 2,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
kvfree(ret);
return ERR_PTR(rc);
@@ -808,6 +802,7 @@ static const uuid_t log_uuid[] = {
*/
int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_supported_logs *gsl;
struct device *dev = mds->cxlds.dev;
struct cxl_mem_command *cmd;
@@ -846,7 +841,7 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
/* In case CEL was bogus, enable some default commands. */
cxl_for_each_cmd(cmd)
if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
- set_bit(cmd->info.id, mds->enabled_cmds);
+ set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
/* Found the required CEL */
rc = 0;
@@ -964,7 +959,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
if (i == max_handles) {
payload->nr_recs = i;
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto free_pl;
i = 0;
@@ -975,7 +970,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds,
if (i) {
payload->nr_recs = i;
mbox_cmd.size_in = struct_size(payload, handles, i);
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto free_pl;
}
@@ -1009,7 +1004,7 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
.min_out = struct_size(payload, records, 0),
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc) {
dev_err_ratelimited(dev,
"Event log '%d': Failed to query event records : %d",
@@ -1080,6 +1075,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
*/
static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_partition_info pi;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -1089,7 +1085,7 @@ static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
.size_out = sizeof(pi),
.payload_out = &pi,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
return rc;
@@ -1116,6 +1112,7 @@ static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
*/
int cxl_dev_state_identify(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
struct cxl_mbox_identify id;
struct cxl_mbox_cmd mbox_cmd;
@@ -1130,7 +1127,7 @@ int cxl_dev_state_identify(struct cxl_memdev_state *mds)
.size_out = sizeof(id),
.payload_out = &id,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
return rc;
@@ -1170,11 +1167,12 @@ static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
};
struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
return -EINVAL;
- rc = cxl_internal_send_cmd(mds, &sec_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &sec_cmd);
if (rc < 0) {
dev_err(cxlds->dev, "Failed to get security state : %d", rc);
return rc;
@@ -1193,7 +1191,7 @@ static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
sec_out & CXL_PMEM_SEC_STATE_LOCKED)
return -EINVAL;
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
dev_err(cxlds->dev, "Failed to sanitize device : %d", rc);
return rc;
@@ -1310,6 +1308,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
int cxl_set_timestamp(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd;
struct cxl_mbox_set_timestamp_in pi;
int rc;
@@ -1321,7 +1320,7 @@ int cxl_set_timestamp(struct cxl_memdev_state *mds)
.payload_in = &pi,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
/*
* Command is optional. Devices may have another way of providing
* a timestamp, or may return all 0s in timestamp fields.
@@ -1362,7 +1361,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
.min_out = struct_size(po, record, 0),
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
break;
@@ -1438,6 +1437,7 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
mutex_init(&mds->event.log_lock);
mds->cxlds.dev = dev;
mds->cxlds.reg_map.host = dev;
+ mds->cxlds.cxl_mbox.host = dev;
mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;
mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
mds->ram_perf.qos_class = CXL_QOS_CLASS_INVALID;
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 05bb84cb1274..9f0fe698414d 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -279,6 +279,7 @@ static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_inject_poison inject;
struct cxl_poison_record record;
struct cxl_mbox_cmd mbox_cmd;
@@ -308,7 +309,7 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
.size_in = sizeof(inject),
.payload_in = &inject,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto out;
@@ -334,6 +335,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_clear_poison clear;
struct cxl_poison_record record;
struct cxl_mbox_cmd mbox_cmd;
@@ -372,7 +374,7 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
.payload_in = &clear,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto out;
@@ -564,9 +566,11 @@ EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
unsigned long *cmds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
down_write(&cxl_memdev_rwsem);
- bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
- CXL_MEM_COMMAND_ID_MAX);
+ bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
+ cmds, CXL_MEM_COMMAND_ID_MAX);
up_write(&cxl_memdev_rwsem);
}
EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
@@ -579,9 +583,11 @@ EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
unsigned long *cmds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
down_write(&cxl_memdev_rwsem);
- bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
- CXL_MEM_COMMAND_ID_MAX);
+ bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
+ cmds, CXL_MEM_COMMAND_ID_MAX);
up_write(&cxl_memdev_rwsem);
}
EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
@@ -656,11 +662,14 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
unsigned long arg)
{
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
+
switch (cmd) {
case CXL_MEM_QUERY_COMMANDS:
- return cxl_query_cmd(cxlmd, (void __user *)arg);
+ return cxl_query_cmd(cxl_mbox, (void __user *)arg);
case CXL_MEM_SEND_COMMAND:
- return cxl_send_cmd(cxlmd, (void __user *)arg);
+ return cxl_send_cmd(cxl_mbox, (void __user *)arg);
default:
return -ENOTTY;
}
@@ -715,6 +724,7 @@ static int cxl_memdev_release_file(struct inode *inode, struct file *file)
*/
static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_fw_info info;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -725,7 +735,7 @@ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
.payload_out = &info,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
return rc;
@@ -749,6 +759,7 @@ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
*/
static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_activate_fw activate;
struct cxl_mbox_cmd mbox_cmd;
@@ -765,7 +776,7 @@ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
activate.action = CXL_FW_ACTIVATE_OFFLINE;
activate.slot = slot;
- return cxl_internal_send_cmd(mds, &mbox_cmd);
+ return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}
/**
@@ -780,6 +791,7 @@ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
*/
static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_transfer_fw *transfer;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -799,7 +811,7 @@ static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
kfree(transfer);
return rc;
}
@@ -924,7 +936,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
.poll_count = 30,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
rc = FW_UPLOAD_ERR_RW_ERROR;
goto out_free;
@@ -991,10 +1003,11 @@ static void cxl_remove_fw_upload(void *fwl)
int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
{
struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
struct device *dev = &cxlds->cxlmd->dev;
struct fw_upload *fwl;
- if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
+ if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
return 0;
fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 8f683fe08410..efdf833f2c51 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -417,8 +417,6 @@ struct cxl_dev_state {
* @lsa_size: Size of Label Storage Area
* (CXL 2.0 8.2.9.5.1.1 Identify Memory Device)
* @firmware_version: Firmware version for the memory device.
- * @enabled_cmds: Hardware commands found enabled in CEL.
- * @exclusive_cmds: Commands that are kernel-internal only
* @total_bytes: sum of all possible capacities
* @volatile_only_bytes: hard volatile capacity
* @persistent_only_bytes: hard persistent capacity
@@ -441,8 +439,6 @@ struct cxl_memdev_state {
struct cxl_dev_state cxlds;
size_t lsa_size;
char firmware_version[0x10];
- DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
- DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
u64 total_bytes;
u64 volatile_only_bytes;
u64 persistent_only_bytes;
@@ -769,7 +765,7 @@ enum {
CXL_PMEM_SEC_PASS_USER,
};
-int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
+int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd);
int cxl_dev_state_identify(struct cxl_memdev_state *mds);
int cxl_await_media_ready(struct cxl_dev_state *cxlds);
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 45b184b6ef97..7e26da706921 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -661,6 +661,7 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd = {
.opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY,
.payload_out = policy,
@@ -668,7 +669,7 @@ static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
};
int rc;
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
dev_err(mds->cxlds.dev,
"Failed to get event interrupt policy : %d", rc);
@@ -679,6 +680,7 @@ static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -695,7 +697,7 @@ static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
.size_in = sizeof(*policy),
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d",
rc);
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 9f62dc03c883..6a83f33dde5b 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -120,6 +120,7 @@ static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
struct nd_cmd_get_config_data_hdr *cmd,
unsigned int buf_len)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_get_lsa get_lsa;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -141,7 +142,7 @@ static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
.payload_out = cmd->out_buf,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
cmd->status = 0;
return rc;
@@ -151,6 +152,7 @@ static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
struct nd_cmd_set_config_hdr *cmd,
unsigned int buf_len)
{
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_set_lsa *set_lsa;
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -177,7 +179,7 @@ static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
.size_in = struct_size(set_lsa, data, cmd->in_length),
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
/*
* Set "firmware" status (4-packed bytes at the end of the input
diff --git a/drivers/cxl/security.c b/drivers/cxl/security.c
index 21856a3f408e..95a92b87c99a 100644
--- a/drivers/cxl/security.c
+++ b/drivers/cxl/security.c
@@ -15,6 +15,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
unsigned long security_flags = 0;
struct cxl_get_security_output {
__le32 flags;
@@ -29,7 +30,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm,
.payload_out = &out,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
return 0;
@@ -71,6 +72,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd;
struct cxl_set_pass set_pass;
@@ -87,7 +89,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm,
.payload_in = &set_pass,
};
- return cxl_internal_send_cmd(mds, &mbox_cmd);
+ return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}
static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -97,6 +99,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_disable_pass dis_pass;
struct cxl_mbox_cmd mbox_cmd;
@@ -112,7 +115,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
.payload_in = &dis_pass,
};
- return cxl_internal_send_cmd(mds, &mbox_cmd);
+ return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}
static int cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -132,11 +135,12 @@ static int cxl_pmem_security_freeze(struct nvdimm *nvdimm)
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd = {
.opcode = CXL_MBOX_OP_FREEZE_SECURITY,
};
- return cxl_internal_send_cmd(mds, &mbox_cmd);
+ return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}
static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
@@ -145,6 +149,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
u8 pass[NVDIMM_PASSPHRASE_LEN];
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -156,7 +161,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
.payload_in = pass,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
return rc;
@@ -170,6 +175,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd;
struct cxl_pass_erase erase;
int rc;
@@ -185,7 +191,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm,
.payload_in = &erase,
};
- rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
return rc;
diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
index 654df6175828..2380b22d7a12 100644
--- a/include/linux/cxl/mailbox.h
+++ b/include/linux/cxl/mailbox.h
@@ -3,6 +3,7 @@
#ifndef __CXL_MBOX_H__
#define __CXL_MBOX_H__
+#include <uapi/linux/cxl_mem.h>
#include <linux/auxiliary_bus.h>
/**
@@ -45,6 +46,8 @@ struct cxl_mbox_cmd {
* struct cxl_mailbox - context for CXL mailbox operations
* @host: device that hosts the mailbox
* @adev: auxiliary device for fw-ctl
+ * @enabled_cmds: Hardware commands found enabled in CEL.
+ * @exclusive_cmds: Commands that are kernel-internal only
* @payload_size: Size of space for payload
* (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register)
* @mbox_mutex: mutex protects device mailbox and firmware
@@ -54,6 +57,8 @@ struct cxl_mbox_cmd {
struct cxl_mailbox {
struct device *host;
struct auxiliary_device adev; /* For fw-ctl */
+ DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
+ DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
size_t payload_size;
struct mutex mbox_mutex; /* lock to protect mailbox context */
struct rcuwait mbox_wait;
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (2 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 03/13] cxl: Refactor user ioctl command path from mds to mailbox Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-26 17:50 ` Jonathan Cameron
2024-08-21 16:05 ` Shiju Jose
2024-07-18 21:32 ` [RFC PATCH 05/13] cxl/test: Add Get Supported Features mailbox command support Dave Jiang
` (11 subsequent siblings)
15 siblings, 2 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
CXL spec r3.1 8.2.9.6.1 Get Supported Features (Opcode 0500h)
The command retrieve the list of supported device-specific features
(identified by UUID) and general information about each Feature.
The driver will retrieve the feature entries in order to make checks and
provide information for the Get Feature and Set Feature command. One of
the main piece of information retrieved are the effects a Set Feature
command would have for a particular feature.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/mbox.c | 151 +++++++++++++++++++++++++++++++++++
drivers/cxl/cxlmem.h | 29 +++++++
drivers/cxl/pci.c | 4 +
include/linux/cxl/mailbox.h | 3 +
include/uapi/linux/cxl_mem.h | 1 +
5 files changed, 188 insertions(+)
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index b9c64f1837a8..70e3962ed570 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -67,6 +67,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
+ CXL_CMD(GET_SUPPORTED_FEATURES, 0x8, CXL_VARIABLE_PAYLOAD, 0),
};
/*
@@ -790,6 +791,156 @@ static const uuid_t log_uuid[] = {
[VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
};
+static void cxl_free_features(void *features)
+{
+ kvfree(features);
+}
+
+static int cxl_get_supported_features_count(struct cxl_mailbox *cxl_mbox)
+{
+ struct cxl_mbox_get_sup_feats_out mbox_out;
+ struct cxl_mbox_get_sup_feats_in mbox_in;
+ struct cxl_mbox_cmd mbox_cmd;
+ int rc;
+
+ memset(&mbox_in, 0, sizeof(mbox_in));
+ mbox_in.count = sizeof(mbox_out);
+ memset(&mbox_out, 0, sizeof(mbox_out));
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
+ .size_in = sizeof(mbox_in),
+ .payload_in = &mbox_in,
+ .size_out = sizeof(mbox_out),
+ .payload_out = &mbox_out,
+ .min_out = sizeof(mbox_out),
+ };
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
+ if (rc < 0)
+ return rc;
+
+ cxl_mbox->num_features = le16_to_cpu(mbox_out.supported_feats);
+ if (!cxl_mbox->num_features)
+ return -ENOENT;
+
+ return 0;
+}
+
+int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox)
+{
+ int remain_feats, max_size, max_feats, start, rc;
+ int feat_size = sizeof(struct cxl_feat_entry);
+ struct cxl_mbox_get_sup_feats_out *mbox_out;
+ struct cxl_mbox_get_sup_feats_in mbox_in;
+ int hdr_size = sizeof(*mbox_out);
+ struct cxl_mbox_cmd mbox_cmd;
+ struct cxl_mem_command *cmd;
+ void *ptr;
+
+ /* Get supported features is optional, need to check */
+ cmd = cxl_mem_find_command(CXL_MBOX_OP_GET_SUPPORTED_FEATURES);
+ if (!cmd)
+ return -EOPNOTSUPP;
+ if (!test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
+ return -EOPNOTSUPP;
+
+ rc = cxl_get_supported_features_count(cxl_mbox);
+ if (rc)
+ return rc;
+
+ struct cxl_feat_entry *entries __free(kvfree) =
+ kvmalloc(cxl_mbox->num_features * feat_size, GFP_KERNEL);
+
+ if (!entries)
+ return -ENOMEM;
+
+ cxl_mbox->entries = no_free_ptr(entries);
+ rc = devm_add_action_or_reset(cxl_mbox->host, cxl_free_features,
+ cxl_mbox->entries);
+ if (rc)
+ return rc;
+
+ max_size = cxl_mbox->payload_size - hdr_size;
+ /* max feat entries that can fit in mailbox max payload size */
+ max_feats = max_size / feat_size;
+ ptr = &cxl_mbox->entries[0];
+
+ mbox_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
+ if (!mbox_out)
+ return -ENOMEM;
+
+ start = 0;
+ remain_feats = cxl_mbox->num_features;
+ do {
+ int retrieved, alloc_size, copy_feats;
+
+ if (remain_feats > max_feats) {
+ alloc_size = sizeof(*mbox_out) + max_feats * feat_size;
+ remain_feats = remain_feats - max_feats;
+ copy_feats = max_feats;
+ } else {
+ alloc_size = sizeof(*mbox_out) + remain_feats * feat_size;
+ copy_feats = remain_feats;
+ remain_feats = 0;
+ }
+
+ memset(&mbox_in, 0, sizeof(mbox_in));
+ mbox_in.count = alloc_size;
+ mbox_in.start_idx = start;
+ memset(mbox_out, 0, alloc_size);
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
+ .size_in = sizeof(mbox_in),
+ .payload_in = &mbox_in,
+ .size_out = alloc_size,
+ .payload_out = mbox_out,
+ .min_out = hdr_size,
+ };
+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
+ if (rc < 0)
+ return rc;
+
+ if (mbox_cmd.size_out <= hdr_size) {
+ rc = -ENXIO;
+ goto err;
+ }
+
+ /*
+ * Make sure retrieved out buffer is multiple of feature
+ * entries.
+ */
+ retrieved = mbox_cmd.size_out - hdr_size;
+ if (retrieved % sizeof(struct cxl_feat_entry)) {
+ rc = -ENXIO;
+ goto err;
+ }
+
+ /*
+ * If the reported output entries * defined entry size !=
+ * retrieved output bytes, then the output package is incorrect.
+ */
+ if (mbox_out->num_entries * feat_size != retrieved) {
+ rc = -ENXIO;
+ goto err;
+ }
+
+ memcpy(ptr, mbox_out->ents, retrieved);
+ ptr += retrieved;
+ /*
+ * If the number of output entries is less than expected, add the
+ * remaining entries to the next batch.
+ */
+ remain_feats += copy_feats - mbox_out->num_entries;
+ start += mbox_out->num_entries;
+ } while (remain_feats);
+
+ return 0;
+
+err:
+ cxl_mbox->num_features = 0;
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_get_supported_features, CXL);
+
/**
* cxl_enumerate_cmds() - Enumerate commands for a device.
* @mds: The driver data for the operation
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index efdf833f2c51..4d3690aa2f3b 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -482,6 +482,7 @@ enum cxl_opcode {
CXL_MBOX_OP_GET_LOG_CAPS = 0x0402,
CXL_MBOX_OP_CLEAR_LOG = 0x0403,
CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
+ CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
CXL_MBOX_OP_IDENTIFY = 0x4000,
CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
@@ -765,6 +766,32 @@ enum {
CXL_PMEM_SEC_PASS_USER,
};
+/* Get Supported Features (0x500h) CXL r3.1 8.2.9.6.1 */
+struct cxl_mbox_get_sup_feats_in {
+ __le32 count;
+ __le16 start_idx;
+ __le16 reserved;
+} __packed;
+
+struct cxl_feat_entry {
+ uuid_t uuid;
+ __le16 id;
+ __le16 get_feat_size;
+ __le16 set_feat_size;
+ __le32 flags;
+ u8 get_feat_ver;
+ u8 set_feat_ver;
+ __le16 effects;
+ u8 reserved[18];
+} __packed;
+
+struct cxl_mbox_get_sup_feats_out {
+ __le16 num_entries;
+ __le16 supported_feats;
+ __le32 reserved;
+ struct cxl_feat_entry ents[] __counted_by(le32_to_cpu(supported_feats));
+} __packed;
+
int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd);
int cxl_dev_state_identify(struct cxl_memdev_state *mds);
@@ -824,4 +851,6 @@ struct cxl_hdm {
struct seq_file;
struct dentry *cxl_debugfs_create_dir(const char *dir);
void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds);
+
+int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox);
#endif /* __CXL_MEM_H__ */
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 7e26da706921..6a00238446f9 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -872,6 +872,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
+ rc = cxl_get_supported_features(&cxlds->cxl_mbox);
+ if (rc)
+ dev_dbg(&pdev->dev, "No features enumerated.\n");
+
rc = cxl_set_timestamp(mds);
if (rc)
return rc;
diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
index 2380b22d7a12..570864239b8f 100644
--- a/include/linux/cxl/mailbox.h
+++ b/include/linux/cxl/mailbox.h
@@ -53,6 +53,7 @@ struct cxl_mbox_cmd {
* @mbox_mutex: mutex protects device mailbox and firmware
* @mbox_wait: rcuwait for mailbox
* @mbox_send: @dev specific transport for transmitting mailbox commands
+ * @features: number of supported features
*/
struct cxl_mailbox {
struct device *host;
@@ -63,6 +64,8 @@ struct cxl_mailbox {
struct mutex mbox_mutex; /* lock to protect mailbox context */
struct rcuwait mbox_wait;
int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
+ int num_features;
+ struct cxl_feat_entry *entries;
};
#endif
diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
index c6c0fe27495d..bd2535962f70 100644
--- a/include/uapi/linux/cxl_mem.h
+++ b/include/uapi/linux/cxl_mem.h
@@ -50,6 +50,7 @@
___C(GET_LOG_CAPS, "Get Log Capabilities"), \
___C(CLEAR_LOG, "Clear Log"), \
___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
+ ___C(GET_SUPPORTED_FEATURES, "Get Supported Features"), \
___C(MAX, "invalid / last command")
#define ___C(a, b) CXL_MEM_COMMAND_ID_##a
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage
2024-07-18 21:32 ` [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage Dave Jiang
@ 2024-07-26 17:50 ` Jonathan Cameron
2024-09-27 16:22 ` Dave Jiang
2024-08-21 16:05 ` Shiju Jose
1 sibling, 1 reply; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-26 17:50 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:22 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> CXL spec r3.1 8.2.9.6.1 Get Supported Features (Opcode 0500h)
> The command retrieve the list of supported device-specific features
> (identified by UUID) and general information about each Feature.
>
> The driver will retrieve the feature entries in order to make checks and
> provide information for the Get Feature and Set Feature command. One of
> the main piece of information retrieved are the effects a Set Feature
> command would have for a particular feature.
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Already a fairly well reviewed implementation of this in
https://lore.kernel.org/linux-cxl/20240726160556.2079-5-shiju.jose@huawei.com/
Though it wasn't focused on providing userspace access to it and doesn't handle
large feature lists which would be eventually needed.
I'd have preferred to see review on that rather than another version
of the same functionality.
Note the QEMU testing used for that set will be useable here too
and is now upstream.
A few comments inline.
Jonathan
> +int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox)
> +{
> + int remain_feats, max_size, max_feats, start, rc;
> + int feat_size = sizeof(struct cxl_feat_entry);
> + struct cxl_mbox_get_sup_feats_out *mbox_out;
> + struct cxl_mbox_get_sup_feats_in mbox_in;
> + int hdr_size = sizeof(*mbox_out);
> + struct cxl_mbox_cmd mbox_cmd;
> + struct cxl_mem_command *cmd;
> + void *ptr;
> +
> + /* Get supported features is optional, need to check */
> + cmd = cxl_mem_find_command(CXL_MBOX_OP_GET_SUPPORTED_FEATURES);
> + if (!cmd)
> + return -EOPNOTSUPP;
> + if (!test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
> + return -EOPNOTSUPP;
> +
> + rc = cxl_get_supported_features_count(cxl_mbox);
> + if (rc)
> + return rc;
> +
> + struct cxl_feat_entry *entries __free(kvfree) =
> + kvmalloc(cxl_mbox->num_features * feat_size, GFP_KERNEL);
> +
> + if (!entries)
> + return -ENOMEM;
> +
> + cxl_mbox->entries = no_free_ptr(entries);
Hmm. Not srue I'd bother with __free(kvfree) for one line that doesn't
do the free.
> + rc = devm_add_action_or_reset(cxl_mbox->host, cxl_free_features,
> + cxl_mbox->entries);
> + if (rc)
> + return rc;
> +
> + max_size = cxl_mbox->payload_size - hdr_size;
> + /* max feat entries that can fit in mailbox max payload size */
> + max_feats = max_size / feat_size;
> + ptr = &cxl_mbox->entries[0];
> +
> + mbox_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
Freed?
> + if (!mbox_out)
> + return -ENOMEM;
> +
> + start = 0;
> + remain_feats = cxl_mbox->num_features;
> + do {
> + int retrieved, alloc_size, copy_feats;
> +
> + if (remain_feats > max_feats) {
> + alloc_size = sizeof(*mbox_out) + max_feats * feat_size;
> + remain_feats = remain_feats - max_feats;
> + copy_feats = max_feats;
> + } else {
> + alloc_size = sizeof(*mbox_out) + remain_feats * feat_size;
> + copy_feats = remain_feats;
> + remain_feats = 0;
> + }
> +
> + memset(&mbox_in, 0, sizeof(mbox_in));
> + mbox_in.count = alloc_size;
> + mbox_in.start_idx = start;
> + memset(mbox_out, 0, alloc_size);
> + mbox_cmd = (struct cxl_mbox_cmd) {
> + .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
> + .size_in = sizeof(mbox_in),
> + .payload_in = &mbox_in,
> + .size_out = alloc_size,
> + .payload_out = mbox_out,
> + .min_out = hdr_size,
> + };
> + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
> + if (rc < 0)
> + return rc;
> +
> + if (mbox_cmd.size_out <= hdr_size) {
> + rc = -ENXIO;
> + goto err;
> + }
> +
> + /*
> + * Make sure retrieved out buffer is multiple of feature
> + * entries.
> + */
> + retrieved = mbox_cmd.size_out - hdr_size;
> + if (retrieved % sizeof(struct cxl_feat_entry)) {
> + rc = -ENXIO;
> + goto err;
> + }
> +
> + /*
> + * If the reported output entries * defined entry size !=
> + * retrieved output bytes, then the output package is incorrect.
> + */
> + if (mbox_out->num_entries * feat_size != retrieved) {
> + rc = -ENXIO;
> + goto err;
> + }
> +
> + memcpy(ptr, mbox_out->ents, retrieved);
> + ptr += retrieved;
> + /*
> + * If the number of output entries is less than expected, add the
> + * remaining entries to the next batch.
> + */
> + remain_feats += copy_feats - mbox_out->num_entries;
> + start += mbox_out->num_entries;
> + } while (remain_feats);
> +
> + return 0;
> +
> +err:
> + cxl_mbox->num_features = 0;
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_get_supported_features, CXL);
> +
> /**
> * cxl_enumerate_cmds() - Enumerate commands for a device.
> * @mds: The driver data for the operation
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index efdf833f2c51..4d3690aa2f3b 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> +struct cxl_mbox_get_sup_feats_out {
> + __le16 num_entries;
> + __le16 supported_feats;
> + __le32 reserved;
> + struct cxl_feat_entry ents[] __counted_by(le32_to_cpu(supported_feats));
I didn't know __counted_by worked with function calls. Interesting.
> +} __packed;
> +
> #endif /* __CXL_MEM_H__ */
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 7e26da706921..6a00238446f9 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -872,6 +872,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> if (rc)
> return rc;
>
> + rc = cxl_get_supported_features(&cxlds->cxl_mbox);
I would separate no features from an error in trying to enumerate them.
> + if (rc)
> + dev_dbg(&pdev->dev, "No features enumerated.\n");
> +
> rc = cxl_set_timestamp(mds);
> if (rc)
> return rc;
> diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
> index 2380b22d7a12..570864239b8f 100644
> --- a/include/linux/cxl/mailbox.h
> +++ b/include/linux/cxl/mailbox.h
> @@ -53,6 +53,7 @@ struct cxl_mbox_cmd {
> * @mbox_mutex: mutex protects device mailbox and firmware
> * @mbox_wait: rcuwait for mailbox
> * @mbox_send: @dev specific transport for transmitting mailbox commands
> + * @features: number of supported features
missing docs for entries.
> */
> struct cxl_mailbox {
> struct device *host;
> @@ -63,6 +64,8 @@ struct cxl_mailbox {
> struct mutex mbox_mutex; /* lock to protect mailbox context */
> struct rcuwait mbox_wait;
> int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
> + int num_features;
> + struct cxl_feat_entry *entries;
> };
>
> #endif
> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> index c6c0fe27495d..bd2535962f70 100644
> --- a/include/uapi/linux/cxl_mem.h
> +++ b/include/uapi/linux/cxl_mem.h
> @@ -50,6 +50,7 @@
> ___C(GET_LOG_CAPS, "Get Log Capabilities"), \
> ___C(CLEAR_LOG, "Clear Log"), \
> ___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
> + ___C(GET_SUPPORTED_FEATURES, "Get Supported Features"), \
> ___C(MAX, "invalid / last command")
>
> #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage
2024-07-26 17:50 ` Jonathan Cameron
@ 2024-09-27 16:22 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-09-27 16:22 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On 7/26/24 10:50 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:22 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> CXL spec r3.1 8.2.9.6.1 Get Supported Features (Opcode 0500h)
>> The command retrieve the list of supported device-specific features
>> (identified by UUID) and general information about each Feature.
>>
>> The driver will retrieve the feature entries in order to make checks and
>> provide information for the Get Feature and Set Feature command. One of
>> the main piece of information retrieved are the effects a Set Feature
>> command would have for a particular feature.
>>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>
> Already a fairly well reviewed implementation of this in
>
> https://lore.kernel.org/linux-cxl/20240726160556.2079-5-shiju.jose@huawei.com/
>
> Though it wasn't focused on providing userspace access to it and doesn't handle
> large feature lists which would be eventually needed.
>
> I'd have preferred to see review on that rather than another version
> of the same functionality.
>
> Note the QEMU testing used for that set will be useable here too
> and is now upstream.
>
> A few comments inline.
>
> Jonathan
>
>
>> +int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox)
>> +{
>> + int remain_feats, max_size, max_feats, start, rc;
>> + int feat_size = sizeof(struct cxl_feat_entry);
>> + struct cxl_mbox_get_sup_feats_out *mbox_out;
>> + struct cxl_mbox_get_sup_feats_in mbox_in;
>> + int hdr_size = sizeof(*mbox_out);
>> + struct cxl_mbox_cmd mbox_cmd;
>> + struct cxl_mem_command *cmd;
>> + void *ptr;
>> +
>> + /* Get supported features is optional, need to check */
>> + cmd = cxl_mem_find_command(CXL_MBOX_OP_GET_SUPPORTED_FEATURES);
>> + if (!cmd)
>> + return -EOPNOTSUPP;
>> + if (!test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
>> + return -EOPNOTSUPP;
>> +
>> + rc = cxl_get_supported_features_count(cxl_mbox);
>> + if (rc)
>> + return rc;
>> +
>> + struct cxl_feat_entry *entries __free(kvfree) =
>> + kvmalloc(cxl_mbox->num_features * feat_size, GFP_KERNEL);
>> +
>> + if (!entries)
>> + return -ENOMEM;
>> +
>> + cxl_mbox->entries = no_free_ptr(entries);
>
> Hmm. Not srue I'd bother with __free(kvfree) for one line that doesn't
> do the free.
>
>> + rc = devm_add_action_or_reset(cxl_mbox->host, cxl_free_features,
>> + cxl_mbox->entries);
>> + if (rc)
>> + return rc;
>> +
>> + max_size = cxl_mbox->payload_size - hdr_size;
>> + /* max feat entries that can fit in mailbox max payload size */
>> + max_feats = max_size / feat_size;
>> + ptr = &cxl_mbox->entries[0];
>> +
>> + mbox_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
> Freed?
>
>> + if (!mbox_out)
>> + return -ENOMEM;
>> +
>> + start = 0;
>> + remain_feats = cxl_mbox->num_features;
>> + do {
>> + int retrieved, alloc_size, copy_feats;
>> +
>> + if (remain_feats > max_feats) {
>> + alloc_size = sizeof(*mbox_out) + max_feats * feat_size;
>> + remain_feats = remain_feats - max_feats;
>> + copy_feats = max_feats;
>> + } else {
>> + alloc_size = sizeof(*mbox_out) + remain_feats * feat_size;
>> + copy_feats = remain_feats;
>> + remain_feats = 0;
>> + }
>> +
>> + memset(&mbox_in, 0, sizeof(mbox_in));
>> + mbox_in.count = alloc_size;
>> + mbox_in.start_idx = start;
>> + memset(mbox_out, 0, alloc_size);
>> + mbox_cmd = (struct cxl_mbox_cmd) {
>> + .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
>> + .size_in = sizeof(mbox_in),
>> + .payload_in = &mbox_in,
>> + .size_out = alloc_size,
>> + .payload_out = mbox_out,
>> + .min_out = hdr_size,
>> + };
>> + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
>> + if (rc < 0)
>> + return rc;
>> +
>> + if (mbox_cmd.size_out <= hdr_size) {
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + /*
>> + * Make sure retrieved out buffer is multiple of feature
>> + * entries.
>> + */
>> + retrieved = mbox_cmd.size_out - hdr_size;
>> + if (retrieved % sizeof(struct cxl_feat_entry)) {
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + /*
>> + * If the reported output entries * defined entry size !=
>> + * retrieved output bytes, then the output package is incorrect.
>> + */
>> + if (mbox_out->num_entries * feat_size != retrieved) {
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + memcpy(ptr, mbox_out->ents, retrieved);
>> + ptr += retrieved;
>> + /*
>> + * If the number of output entries is less than expected, add the
>> + * remaining entries to the next batch.
>> + */
>> + remain_feats += copy_feats - mbox_out->num_entries;
>> + start += mbox_out->num_entries;
>> + } while (remain_feats);
>> +
>> + return 0;
>> +
>> +err:
>> + cxl_mbox->num_features = 0;
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_get_supported_features, CXL);
>> +
>> /**
>> * cxl_enumerate_cmds() - Enumerate commands for a device.
>> * @mds: The driver data for the operation
>> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
>> index efdf833f2c51..4d3690aa2f3b 100644
>> --- a/drivers/cxl/cxlmem.h
>> +++ b/drivers/cxl/cxlmem.h
>
>> +struct cxl_mbox_get_sup_feats_out {
>> + __le16 num_entries;
>> + __le16 supported_feats;
>> + __le32 reserved;
>> + struct cxl_feat_entry ents[] __counted_by(le32_to_cpu(supported_feats));
>
> I didn't know __counted_by worked with function calls. Interesting.
Actually I need to use __counted_by_le() instead as someone pointed out via kbot reporting.
DJ
>
>> +} __packed;
>> +
>
>> #endif /* __CXL_MEM_H__ */
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 7e26da706921..6a00238446f9 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -872,6 +872,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> if (rc)
>> return rc;
>>
>> + rc = cxl_get_supported_features(&cxlds->cxl_mbox);
>
> I would separate no features from an error in trying to enumerate them.
>
>> + if (rc)
>> + dev_dbg(&pdev->dev, "No features enumerated.\n");
>> +
>> rc = cxl_set_timestamp(mds);
>> if (rc)
>> return rc;
>> diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
>> index 2380b22d7a12..570864239b8f 100644
>> --- a/include/linux/cxl/mailbox.h
>> +++ b/include/linux/cxl/mailbox.h
>> @@ -53,6 +53,7 @@ struct cxl_mbox_cmd {
>> * @mbox_mutex: mutex protects device mailbox and firmware
>> * @mbox_wait: rcuwait for mailbox
>> * @mbox_send: @dev specific transport for transmitting mailbox commands
>> + * @features: number of supported features
>
> missing docs for entries.
>
>> */
>> struct cxl_mailbox {
>> struct device *host;
>> @@ -63,6 +64,8 @@ struct cxl_mailbox {
>> struct mutex mbox_mutex; /* lock to protect mailbox context */
>> struct rcuwait mbox_wait;
>> int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
>> + int num_features;
>> + struct cxl_feat_entry *entries;
>> };
>>
>> #endif
>> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
>> index c6c0fe27495d..bd2535962f70 100644
>> --- a/include/uapi/linux/cxl_mem.h
>> +++ b/include/uapi/linux/cxl_mem.h
>> @@ -50,6 +50,7 @@
>> ___C(GET_LOG_CAPS, "Get Log Capabilities"), \
>> ___C(CLEAR_LOG, "Clear Log"), \
>> ___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
>> + ___C(GET_SUPPORTED_FEATURES, "Get Supported Features"), \
>> ___C(MAX, "invalid / last command")
>>
>> #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* RE: [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage
2024-07-18 21:32 ` [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage Dave Jiang
2024-07-26 17:50 ` Jonathan Cameron
@ 2024-08-21 16:05 ` Shiju Jose
2024-08-21 18:10 ` Dave Jiang
1 sibling, 1 reply; 47+ messages in thread
From: Shiju Jose @ 2024-08-21 16:05 UTC (permalink / raw)
To: Dave Jiang, linux-cxl@vger.kernel.org
Cc: dan.j.williams@intel.com, ira.weiny@intel.com,
vishal.l.verma@intel.com, alison.schofield@intel.com,
Jonathan Cameron, dave@stgolabs.net, jgg@nvidia.com
Hi Dave,
Few comments inline.
>-----Original Message-----
>From: Dave Jiang <dave.jiang@intel.com>
>Sent: 18 July 2024 22:32
>To: linux-cxl@vger.kernel.org
>Cc: dan.j.williams@intel.com; ira.weiny@intel.com; vishal.l.verma@intel.com;
>alison.schofield@intel.com; Jonathan Cameron
><jonathan.cameron@huawei.com>; dave@stgolabs.net; jgg@nvidia.com; Shiju
>Jose <shiju.jose@huawei.com>
>Subject: [RFC PATCH 04/13] cxl: Add Get Supported Features command for
>kernel usage
>
>CXL spec r3.1 8.2.9.6.1 Get Supported Features (Opcode 0500h) The command
>retrieve the list of supported device-specific features (identified by UUID) and
>general information about each Feature.
>
>The driver will retrieve the feature entries in order to make checks and provide
>information for the Get Feature and Set Feature command. One of the main
>piece of information retrieved are the effects a Set Feature command would
>have for a particular feature.
>
>Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>---
> drivers/cxl/core/mbox.c | 151 +++++++++++++++++++++++++++++++++++
> drivers/cxl/cxlmem.h | 29 +++++++
> drivers/cxl/pci.c | 4 +
> include/linux/cxl/mailbox.h | 3 +
> include/uapi/linux/cxl_mem.h | 1 +
> 5 files changed, 188 insertions(+)
>
>diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index
>b9c64f1837a8..70e3962ed570 100644
>--- a/drivers/cxl/core/mbox.c
>+++ b/drivers/cxl/core/mbox.c
>@@ -67,6 +67,7 @@ static struct cxl_mem_command
>cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
> CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
> CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
>+ CXL_CMD(GET_SUPPORTED_FEATURES, 0x8, CXL_VARIABLE_PAYLOAD,
>0),
> };
>
> /*
>@@ -790,6 +791,156 @@ static const uuid_t log_uuid[] = {
> [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID, };
>
>+static void cxl_free_features(void *features) {
>+ kvfree(features);
>+}
>+
>+static int cxl_get_supported_features_count(struct cxl_mailbox
>+*cxl_mbox) {
>+ struct cxl_mbox_get_sup_feats_out mbox_out;
>+ struct cxl_mbox_get_sup_feats_in mbox_in;
>+ struct cxl_mbox_cmd mbox_cmd;
>+ int rc;
>+
>+ memset(&mbox_in, 0, sizeof(mbox_in));
>+ mbox_in.count = sizeof(mbox_out);
>+ memset(&mbox_out, 0, sizeof(mbox_out));
>+ mbox_cmd = (struct cxl_mbox_cmd) {
>+ .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
>+ .size_in = sizeof(mbox_in),
>+ .payload_in = &mbox_in,
>+ .size_out = sizeof(mbox_out),
>+ .payload_out = &mbox_out,
>+ .min_out = sizeof(mbox_out),
>+ };
>+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
>+ if (rc < 0)
>+ return rc;
>+
>+ cxl_mbox->num_features = le16_to_cpu(mbox_out.supported_feats);
>+ if (!cxl_mbox->num_features)
>+ return -ENOENT;
>+
>+ return 0;
>+}
>+
>+int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox) {
>+ int remain_feats, max_size, max_feats, start, rc;
>+ int feat_size = sizeof(struct cxl_feat_entry);
>+ struct cxl_mbox_get_sup_feats_out *mbox_out;
>+ struct cxl_mbox_get_sup_feats_in mbox_in;
>+ int hdr_size = sizeof(*mbox_out);
>+ struct cxl_mbox_cmd mbox_cmd;
>+ struct cxl_mem_command *cmd;
>+ void *ptr;
>+
>+ /* Get supported features is optional, need to check */
>+ cmd =
>cxl_mem_find_command(CXL_MBOX_OP_GET_SUPPORTED_FEATURES);
>+ if (!cmd)
>+ return -EOPNOTSUPP;
>+ if (!test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
>+ return -EOPNOTSUPP;
>+
>+ rc = cxl_get_supported_features_count(cxl_mbox);
>+ if (rc)
>+ return rc;
>+
>+ struct cxl_feat_entry *entries __free(kvfree) =
>+ kvmalloc(cxl_mbox->num_features * feat_size, GFP_KERNEL);
>+
>+ if (!entries)
>+ return -ENOMEM;
>+
>+ cxl_mbox->entries = no_free_ptr(entries);
>+ rc = devm_add_action_or_reset(cxl_mbox->host, cxl_free_features,
>+ cxl_mbox->entries);
>+ if (rc)
>+ return rc;
>+
>+ max_size = cxl_mbox->payload_size - hdr_size;
>+ /* max feat entries that can fit in mailbox max payload size */
>+ max_feats = max_size / feat_size;
>+ ptr = &cxl_mbox->entries[0];
>+
>+ mbox_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
>+ if (!mbox_out)
>+ return -ENOMEM;
>+
>+ start = 0;
>+ remain_feats = cxl_mbox->num_features;
>+ do {
>+ int retrieved, alloc_size, copy_feats;
>+
>+ if (remain_feats > max_feats) {
>+ alloc_size = sizeof(*mbox_out) + max_feats * feat_size;
>+ remain_feats = remain_feats - max_feats;
>+ copy_feats = max_feats;
>+ } else {
>+ alloc_size = sizeof(*mbox_out) + remain_feats *
>feat_size;
>+ copy_feats = remain_feats;
>+ remain_feats = 0;
>+ }
>+
>+ memset(&mbox_in, 0, sizeof(mbox_in));
>+ mbox_in.count = alloc_size;
>+ mbox_in.start_idx = start;
>+ memset(mbox_out, 0, alloc_size);
>+ mbox_cmd = (struct cxl_mbox_cmd) {
>+ .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
>+ .size_in = sizeof(mbox_in),
>+ .payload_in = &mbox_in,
>+ .size_out = alloc_size,
>+ .payload_out = mbox_out,
>+ .min_out = hdr_size,
>+ };
>+ rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
>+ if (rc < 0)
Please free mbox_out here.
>+ return rc;
>+
>+ if (mbox_cmd.size_out <= hdr_size) {
>+ rc = -ENXIO;
>+ goto err;
>+ }
>+
>+ /*
>+ * Make sure retrieved out buffer is multiple of feature
>+ * entries.
>+ */
>+ retrieved = mbox_cmd.size_out - hdr_size;
>+ if (retrieved % sizeof(struct cxl_feat_entry)) {
May be replace with feat_size as it was set to sizeof(struct cxl_feat_entry)?
>+ rc = -ENXIO;
>+ goto err;
>+ }
>+
>+ /*
>+ * If the reported output entries * defined entry size !=
>+ * retrieved output bytes, then the output package is incorrect.
>+ */
>+ if (mbox_out->num_entries * feat_size != retrieved) {
>+ rc = -ENXIO;
>+ goto err;
>+ }
>+
>+ memcpy(ptr, mbox_out->ents, retrieved);
>+ ptr += retrieved;
>+ /*
>+ * If the number of output entries is less than expected, add the
>+ * remaining entries to the next batch.
>+ */
>+ remain_feats += copy_feats - mbox_out->num_entries;
>+ start += mbox_out->num_entries;
>+ } while (remain_feats);
>+
>+ return 0;
>+
>+err:
Please free mbox_out here.
>+ cxl_mbox->num_features = 0;
>+ return rc;
>+}
>+EXPORT_SYMBOL_NS_GPL(cxl_get_supported_features, CXL);
>+
> /**
> * cxl_enumerate_cmds() - Enumerate commands for a device.
> * @mds: The driver data for the operation diff --git a/drivers/cxl/cxlmem.h
>b/drivers/cxl/cxlmem.h index efdf833f2c51..4d3690aa2f3b 100644
>--- a/drivers/cxl/cxlmem.h
>+++ b/drivers/cxl/cxlmem.h
>@@ -482,6 +482,7 @@ enum cxl_opcode {
> CXL_MBOX_OP_GET_LOG_CAPS = 0x0402,
> CXL_MBOX_OP_CLEAR_LOG = 0x0403,
> CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
>+ CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
> CXL_MBOX_OP_IDENTIFY = 0x4000,
> CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
> CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
>@@ -765,6 +766,32 @@ enum {
> CXL_PMEM_SEC_PASS_USER,
> };
>
>+/* Get Supported Features (0x500h) CXL r3.1 8.2.9.6.1 */ struct
>+cxl_mbox_get_sup_feats_in {
>+ __le32 count;
>+ __le16 start_idx;
>+ __le16 reserved;
Is u8 reserved[2]; better?
>+} __packed;
>+
>+struct cxl_feat_entry {
>+ uuid_t uuid;
>+ __le16 id;
>+ __le16 get_feat_size;
>+ __le16 set_feat_size;
>+ __le32 flags;
>+ u8 get_feat_ver;
>+ u8 set_feat_ver;
>+ __le16 effects;
May be set_effects?
>+ u8 reserved[18];
>+} __packed;
>+
>+struct cxl_mbox_get_sup_feats_out {
>+ __le16 num_entries;
>+ __le16 supported_feats;
>+ __le32 reserved;
u8 reserved[4]?
>+ struct cxl_feat_entry ents[]
>+__counted_by(le32_to_cpu(supported_feats));
>+} __packed;
>+
> int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
> struct cxl_mbox_cmd *cmd);
> int cxl_dev_state_identify(struct cxl_memdev_state *mds); @@ -824,4 +851,6
>@@ struct cxl_hdm { struct seq_file; struct dentry
>*cxl_debugfs_create_dir(const char *dir); void cxl_dpa_debug(struct seq_file
>*file, struct cxl_dev_state *cxlds);
>+
>+int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox);
> #endif /* __CXL_MEM_H__ */
>diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index
>7e26da706921..6a00238446f9 100644
>--- a/drivers/cxl/pci.c
>+++ b/drivers/cxl/pci.c
>@@ -872,6 +872,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const
>struct pci_device_id *id)
> if (rc)
> return rc;
>
>+ rc = cxl_get_supported_features(&cxlds->cxl_mbox);
>+ if (rc)
>+ dev_dbg(&pdev->dev, "No features enumerated.\n");
>+
> rc = cxl_set_timestamp(mds);
> if (rc)
> return rc;
>diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h index
>2380b22d7a12..570864239b8f 100644
>--- a/include/linux/cxl/mailbox.h
>+++ b/include/linux/cxl/mailbox.h
>@@ -53,6 +53,7 @@ struct cxl_mbox_cmd {
> * @mbox_mutex: mutex protects device mailbox and firmware
> * @mbox_wait: rcuwait for mailbox
> * @mbox_send: @dev specific transport for transmitting mailbox commands
>+ * @features: number of supported features
> */
> struct cxl_mailbox {
> struct device *host;
>@@ -63,6 +64,8 @@ struct cxl_mailbox {
> struct mutex mbox_mutex; /* lock to protect mailbox context */
> struct rcuwait mbox_wait;
> int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd
>*cmd);
>+ int num_features;
>+ struct cxl_feat_entry *entries;
Not sure storing the unrelated feature details in the struct cxl_mailbox is appropriate?
> };
>
> #endif
>diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index
>c6c0fe27495d..bd2535962f70 100644
>--- a/include/uapi/linux/cxl_mem.h
>+++ b/include/uapi/linux/cxl_mem.h
>@@ -50,6 +50,7 @@
> ___C(GET_LOG_CAPS, "Get Log Capabilities"), \
> ___C(CLEAR_LOG, "Clear Log"), \
> ___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
>+ ___C(GET_SUPPORTED_FEATURES, "Get Supported Features"),
> \
> ___C(MAX, "invalid / last command")
>
> #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
>--
>2.45.2
Thanks,
Shiju
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage
2024-08-21 16:05 ` Shiju Jose
@ 2024-08-21 18:10 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-08-21 18:10 UTC (permalink / raw)
To: Shiju Jose, linux-cxl@vger.kernel.org
Cc: dan.j.williams@intel.com, ira.weiny@intel.com,
vishal.l.verma@intel.com, alison.schofield@intel.com,
Jonathan Cameron, dave@stgolabs.net, jgg@nvidia.com
On 8/21/24 9:05 AM, Shiju Jose wrote:
> Hi Dave,
>
> Few comments inline.
>
>> -----Original Message-----
>> From: Dave Jiang <dave.jiang@intel.com>
>> Sent: 18 July 2024 22:32
>> To: linux-cxl@vger.kernel.org
>> Cc: dan.j.williams@intel.com; ira.weiny@intel.com; vishal.l.verma@intel.com;
>> alison.schofield@intel.com; Jonathan Cameron
>> <jonathan.cameron@huawei.com>; dave@stgolabs.net; jgg@nvidia.com; Shiju
>> Jose <shiju.jose@huawei.com>
>> Subject: [RFC PATCH 04/13] cxl: Add Get Supported Features command for
>> kernel usage
>>
>> CXL spec r3.1 8.2.9.6.1 Get Supported Features (Opcode 0500h) The command
>> retrieve the list of supported device-specific features (identified by UUID) and
>> general information about each Feature.
>>
>> The driver will retrieve the feature entries in order to make checks and provide
>> information for the Get Feature and Set Feature command. One of the main
>> piece of information retrieved are the effects a Set Feature command would
>> have for a particular feature.
>>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>> ---
>> drivers/cxl/core/mbox.c | 151 +++++++++++++++++++++++++++++++++++
>> drivers/cxl/cxlmem.h | 29 +++++++
>> drivers/cxl/pci.c | 4 +
>> include/linux/cxl/mailbox.h | 3 +
>> include/uapi/linux/cxl_mem.h | 1 +
>> 5 files changed, 188 insertions(+)
>>
>> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index
>> b9c64f1837a8..70e3962ed570 100644
>> --- a/drivers/cxl/core/mbox.c
>> +++ b/drivers/cxl/core/mbox.c
>> @@ -67,6 +67,7 @@ static struct cxl_mem_command
>> cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
>> CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
>> CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
>> CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
>> + CXL_CMD(GET_SUPPORTED_FEATURES, 0x8, CXL_VARIABLE_PAYLOAD,
>> 0),
>> };
>>
>> /*
>> @@ -790,6 +791,156 @@ static const uuid_t log_uuid[] = {
>> [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID, };
>>
>> +static void cxl_free_features(void *features) {
>> + kvfree(features);
>> +}
>> +
>> +static int cxl_get_supported_features_count(struct cxl_mailbox
>> +*cxl_mbox) {
>> + struct cxl_mbox_get_sup_feats_out mbox_out;
>> + struct cxl_mbox_get_sup_feats_in mbox_in;
>> + struct cxl_mbox_cmd mbox_cmd;
>> + int rc;
>> +
>> + memset(&mbox_in, 0, sizeof(mbox_in));
>> + mbox_in.count = sizeof(mbox_out);
>> + memset(&mbox_out, 0, sizeof(mbox_out));
>> + mbox_cmd = (struct cxl_mbox_cmd) {
>> + .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
>> + .size_in = sizeof(mbox_in),
>> + .payload_in = &mbox_in,
>> + .size_out = sizeof(mbox_out),
>> + .payload_out = &mbox_out,
>> + .min_out = sizeof(mbox_out),
>> + };
>> + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
>> + if (rc < 0)
>> + return rc;
>> +
>> + cxl_mbox->num_features = le16_to_cpu(mbox_out.supported_feats);
>> + if (!cxl_mbox->num_features)
>> + return -ENOENT;
>> +
>> + return 0;
>> +}
>> +
>> +int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox) {
>> + int remain_feats, max_size, max_feats, start, rc;
>> + int feat_size = sizeof(struct cxl_feat_entry);
>> + struct cxl_mbox_get_sup_feats_out *mbox_out;
>> + struct cxl_mbox_get_sup_feats_in mbox_in;
>> + int hdr_size = sizeof(*mbox_out);
>> + struct cxl_mbox_cmd mbox_cmd;
>> + struct cxl_mem_command *cmd;
>> + void *ptr;
>> +
>> + /* Get supported features is optional, need to check */
>> + cmd =
>> cxl_mem_find_command(CXL_MBOX_OP_GET_SUPPORTED_FEATURES);
>> + if (!cmd)
>> + return -EOPNOTSUPP;
>> + if (!test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
>> + return -EOPNOTSUPP;
>> +
>> + rc = cxl_get_supported_features_count(cxl_mbox);
>> + if (rc)
>> + return rc;
>> +
>> + struct cxl_feat_entry *entries __free(kvfree) =
>> + kvmalloc(cxl_mbox->num_features * feat_size, GFP_KERNEL);
>> +
>> + if (!entries)
>> + return -ENOMEM;
>> +
>> + cxl_mbox->entries = no_free_ptr(entries);
>> + rc = devm_add_action_or_reset(cxl_mbox->host, cxl_free_features,
>> + cxl_mbox->entries);
>> + if (rc)
>> + return rc;
>> +
>> + max_size = cxl_mbox->payload_size - hdr_size;
>> + /* max feat entries that can fit in mailbox max payload size */
>> + max_feats = max_size / feat_size;
>> + ptr = &cxl_mbox->entries[0];
>> +
>> + mbox_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
>> + if (!mbox_out)
>> + return -ENOMEM;
>> +
>> + start = 0;
>> + remain_feats = cxl_mbox->num_features;
>> + do {
>> + int retrieved, alloc_size, copy_feats;
>> +
>> + if (remain_feats > max_feats) {
>> + alloc_size = sizeof(*mbox_out) + max_feats * feat_size;
>> + remain_feats = remain_feats - max_feats;
>> + copy_feats = max_feats;
>> + } else {
>> + alloc_size = sizeof(*mbox_out) + remain_feats *
>> feat_size;
>> + copy_feats = remain_feats;
>> + remain_feats = 0;
>> + }
>> +
>> + memset(&mbox_in, 0, sizeof(mbox_in));
>> + mbox_in.count = alloc_size;
>> + mbox_in.start_idx = start;
>> + memset(mbox_out, 0, alloc_size);
>> + mbox_cmd = (struct cxl_mbox_cmd) {
>> + .opcode = CXL_MBOX_OP_GET_SUPPORTED_FEATURES,
>> + .size_in = sizeof(mbox_in),
>> + .payload_in = &mbox_in,
>> + .size_out = alloc_size,
>> + .payload_out = mbox_out,
>> + .min_out = hdr_size,
>> + };
>> + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
>> + if (rc < 0)
> Please free mbox_out here.
ok
>> + return rc;
>> +
>> + if (mbox_cmd.size_out <= hdr_size) {
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + /*
>> + * Make sure retrieved out buffer is multiple of feature
>> + * entries.
>> + */
>> + retrieved = mbox_cmd.size_out - hdr_size;
>> + if (retrieved % sizeof(struct cxl_feat_entry)) {
> May be replace with feat_size as it was set to sizeof(struct cxl_feat_entry)?
ok
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + /*
>> + * If the reported output entries * defined entry size !=
>> + * retrieved output bytes, then the output package is incorrect.
>> + */
>> + if (mbox_out->num_entries * feat_size != retrieved) {
>> + rc = -ENXIO;
>> + goto err;
>> + }
>> +
>> + memcpy(ptr, mbox_out->ents, retrieved);
>> + ptr += retrieved;
>> + /*
>> + * If the number of output entries is less than expected, add the
>> + * remaining entries to the next batch.
>> + */
>> + remain_feats += copy_feats - mbox_out->num_entries;
>> + start += mbox_out->num_entries;
>> + } while (remain_feats);
>> +
>> + return 0;
>> +
>> +err:
> Please free mbox_out here.
ok
>> + cxl_mbox->num_features = 0;
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_get_supported_features, CXL);
>> +
>> /**
>> * cxl_enumerate_cmds() - Enumerate commands for a device.
>> * @mds: The driver data for the operation diff --git a/drivers/cxl/cxlmem.h
>> b/drivers/cxl/cxlmem.h index efdf833f2c51..4d3690aa2f3b 100644
>> --- a/drivers/cxl/cxlmem.h
>> +++ b/drivers/cxl/cxlmem.h
>> @@ -482,6 +482,7 @@ enum cxl_opcode {
>> CXL_MBOX_OP_GET_LOG_CAPS = 0x0402,
>> CXL_MBOX_OP_CLEAR_LOG = 0x0403,
>> CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
>> + CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
>> CXL_MBOX_OP_IDENTIFY = 0x4000,
>> CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
>> CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
>> @@ -765,6 +766,32 @@ enum {
>> CXL_PMEM_SEC_PASS_USER,
>> };
>>
>> +/* Get Supported Features (0x500h) CXL r3.1 8.2.9.6.1 */ struct
>> +cxl_mbox_get_sup_feats_in {
>> + __le32 count;
>> + __le16 start_idx;
>> + __le16 reserved;
> Is u8 reserved[2]; better?
>> +} __packed;
>> +
>> +struct cxl_feat_entry {
>> + uuid_t uuid;
>> + __le16 id;
>> + __le16 get_feat_size;
>> + __le16 set_feat_size;
>> + __le32 flags;
>> + u8 get_feat_ver;
>> + u8 set_feat_ver;
>> + __le16 effects;
> May be set_effects?
ok
>> + u8 reserved[18];
>> +} __packed;
>> +
>> +struct cxl_mbox_get_sup_feats_out {
>> + __le16 num_entries;
>> + __le16 supported_feats;
>> + __le32 reserved;
> u8 reserved[4]?
ok
>> + struct cxl_feat_entry ents[]
>> +__counted_by(le32_to_cpu(supported_feats));
>> +} __packed;
>> +
>> int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
>> struct cxl_mbox_cmd *cmd);
>> int cxl_dev_state_identify(struct cxl_memdev_state *mds); @@ -824,4 +851,6
>> @@ struct cxl_hdm { struct seq_file; struct dentry
>> *cxl_debugfs_create_dir(const char *dir); void cxl_dpa_debug(struct seq_file
>> *file, struct cxl_dev_state *cxlds);
>> +
>> +int cxl_get_supported_features(struct cxl_mailbox *cxl_mbox);
>> #endif /* __CXL_MEM_H__ */
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index
>> 7e26da706921..6a00238446f9 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -872,6 +872,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const
>> struct pci_device_id *id)
>> if (rc)
>> return rc;
>>
>> + rc = cxl_get_supported_features(&cxlds->cxl_mbox);
>> + if (rc)
>> + dev_dbg(&pdev->dev, "No features enumerated.\n");
>> +
>> rc = cxl_set_timestamp(mds);
>> if (rc)
>> return rc;
>> diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h index
>> 2380b22d7a12..570864239b8f 100644
>> --- a/include/linux/cxl/mailbox.h
>> +++ b/include/linux/cxl/mailbox.h
>> @@ -53,6 +53,7 @@ struct cxl_mbox_cmd {
>> * @mbox_mutex: mutex protects device mailbox and firmware
>> * @mbox_wait: rcuwait for mailbox
>> * @mbox_send: @dev specific transport for transmitting mailbox commands
>> + * @features: number of supported features
>> */
>> struct cxl_mailbox {
>> struct device *host;
>> @@ -63,6 +64,8 @@ struct cxl_mailbox {
>> struct mutex mbox_mutex; /* lock to protect mailbox context */
>> struct rcuwait mbox_wait;
>> int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd
>> *cmd);
>> + int num_features;
>> + struct cxl_feat_entry *entries;
> Not sure storing the unrelated feature details in the struct cxl_mailbox is appropriate?
Yeah I suppose it can be up leveled to the dev_state context.
>> };
>>
>> #endif
>> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index
>> c6c0fe27495d..bd2535962f70 100644
>> --- a/include/uapi/linux/cxl_mem.h
>> +++ b/include/uapi/linux/cxl_mem.h
>> @@ -50,6 +50,7 @@
>> ___C(GET_LOG_CAPS, "Get Log Capabilities"), \
>> ___C(CLEAR_LOG, "Clear Log"), \
>> ___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
>> + ___C(GET_SUPPORTED_FEATURES, "Get Supported Features"),
>> \
>> ___C(MAX, "invalid / last command")
>>
>> #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
>> --
>> 2.45.2
>
> Thanks,
> Shiju
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 05/13] cxl/test: Add Get Supported Features mailbox command support
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (3 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 04/13] cxl: Add Get Supported Features command for kernel usage Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 06/13] cxl: Add Get Feature " Dave Jiang
` (10 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add cxl-test emulation of Get Supported Features mailbox command.
Currently only planning to add the Device Patrol Scrub Control feature
as a testing vehicle.
CXL spec r3.1 8.2.9.9.11.1 Device Patrol Scrub Control Feature.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
tools/testing/cxl/test/mem.c | 69 ++++++++++++++++++++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index 20d58f883ca2..c2e04dbc8af6 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -44,6 +44,10 @@ static struct cxl_cel_entry mock_cel[] = {
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_SUPPORTED_LOGS),
.effect = CXL_CMD_EFFECT_NONE,
},
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_GET_SUPPORTED_FEATURES),
+ .effect = CXL_CMD_EFFECT_NONE,
+ },
{
.opcode = cpu_to_le16(CXL_MBOX_OP_IDENTIFY),
.effect = CXL_CMD_EFFECT_NONE,
@@ -1334,6 +1338,64 @@ static int mock_activate_fw(struct cxl_mockmem_data *mdata,
return -EINVAL;
}
+#define DEV_PATROL_SCRUB_FEAT_UUID \
+ UUID_INIT(0x96dad7d6, 0xfde8, 0x482b, 0xa7, 0x33, 0x75, 0x77, 0x4e, \
+ 0x06, 0xdb, 0x8a)
+
+static void fill_feature_device_patrol_scrub_control(struct cxl_feat_entry *feat)
+{
+ feat->uuid = DEV_PATROL_SCRUB_FEAT_UUID;
+ feat->id = 0;
+ feat->get_feat_size = cpu_to_le16(4);
+ feat->set_feat_size = cpu_to_le16(2);
+ feat->flags = cpu_to_le32(0x21);
+ feat->get_feat_ver = 1;
+ feat->set_feat_ver = 1;
+ feat->effects = cpu_to_le16(0x202);
+}
+
+static int mock_get_supported_features(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct cxl_mbox_get_sup_feats_in *in = cmd->payload_in;
+ struct cxl_mbox_get_sup_feats_out *out = cmd->payload_out;
+ struct cxl_feat_entry *feat;
+ u16 start_idx, count;
+
+ if (cmd->size_out < sizeof(*out)) {
+ cmd->return_code = CXL_MBOX_CMD_RC_PAYLOADLEN;
+ return -EINVAL;
+ }
+
+ /*
+ * Current emulation only supports 1 feature
+ */
+ start_idx = le16_to_cpu(in->start_idx);
+ if (start_idx != 0) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ count = le16_to_cpu(in->count);
+ if (count < sizeof(*out)) {
+ cmd->return_code = CXL_MBOX_CMD_RC_PAYLOADLEN;
+ return -EINVAL;
+ }
+
+ out->supported_feats = cpu_to_le16(1);
+ cmd->return_code = 0;
+ if (count < sizeof(*out) + sizeof(*feat)) {
+ out->num_entries = 0;
+ return 0;
+ }
+
+ out->num_entries = 1;
+ feat = out->ents;
+ fill_feature_device_patrol_scrub_control(feat);
+
+ return 0;
+}
+
static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd)
{
@@ -1421,6 +1483,9 @@ static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
case CXL_MBOX_OP_ACTIVATE_FW:
rc = mock_activate_fw(mdata, cmd);
break;
+ case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
+ rc = mock_get_supported_features(mdata, cmd);
+ break;
default:
break;
}
@@ -1508,6 +1573,10 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
if (rc)
return rc;
+ rc = cxl_get_supported_features(&cxlds->cxl_mbox);
+ if (rc)
+ dev_dbg(dev, "No features enumerated\n");
+
rc = cxl_poison_state_init(mds);
if (rc)
return rc;
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 06/13] cxl: Add Get Feature command support
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (4 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 05/13] cxl/test: Add Get Supported Features mailbox command support Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 07/13] cxl: Add Set " Dave Jiang
` (9 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add enumeration of Get Feature mailbox command for the kernel to recognize
the command being passed in from user space.
CXL spec r3.1 8.2.9.6.2 Get Feature (Opcode 0501h)
The feature requested is identified by specific UUID.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/mbox.c | 1 +
drivers/cxl/cxlmem.h | 1 +
include/uapi/linux/cxl_mem.h | 1 +
3 files changed, 3 insertions(+)
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 70e3962ed570..b58b8cc517c9 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
CXL_CMD(GET_SUPPORTED_FEATURES, 0x8, CXL_VARIABLE_PAYLOAD, 0),
+ CXL_CMD(GET_FEATURE, 0x15, CXL_VARIABLE_PAYLOAD, 0),
};
/*
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 4d3690aa2f3b..317cb92b0bfb 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -483,6 +483,7 @@ enum cxl_opcode {
CXL_MBOX_OP_CLEAR_LOG = 0x0403,
CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
+ CXL_MBOX_OP_GET_FEATURE = 0x0501,
CXL_MBOX_OP_IDENTIFY = 0x4000,
CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
index bd2535962f70..90dcb9723997 100644
--- a/include/uapi/linux/cxl_mem.h
+++ b/include/uapi/linux/cxl_mem.h
@@ -51,6 +51,7 @@
___C(CLEAR_LOG, "Clear Log"), \
___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
___C(GET_SUPPORTED_FEATURES, "Get Supported Features"), \
+ ___C(GET_FEATURE, "Get Feature"), \
___C(MAX, "invalid / last command")
#define ___C(a, b) CXL_MEM_COMMAND_ID_##a
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 07/13] cxl: Add Set Feature command support
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (5 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 06/13] cxl: Add Get Feature " Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands Dave Jiang
` (8 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add enumeration of Set Feature mailbox command for the kernel to recognize
the command being passed in from user space.
CXL spec r3.1 8.2.9.6.3 Set Feature (Opcode 0502h)
The feature requested is identified by specific UUID.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/mbox.c | 1 +
drivers/cxl/cxlmem.h | 1 +
include/uapi/linux/cxl_mem.h | 1 +
3 files changed, 3 insertions(+)
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index b58b8cc517c9..14ebe69c47bf 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -69,6 +69,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
CXL_CMD(GET_SUPPORTED_FEATURES, 0x8, CXL_VARIABLE_PAYLOAD, 0),
CXL_CMD(GET_FEATURE, 0x15, CXL_VARIABLE_PAYLOAD, 0),
+ CXL_CMD(SET_FEATURE, CXL_VARIABLE_PAYLOAD, 0, 0),
};
/*
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 317cb92b0bfb..5e190f941e58 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -484,6 +484,7 @@ enum cxl_opcode {
CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
CXL_MBOX_OP_GET_FEATURE = 0x0501,
+ CXL_MBOX_OP_SET_FEATURE = 0x0502,
CXL_MBOX_OP_IDENTIFY = 0x4000,
CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
index 90dcb9723997..469dd481795f 100644
--- a/include/uapi/linux/cxl_mem.h
+++ b/include/uapi/linux/cxl_mem.h
@@ -52,6 +52,7 @@
___C(GET_SUP_LOG_SUBLIST, "Get Supported Logs Sub-List"), \
___C(GET_SUPPORTED_FEATURES, "Get Supported Features"), \
___C(GET_FEATURE, "Get Feature"), \
+ ___C(SET_FEATURE, "Set Feature"), \
___C(MAX, "invalid / last command")
#define ___C(a, b) CXL_MEM_COMMAND_ID_##a
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (6 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 07/13] cxl: Add Set " Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-22 15:31 ` Jason Gunthorpe
2024-07-26 18:02 ` Jonathan Cameron
2024-07-18 21:32 ` [RFC PATCH 09/13] fwctl/cxl: Add support for get driver information Dave Jiang
` (7 subsequent siblings)
15 siblings, 2 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add an fwctl (auxiliary bus) driver to allow sending of CXL feature
commands from userspace through as ioctls. Create a driver skeleton for
initial setup.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
MAINTAINERS | 7 +++
drivers/cxl/core/memdev.c | 19 +++++++
drivers/fwctl/Kconfig | 9 ++++
drivers/fwctl/Makefile | 1 +
drivers/fwctl/cxl/Makefile | 4 ++
drivers/fwctl/cxl/cxl.c | 103 +++++++++++++++++++++++++++++++++++++
include/uapi/fwctl/fwctl.h | 1 +
7 files changed, 144 insertions(+)
create mode 100644 drivers/fwctl/cxl/Makefile
create mode 100644 drivers/fwctl/cxl/cxl.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 3e1934b7e044..5d6c470c06ea 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -9073,6 +9073,13 @@ L: linux-kernel@vger.kernel.org
S: Maintained
F: drivers/fwctl/mlx5/
+FWCTL CXL DRIVER
+M: Dave Jiang <dave.jiang@intel.com>
+R: Dan Williams <dan.j.williams@intel.com>
+L: linux-cxl@vger.kernel.org
+S: Maintained
+F: drivers/fwctl/cxl/
+
GALAXYCORE GC0308 CAMERA SENSOR DRIVER
M: Sebastian Reichel <sre@kernel.org>
L: linux-media@vger.kernel.org
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 9f0fe698414d..0f6ec85ef9c4 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -13,6 +13,8 @@
static DECLARE_RWSEM(cxl_memdev_rwsem);
+#define CXL_ADEV_NAME "fwctl-cxl"
+
/*
* An entire PCI topology full of devices should be enough for any
* config
@@ -1030,6 +1032,7 @@ static const struct file_operations cxl_memdev_fops = {
struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
struct cxl_dev_state *cxlds)
{
+ struct auxiliary_device *adev;
struct cxl_memdev *cxlmd;
struct device *dev;
struct cdev *cdev;
@@ -1056,11 +1059,27 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
if (rc)
goto err;
+ adev = &cxlds->cxl_mbox.adev;
+ adev->id = cxlmd->id;
+ adev->name = CXL_ADEV_NAME;
+ adev->dev.parent = dev;
+
+ rc = auxiliary_device_init(adev);
+ if (rc)
+ goto err;
+
+ rc = auxiliary_device_add(adev);
+ if (rc)
+ goto aux_err;
+
rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
if (rc)
return ERR_PTR(rc);
return cxlmd;
+aux_err:
+ auxiliary_device_uninit(adev);
+
err:
/*
* The cdev was briefly live, shutdown any ioctl operations that
diff --git a/drivers/fwctl/Kconfig b/drivers/fwctl/Kconfig
index e5ee2d46d431..e49903a9d0d3 100644
--- a/drivers/fwctl/Kconfig
+++ b/drivers/fwctl/Kconfig
@@ -19,5 +19,14 @@ config FWCTL_MLX5
This will allow configuration and debug tools to work out of the box on
mainstream kernel.
+ If you don't know what to do here, say N.
+
+config FWCTL_CXL
+ tristate "CXL fwctl driver"
+ depends on CXL_BUS
+ help
+ CXLCTL provides interface for the user process to access user allowed
+ mailbox commands for CXL device.
+
If you don't know what to do here, say N.
endif
diff --git a/drivers/fwctl/Makefile b/drivers/fwctl/Makefile
index 1c535f694d7f..bd356e6f2e5a 100644
--- a/drivers/fwctl/Makefile
+++ b/drivers/fwctl/Makefile
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_FWCTL) += fwctl.o
obj-$(CONFIG_FWCTL_MLX5) += mlx5/
+obj-$(CONFIG_FWCTL_CXL) += cxl/
fwctl-y += main.o
diff --git a/drivers/fwctl/cxl/Makefile b/drivers/fwctl/cxl/Makefile
new file mode 100644
index 000000000000..623194521572
--- /dev/null
+++ b/drivers/fwctl/cxl/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_FWCTL_CXL) += cxl_fwctl.o
+
+cxl_fwctl-y += cxl.o
diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
new file mode 100644
index 000000000000..518ba2afada8
--- /dev/null
+++ b/drivers/fwctl/cxl/cxl.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2024, Intel Corporation
+ */
+#include <linux/fwctl.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/cxl/mailbox.h>
+#include <linux/auxiliary_bus.h>
+
+struct cxlctl_uctx {
+ struct fwctl_uctx uctx;
+ u32 uctx_caps;
+ u32 uctx_uid;
+};
+
+struct cxlctl_dev {
+ struct fwctl_device fwctl;
+ struct cxl_mailbox *mbox;
+};
+
+DEFINE_FREE(cxlctl, struct cxlctl_dev *, if (_T) fwctl_put(&_T->fwctl))
+
+static int cxlctl_open_uctx(struct fwctl_uctx *uctx)
+{
+ return 0;
+}
+
+static void cxlctl_close_uctx(struct fwctl_uctx *uctx)
+{
+}
+
+static void *cxlctl_info(struct fwctl_uctx *uctx, size_t *length)
+{
+ /* Place holder */
+ return ERR_PTR(-EOPNOTSUPP);
+}
+
+static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
+ void *rpc_in, size_t in_len, size_t *out_len)
+{
+ /* Place holder */
+ return ERR_PTR(-EOPNOTSUPP);
+}
+
+static const struct fwctl_ops cxlctl_ops = {
+ .device_type = FWCTL_DEVICE_TYPE_CXL,
+ .uctx_size = sizeof(struct cxlctl_uctx),
+ .open_uctx = cxlctl_open_uctx,
+ .close_uctx = cxlctl_close_uctx,
+ .info = cxlctl_info,
+ .fw_rpc = cxlctl_fw_rpc,
+};
+
+static int cxlctl_probe(struct auxiliary_device *adev,
+ const struct auxiliary_device_id *id)
+{
+ struct cxl_mailbox *mbox = container_of(adev, struct cxl_mailbox, adev);
+ struct cxlctl_dev *cxlctl __free(cxlctl) =
+ fwctl_alloc_device(mbox->host, &cxlctl_ops,
+ struct cxlctl_dev, fwctl);
+ int rc;
+
+ if (!cxlctl)
+ return -ENOMEM;
+
+ cxlctl->mbox = mbox;
+
+ rc = fwctl_register(&cxlctl->fwctl);
+ if (rc)
+ return rc;
+
+ auxiliary_set_drvdata(adev, no_free_ptr(cxlctl));
+
+ return 0;
+}
+
+static void cxlctl_remove(struct auxiliary_device *adev)
+{
+ struct cxlctl_dev *ctldev __free(cxlctl) = auxiliary_get_drvdata(adev);
+
+ fwctl_unregister(&ctldev->fwctl);
+}
+
+static const struct auxiliary_device_id cxlctl_id_table[] = {
+ { .name = "CXL.fwctl", },
+ {},
+};
+MODULE_DEVICE_TABLE(auxiliary, cxlctl_id_table);
+
+static struct auxiliary_driver cxlctl_driver = {
+ .name = "cxl_fwctl",
+ .probe = cxlctl_probe,
+ .remove = cxlctl_remove,
+ .id_table = cxlctl_id_table,
+};
+
+module_auxiliary_driver(cxlctl_driver);
+
+MODULE_IMPORT_NS(CXL);
+MODULE_IMPORT_NS(FWCTL);
+MODULE_DESCRIPTION("CXL fwctl driver");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL");
diff --git a/include/uapi/fwctl/fwctl.h b/include/uapi/fwctl/fwctl.h
index 49a357e1bc71..6edcc38a6eee 100644
--- a/include/uapi/fwctl/fwctl.h
+++ b/include/uapi/fwctl/fwctl.h
@@ -43,6 +43,7 @@ enum {
enum fwctl_device_type {
FWCTL_DEVICE_TYPE_ERROR = 0,
FWCTL_DEVICE_TYPE_MLX5 = 1,
+ FWCTL_DEVICE_TYPE_CXL = 2,
};
/**
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-07-18 21:32 ` [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands Dave Jiang
@ 2024-07-22 15:31 ` Jason Gunthorpe
2024-07-22 21:42 ` Dave Jiang
2024-07-26 18:02 ` Jonathan Cameron
1 sibling, 1 reply; 47+ messages in thread
From: Jason Gunthorpe @ 2024-07-22 15:31 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On Thu, Jul 18, 2024 at 02:32:26PM -0700, Dave Jiang wrote:
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 9f0fe698414d..0f6ec85ef9c4 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -13,6 +13,8 @@
>
> static DECLARE_RWSEM(cxl_memdev_rwsem);
>
> +#define CXL_ADEV_NAME "fwctl-cxl"
> +
This should be in a header?
> @@ -1056,11 +1059,27 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> if (rc)
> goto err;
>
> + adev = &cxlds->cxl_mbox.adev;
> + adev->id = cxlmd->id;
> + adev->name = CXL_ADEV_NAME;
> + adev->dev.parent = dev;
It goes there
> +static const struct auxiliary_device_id cxlctl_id_table[] = {
> + { .name = "CXL.fwctl", },
> + {},
But then how does this match work? Shouldn't it be CXL_ADEV_NAME ?
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-07-22 15:31 ` Jason Gunthorpe
@ 2024-07-22 21:42 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-22 21:42 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On 7/22/24 8:31 AM, Jason Gunthorpe wrote:
> On Thu, Jul 18, 2024 at 02:32:26PM -0700, Dave Jiang wrote:
>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 9f0fe698414d..0f6ec85ef9c4 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -13,6 +13,8 @@
>>
>> static DECLARE_RWSEM(cxl_memdev_rwsem);
>>
>> +#define CXL_ADEV_NAME "fwctl-cxl"
>> +
>
> This should be in a header?
yes. will fix.
>
>> @@ -1056,11 +1059,27 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> if (rc)
>> goto err;
>>
>> + adev = &cxlds->cxl_mbox.adev;
>> + adev->id = cxlmd->id;
>> + adev->name = CXL_ADEV_NAME;
>> + adev->dev.parent = dev;
>
> It goes there
>
>> +static const struct auxiliary_device_id cxlctl_id_table[] = {
>> + { .name = "CXL.fwctl", },
>> + {},
>
> But then how does this match work? Shouldn't it be CXL_ADEV_NAME ?
Yes will fix.
>
> Jason
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-07-18 21:32 ` [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands Dave Jiang
2024-07-22 15:31 ` Jason Gunthorpe
@ 2024-07-26 18:02 ` Jonathan Cameron
2024-09-24 23:44 ` Dave Jiang
1 sibling, 1 reply; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-26 18:02 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:26 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> Add an fwctl (auxiliary bus) driver to allow sending of CXL feature
> commands from userspace through as ioctls. Create a driver skeleton for
> initial setup.
>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Main question is why auxiliary bus?
We already do similar stuff in CXL on the cxl bus (e.g.
the CPMU devices etc) that then register a class
device with another subsystem.
> GALAXYCORE GC0308 CAMERA SENSOR DRIVER
> M: Sebastian Reichel <sre@kernel.org>
> L: linux-media@vger.kernel.org
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 9f0fe698414d..0f6ec85ef9c4 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -13,6 +13,8 @@
>
> static DECLARE_RWSEM(cxl_memdev_rwsem);
>
> +#define CXL_ADEV_NAME "fwctl-cxl"
> +
> /*
> * An entire PCI topology full of devices should be enough for any
> * config
> @@ -1030,6 +1032,7 @@ static const struct file_operations cxl_memdev_fops = {
> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> struct cxl_dev_state *cxlds)
> {
> + struct auxiliary_device *adev;
Why an auxdev? It can be any convienient dev to which a driver
will bind. Why not spin one on the CXL bus for this purpose?
Maybe that creates and implied relationship you don't want, but
we already spin lots of devices there like the pmus, so not sure
why one more is a problem.
> struct cxl_memdev *cxlmd;
> struct device *dev;
> struct cdev *cdev;
> @@ -1056,11 +1059,27 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> if (rc)
> goto err;
>
> + adev = &cxlds->cxl_mbox.adev;
> + adev->id = cxlmd->id;
> + adev->name = CXL_ADEV_NAME;
> + adev->dev.parent = dev;
> +
> + rc = auxiliary_device_init(adev);
> + if (rc)
> + goto err;
> +
> + rc = auxiliary_device_add(adev);
> + if (rc)
> + goto aux_err;
> +
> rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
> if (rc)
> return ERR_PTR(rc);
> return cxlmd;
>
> +aux_err:
> + auxiliary_device_uninit(adev);
> +
> err:
> /*
> * The cdev was briefly live, shutdown any ioctl operations that
> diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
> new file mode 100644
> index 000000000000..518ba2afada8
> --- /dev/null
> +++ b/drivers/fwctl/cxl/cxl.c
> @@ -0,0 +1,103 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (c) 2024, Intel Corporation
> + */
> +#include <linux/fwctl.h>
> +#include <linux/auxiliary_bus.h>
> +#include <linux/cxl/mailbox.h>
> +#include <linux/auxiliary_bus.h>
Pick a standard ordering for header.
Any ordering is fine.
> +
> +static void cxlctl_remove(struct auxiliary_device *adev)
> +{
> + struct cxlctl_dev *ctldev __free(cxlctl) = auxiliary_get_drvdata(adev);
> +
> + fwctl_unregister(&ctldev->fwctl);
Same question on reference counting I asked in the set Jason posted
I don't get why we need the __free()
> +}
> +
> +static const struct auxiliary_device_id cxlctl_id_table[] = {
> + { .name = "CXL.fwctl", },
> + {},
No trailing ,
> +};
> +MODULE_DEVICE_TABLE(auxiliary, cxlctl_id_table);
> +
> +static struct auxiliary_driver cxlctl_driver = {
> + .name = "cxl_fwctl",
> + .probe = cxlctl_probe,
> + .remove = cxlctl_remove,
> + .id_table = cxlctl_id_table,
> +};
> +
> +module_auxiliary_driver(cxlctl_driver);
> +
> +MODULE_IMPORT_NS(CXL);
> +MODULE_IMPORT_NS(FWCTL);
> +MODULE_DESCRIPTION("CXL fwctl driver");
> +MODULE_AUTHOR("Intel Corporation");
> +MODULE_LICENSE("GPL");
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-07-26 18:02 ` Jonathan Cameron
@ 2024-09-24 23:44 ` Dave Jiang
2024-09-26 17:37 ` Jason Gunthorpe
0 siblings, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-09-24 23:44 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On 7/26/24 11:02 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:26 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> Add an fwctl (auxiliary bus) driver to allow sending of CXL feature
>> commands from userspace through as ioctls. Create a driver skeleton for
>> initial setup.
>>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>
> Main question is why auxiliary bus?
It was convenient at the time. I just copied what Jason had in the mlx5 driver.
> We already do similar stuff in CXL on the cxl bus (e.g.
> the CPMU devices etc) that then register a class
> device with another subsystem.
Sure. I'll adapt it to the cxl_bus.
DJ
>
>> GALAXYCORE GC0308 CAMERA SENSOR DRIVER
>> M: Sebastian Reichel <sre@kernel.org>
>> L: linux-media@vger.kernel.org
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 9f0fe698414d..0f6ec85ef9c4 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -13,6 +13,8 @@
>>
>> static DECLARE_RWSEM(cxl_memdev_rwsem);
>>
>> +#define CXL_ADEV_NAME "fwctl-cxl"
>> +
>> /*
>> * An entire PCI topology full of devices should be enough for any
>> * config
>> @@ -1030,6 +1032,7 @@ static const struct file_operations cxl_memdev_fops = {
>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> struct cxl_dev_state *cxlds)
>> {
>> + struct auxiliary_device *adev;
>
> Why an auxdev? It can be any convienient dev to which a driver
> will bind. Why not spin one on the CXL bus for this purpose?
>
> Maybe that creates and implied relationship you don't want, but
> we already spin lots of devices there like the pmus, so not sure
> why one more is a problem.
>
>> struct cxl_memdev *cxlmd;
>> struct device *dev;
>> struct cdev *cdev;
>> @@ -1056,11 +1059,27 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> if (rc)
>> goto err;
>>
>> + adev = &cxlds->cxl_mbox.adev;
>> + adev->id = cxlmd->id;
>> + adev->name = CXL_ADEV_NAME;
>> + adev->dev.parent = dev;
>> +
>> + rc = auxiliary_device_init(adev);
>> + if (rc)
>> + goto err;
>> +
>> + rc = auxiliary_device_add(adev);
>> + if (rc)
>> + goto aux_err;
>> +
>> rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
>> if (rc)
>> return ERR_PTR(rc);
>> return cxlmd;
>>
>> +aux_err:
>> + auxiliary_device_uninit(adev);
>> +
>> err:
>> /*
>> * The cdev was briefly live, shutdown any ioctl operations that
>
>> diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
>> new file mode 100644
>> index 000000000000..518ba2afada8
>> --- /dev/null
>> +++ b/drivers/fwctl/cxl/cxl.c
>> @@ -0,0 +1,103 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * Copyright (c) 2024, Intel Corporation
>> + */
>> +#include <linux/fwctl.h>
>> +#include <linux/auxiliary_bus.h>
>> +#include <linux/cxl/mailbox.h>
>> +#include <linux/auxiliary_bus.h>
>
> Pick a standard ordering for header.
> Any ordering is fine.
>
>> +
>> +static void cxlctl_remove(struct auxiliary_device *adev)
>> +{
>> + struct cxlctl_dev *ctldev __free(cxlctl) = auxiliary_get_drvdata(adev);
>> +
>> + fwctl_unregister(&ctldev->fwctl);
>
> Same question on reference counting I asked in the set Jason posted
> I don't get why we need the __free()
>
>> +}
>> +
>> +static const struct auxiliary_device_id cxlctl_id_table[] = {
>> + { .name = "CXL.fwctl", },
>> + {},
> No trailing ,
>
>> +};
>> +MODULE_DEVICE_TABLE(auxiliary, cxlctl_id_table);
>> +
>> +static struct auxiliary_driver cxlctl_driver = {
>> + .name = "cxl_fwctl",
>> + .probe = cxlctl_probe,
>> + .remove = cxlctl_remove,
>> + .id_table = cxlctl_id_table,
>> +};
>> +
>> +module_auxiliary_driver(cxlctl_driver);
>> +
>> +MODULE_IMPORT_NS(CXL);
>> +MODULE_IMPORT_NS(FWCTL);
>> +MODULE_DESCRIPTION("CXL fwctl driver");
>> +MODULE_AUTHOR("Intel Corporation");
>> +MODULE_LICENSE("GPL");
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-09-24 23:44 ` Dave Jiang
@ 2024-09-26 17:37 ` Jason Gunthorpe
2024-09-26 20:26 ` Dave Jiang
0 siblings, 1 reply; 47+ messages in thread
From: Jason Gunthorpe @ 2024-09-26 17:37 UTC (permalink / raw)
To: Dave Jiang
Cc: Jonathan Cameron, linux-cxl, dan.j.williams, ira.weiny,
vishal.l.verma, alison.schofield, dave, shiju.jose
On Tue, Sep 24, 2024 at 04:44:19PM -0700, Dave Jiang wrote:
> >> @@ -1030,6 +1032,7 @@ static const struct file_operations cxl_memdev_fops = {
> >> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> >> struct cxl_dev_state *cxlds)
> >> {
> >> + struct auxiliary_device *adev;
> >
> > Why an auxdev? It can be any convienient dev to which a driver
> > will bind. Why not spin one on the CXL bus for this purpose?
IMHO bus members should adhere to some kinds of rules about the bus
itself. Devices should have the same set of paramters/etc. I don't
know what cxl_bus is, but I'd be surprised that something with that
name has SW only entities on it as devices??
aux bus was specifically for SW only entities that are just for
linking software components.
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
2024-09-26 17:37 ` Jason Gunthorpe
@ 2024-09-26 20:26 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-09-26 20:26 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: Jonathan Cameron, linux-cxl, dan.j.williams, ira.weiny,
vishal.l.verma, alison.schofield, dave, shiju.jose
On 9/26/24 10:37 AM, Jason Gunthorpe wrote:
> On Tue, Sep 24, 2024 at 04:44:19PM -0700, Dave Jiang wrote:
>
>>>> @@ -1030,6 +1032,7 @@ static const struct file_operations cxl_memdev_fops = {
>>>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>>>> struct cxl_dev_state *cxlds)
>>>> {
>>>> + struct auxiliary_device *adev;
>>>
>>> Why an auxdev? It can be any convienient dev to which a driver
>>> will bind. Why not spin one on the CXL bus for this purpose?
>
> IMHO bus members should adhere to some kinds of rules about the bus
> itself. Devices should have the same set of paramters/etc. I don't
> know what cxl_bus is, but I'd be surprised that something with that
> name has SW only entities on it as devices??
So I did do the conversion yesterday and it ended up looking something like this [1]. cxl_bus has cxl objects that are hardware representations (endpoints, ports, decoders, pmu, etc) or software representations (regions). So it's probably ok to keep all the cxl stuff on the same bus. I don't have a strong feeling on whether use cxl bus that already exists or using aux bus.
>
> aux bus was specifically for SW only entities that are just for
> linking software components.
>
> Jason
[1]: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/commit/?h=cxl/fwctl&id=231da137a959331cc5bc80ef4acb3d7e667bfbad
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 09/13] fwctl/cxl: Add support for get driver information
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (7 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 08/13] fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands Dave Jiang
` (6 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add definition for fwctl_ops->info() to return driver information. The
function will return the commands supported by the fwctl char device as
a bitmap of enable commands.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/fwctl/cxl/cxl.c | 20 ++++++++++++++++++--
include/uapi/fwctl/cxl.h | 25 +++++++++++++++++++++++++
2 files changed, 43 insertions(+), 2 deletions(-)
create mode 100644 include/uapi/fwctl/cxl.h
diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
index 518ba2afada8..22f62034c021 100644
--- a/drivers/fwctl/cxl/cxl.c
+++ b/drivers/fwctl/cxl/cxl.c
@@ -6,6 +6,7 @@
#include <linux/auxiliary_bus.h>
#include <linux/cxl/mailbox.h>
#include <linux/auxiliary_bus.h>
+#include <uapi/fwctl/cxl.h>
struct cxlctl_uctx {
struct fwctl_uctx uctx;
@@ -22,6 +23,12 @@ DEFINE_FREE(cxlctl, struct cxlctl_dev *, if (_T) fwctl_put(&_T->fwctl))
static int cxlctl_open_uctx(struct fwctl_uctx *uctx)
{
+ struct cxlctl_uctx *cxlctl_uctx =
+ container_of(uctx, struct cxlctl_uctx, uctx);
+
+ cxlctl_uctx->uctx_caps = BIT(FWCTL_CXL_QUERY_COMMANDS) |
+ BIT(FWCTL_CXL_SEND_COMMAND);
+
return 0;
}
@@ -31,8 +38,17 @@ static void cxlctl_close_uctx(struct fwctl_uctx *uctx)
static void *cxlctl_info(struct fwctl_uctx *uctx, size_t *length)
{
- /* Place holder */
- return ERR_PTR(-EOPNOTSUPP);
+ struct cxlctl_uctx *cxlctl_uctx =
+ container_of(uctx, struct cxlctl_uctx, uctx);
+ struct fwctl_info_cxl *info;
+
+ info = kzalloc(sizeof(*info), GFP_KERNEL);
+ if (!info)
+ return ERR_PTR(-ENOMEM);
+
+ info->uctx_caps = cxlctl_uctx->uctx_caps;
+
+ return info;
}
static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
diff --git a/include/uapi/fwctl/cxl.h b/include/uapi/fwctl/cxl.h
new file mode 100644
index 000000000000..d3a735f1fe4e
--- /dev/null
+++ b/include/uapi/fwctl/cxl.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (c) 2024, Intel Corporation
+ *
+ * These are definitions for the mailbox command interface of CXL subsystem.
+ */
+#ifndef _UAPI_FWCTL_CXL_H_
+#define _UAPI_FWCTL_CXL_H_
+
+enum fwctl_cxl_commands {
+ FWCTL_CXL_QUERY_COMMANDS = 0,
+ FWCTL_CXL_SEND_COMMAND,
+};
+
+/**
+ * struct fwctl_info_cxl - ioctl(FWCTL_INFO) out_device_data
+ * @uctx_caps: The command capabilities driver accepts.
+ *
+ * Return basic information about the FW interface available.
+ */
+struct fwctl_info_cxl {
+ __u32 uctx_caps;
+};
+
+#endif
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (8 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 09/13] fwctl/cxl: Add support for get driver information Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-22 15:29 ` Jason Gunthorpe
2024-07-29 11:29 ` Jonathan Cameron
2024-07-18 21:32 ` [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc() Dave Jiang
` (5 subsequent siblings)
15 siblings, 2 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
fwctl provides a fwctl_ops->fw_rpc() callback in order to issue ioctls
to a device. The cxl fwctl driver will start by supporting the CXL
feature commands: Get Supported Features, Get Feature, and Set Feature.
The fw_rpc() callback provides 'enum fwctl_rpc_scope' parameter where
it indicates the security scope of the call. The Get Supported Features
and Get Feature calls can be executed with the scope of
FWCTL_RPC_DEBUG_READ_ONLY. The Set Feature call is gated by the effects
of the feature reported by Get Supported Features call for the specific
feature.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/core.h | 5 +-
drivers/cxl/core/mbox.c | 232 ++++++++++++++++++++++++++++++++----
drivers/cxl/core/memdev.c | 4 +-
drivers/cxl/cxlmem.h | 83 +------------
drivers/fwctl/cxl/cxl.c | 127 +++++++++++++++++++-
include/linux/cxl/mailbox.h | 101 ++++++++++++++++
include/uapi/fwctl/cxl.h | 64 ++++++++++
7 files changed, 505 insertions(+), 111 deletions(-)
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 7eb33ea07848..f22ee6f84f68 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -6,6 +6,8 @@
#include <linux/cxl/mailbox.h>
+extern struct rw_semaphore cxl_memdev_rwsem;
+
extern const struct device_type cxl_nvdimm_bridge_type;
extern const struct device_type cxl_nvdimm_type;
extern const struct device_type cxl_pmu_type;
@@ -69,7 +71,8 @@ struct cxl_send_command;
struct cxl_mem_query_commands;
int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mem_query_commands __user *q);
-int cxl_send_cmd(struct cxl_mailbox *cxl_mailbox, struct cxl_send_command __user *s);
+int cxl_send_cmd_from_user(struct cxl_mailbox *cxl_mbox,
+ struct cxl_send_command __user *s);
void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
resource_size_t length);
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 14ebe69c47bf..d77817e347d8 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -4,6 +4,7 @@
#include <linux/debugfs.h>
#include <linux/ktime.h>
#include <linux/mutex.h>
+#include <linux/cxl/mailbox.h>
#include <asm/unaligned.h>
#include <cxlpci.h>
#include <cxlmem.h>
@@ -215,6 +216,15 @@ static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
return NULL;
}
+struct cxl_mem_command *cxl_get_mem_command(u32 id)
+{
+ if (id > CXL_MEM_COMMAND_ID_MAX)
+ return NULL;
+
+ return &cxl_mem_commands[id];
+}
+EXPORT_SYMBOL_NS_GPL(cxl_get_mem_command, CXL);
+
static const char *cxl_mem_opcode_to_name(u16 opcode)
{
struct cxl_mem_command *c;
@@ -377,10 +387,13 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd,
}
/* Prepare to handle a full payload for variable sized output */
- if (out_size == CXL_VARIABLE_PAYLOAD)
- mbox_cmd->size_out = cxl_mbox->payload_size;
- else
+ if (out_size == CXL_VARIABLE_PAYLOAD) {
+ /* Adding extra 8 bytes for FWCTL, should not impact operation */
+ mbox_cmd->size_out = cxl_mbox->payload_size +
+ sizeof(struct fwctl_rpc_cxl_out);
+ } else {
mbox_cmd->size_out = out_size;
+ }
if (mbox_cmd->size_out) {
mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL);
@@ -477,6 +490,73 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
return 0;
}
+static int cxl_fwctl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
+ const struct cxl_send_command *send_cmd,
+ struct cxl_mailbox *cxl_mbox)
+{
+ struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
+ const struct cxl_command_info *info = &c->info;
+
+ if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
+ return -EINVAL;
+
+ if (send_cmd->rsvd)
+ return -EINVAL;
+
+ if (send_cmd->in.rsvd || send_cmd->out.rsvd)
+ return -EINVAL;
+
+ /* Check the input buffer is the expected size */
+ if (info->size_in != CXL_VARIABLE_PAYLOAD &&
+ info->size_in != send_cmd->in.size)
+ return -ENOMEM;
+
+ /* Check the output buffer is at least large enough */
+ if (info->size_out != CXL_VARIABLE_PAYLOAD &&
+ send_cmd->out.size < info->size_out)
+ return -ENOMEM;
+
+ *mem_cmd = (struct cxl_mem_command) {
+ .info = {
+ .id = info->id,
+ .flags = info->flags,
+ .size_in = send_cmd->in.size,
+ .size_out = send_cmd->out.size,
+ },
+ .opcode = c->opcode
+ };
+
+ return 0;
+}
+
+static int verify_send_command(const struct cxl_send_command *send_cmd,
+ struct cxl_mailbox *cxl_mbox)
+{
+ if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
+ return -ENOTTY;
+
+ /*
+ * The user can never specify an input payload larger than what hardware
+ * supports, but output can be arbitrarily large (simply write out as
+ * much data as the hardware provides).
+ */
+ if (send_cmd->in.size > cxl_mbox->payload_size)
+ return -EINVAL;
+
+ return 0;
+}
+
+/* Sanitize and construct a cxl_mbox_cmd */
+static int construct_mbox_cmd(struct cxl_mbox_cmd *mbox_cmd,
+ struct cxl_mem_command *mem_cmd,
+ struct cxl_mailbox *cxl_mbox,
+ const struct cxl_send_command *send_cmd)
+{
+ return cxl_mbox_cmd_ctor(mbox_cmd, cxl_mbox, mem_cmd->opcode,
+ mem_cmd->info.size_in, mem_cmd->info.size_out,
+ send_cmd->in.payload);
+}
+
/**
* cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
* @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
@@ -501,16 +581,9 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
struct cxl_mem_command mem_cmd;
int rc;
- if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
- return -ENOTTY;
-
- /*
- * The user can never specify an input payload larger than what hardware
- * supports, but output can be arbitrarily large (simply write out as
- * much data as the hardware provides).
- */
- if (send_cmd->in.size > cxl_mbox->payload_size)
- return -EINVAL;
+ rc = verify_send_command(send_cmd, cxl_mbox);
+ if (rc)
+ return rc;
/* Sanitize and construct a cxl_mem_command */
if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
@@ -521,10 +594,26 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
if (rc)
return rc;
- /* Sanitize and construct a cxl_mbox_cmd */
- return cxl_mbox_cmd_ctor(mbox_cmd, cxl_mbox, mem_cmd.opcode,
- mem_cmd.info.size_in, mem_cmd.info.size_out,
- send_cmd->in.payload);
+ return construct_mbox_cmd(mbox_cmd, &mem_cmd, cxl_mbox, send_cmd);
+}
+
+static int cxl_validate_cmd_from_fwctl(struct cxl_mbox_cmd *mbox_cmd,
+ struct cxl_mailbox *cxl_mbox,
+ const struct cxl_send_command *send_cmd)
+{
+ struct cxl_mem_command mem_cmd;
+ int rc;
+
+ rc = verify_send_command(send_cmd, cxl_mbox);
+ if (rc)
+ return rc;
+
+ /* Sanitize and construct a cxl_mem_command */
+ rc = cxl_fwctl_to_mem_cmd(&mem_cmd, send_cmd, cxl_mbox);
+ if (rc)
+ return rc;
+
+ return construct_mbox_cmd(mbox_cmd, &mem_cmd, cxl_mbox, send_cmd);
}
int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
@@ -631,7 +720,107 @@ static int handle_mailbox_cmd_from_user(struct cxl_mailbox *cxl_mbox,
return rc;
}
-int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command __user *s)
+static int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command *send,
+ struct cxl_mbox_cmd *mbox_cmd)
+{
+ int rc;
+
+ rc = cxl_validate_cmd_from_user(mbox_cmd, cxl_mbox, send);
+ if (rc)
+ return rc;
+
+ rc = handle_mailbox_cmd_from_user(cxl_mbox, mbox_cmd, send->out.payload,
+ &send->out.size, &send->retval);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+/**
+ * handle_mailbox_cmd_from_fwctl() - Dispatch a mailbox command for userspace.
+ * @mailbox: The mailbox context for the operation.
+ * @mbox_cmd: The validated mailbox command.
+ *
+ * Return:
+ * * %0 - Mailbox transaction succeeded. This implies the mailbox
+ * protocol completed successfully not that the operation itself
+ * was successful.
+ * * %-ENOMEM - Couldn't allocate a bounce buffer.
+ * * %-EINTR - Mailbox acquisition interrupted.
+ * * %-EXXX - Transaction level failures.
+ *
+ * Dispatches a mailbox command on behalf of a userspace request.
+ * The output payload is copied to userspace by fwctl.
+ *
+ * See cxl_send_cmd().
+ */
+static int handle_mailbox_cmd_from_fwctl(struct cxl_mailbox *cxl_mbox,
+ struct cxl_mbox_cmd *mbox_cmd)
+{
+ struct device *dev = cxl_mbox->host;
+ struct fwctl_rpc_cxl_out *orig_out;
+ int rc;
+
+ /*
+ * Save the payload_out pointer and move it to where hardware output
+ * can be copied to.
+ */
+ orig_out = mbox_cmd->payload_out;
+ mbox_cmd->payload_out = (void *)orig_out + sizeof(*orig_out);
+
+ dev_dbg(dev,
+ "Submitting %s command for user\n"
+ "\topcode: %x\n"
+ "\tsize: %zx\n",
+ cxl_mem_opcode_to_name(mbox_cmd->opcode),
+ mbox_cmd->opcode, mbox_cmd->size_in);
+
+ rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
+ if (rc)
+ return rc;
+
+ orig_out->retval = mbox_cmd->return_code;
+ mbox_cmd->payload_out = (void *)orig_out;
+
+ return 0;
+}
+
+int cxl_fwctl_send_cmd(struct cxl_mailbox *cxl_mbox,
+ struct fwctl_cxl_command *fwctl_cmd,
+ struct cxl_mbox_cmd *mbox_cmd, size_t *out_len)
+{
+ struct cxl_send_command send_cmd = {
+ .id = fwctl_cmd->id,
+ .flags = fwctl_cmd->flags,
+ .raw.opcode = fwctl_cmd->raw.opcode,
+ .in.size = fwctl_cmd->in.size,
+ .in.payload = fwctl_cmd->in.payload,
+ .out.size = *out_len,
+ };
+ int rc;
+
+ guard(rwsem_read)(&cxl_memdev_rwsem);
+ rc = cxl_validate_cmd_from_fwctl(mbox_cmd, cxl_mbox, &send_cmd);
+ if (rc)
+ return rc;
+
+ rc = handle_mailbox_cmd_from_fwctl(cxl_mbox, mbox_cmd);
+ if (rc)
+ return rc;
+
+ rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
+ if (rc)
+ return rc;
+
+ *out_len = mbox_cmd->size_out;
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_fwctl_send_cmd, CXL);
+
+int cxl_send_cmd_from_user(struct cxl_mailbox *cxl_mbox,
+ struct cxl_send_command __user *s)
{
struct device *dev = cxl_mbox->host;
struct cxl_send_command send;
@@ -643,12 +832,7 @@ int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command __user *s
if (copy_from_user(&send, s, sizeof(send)))
return -EFAULT;
- rc = cxl_validate_cmd_from_user(&mbox_cmd, cxl_mbox, &send);
- if (rc)
- return rc;
-
- rc = handle_mailbox_cmd_from_user(cxl_mbox, &mbox_cmd, send.out.payload,
- &send.out.size, &send.retval);
+ rc = cxl_send_cmd(cxl_mbox, &send, &mbox_cmd);
if (rc)
return rc;
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 0f6ec85ef9c4..f6f33f0f7337 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -11,7 +11,7 @@
#include "trace.h"
#include "core.h"
-static DECLARE_RWSEM(cxl_memdev_rwsem);
+DECLARE_RWSEM(cxl_memdev_rwsem);
#define CXL_ADEV_NAME "fwctl-cxl"
@@ -671,7 +671,7 @@ static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
case CXL_MEM_QUERY_COMMANDS:
return cxl_query_cmd(cxl_mbox, (void __user *)arg);
case CXL_MEM_SEND_COMMAND:
- return cxl_send_cmd(cxl_mbox, (void __user *)arg);
+ return cxl_send_cmd_from_user(cxl_mbox, (void __user *)arg);
default:
return -ENOTTY;
}
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 5e190f941e58..736111f5bc31 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -465,53 +465,6 @@ to_cxl_memdev_state(struct cxl_dev_state *cxlds)
return container_of(cxlds, struct cxl_memdev_state, cxlds);
}
-enum cxl_opcode {
- CXL_MBOX_OP_INVALID = 0x0000,
- CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
- CXL_MBOX_OP_GET_EVENT_RECORD = 0x0100,
- CXL_MBOX_OP_CLEAR_EVENT_RECORD = 0x0101,
- CXL_MBOX_OP_GET_EVT_INT_POLICY = 0x0102,
- CXL_MBOX_OP_SET_EVT_INT_POLICY = 0x0103,
- CXL_MBOX_OP_GET_FW_INFO = 0x0200,
- CXL_MBOX_OP_TRANSFER_FW = 0x0201,
- CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
- CXL_MBOX_OP_GET_TIMESTAMP = 0x0300,
- CXL_MBOX_OP_SET_TIMESTAMP = 0x0301,
- CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
- CXL_MBOX_OP_GET_LOG = 0x0401,
- CXL_MBOX_OP_GET_LOG_CAPS = 0x0402,
- CXL_MBOX_OP_CLEAR_LOG = 0x0403,
- CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
- CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
- CXL_MBOX_OP_GET_FEATURE = 0x0501,
- CXL_MBOX_OP_SET_FEATURE = 0x0502,
- CXL_MBOX_OP_IDENTIFY = 0x4000,
- CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
- CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
- CXL_MBOX_OP_GET_LSA = 0x4102,
- CXL_MBOX_OP_SET_LSA = 0x4103,
- CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200,
- CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201,
- CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202,
- CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203,
- CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204,
- CXL_MBOX_OP_GET_POISON = 0x4300,
- CXL_MBOX_OP_INJECT_POISON = 0x4301,
- CXL_MBOX_OP_CLEAR_POISON = 0x4302,
- CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303,
- CXL_MBOX_OP_SCAN_MEDIA = 0x4304,
- CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305,
- CXL_MBOX_OP_SANITIZE = 0x4400,
- CXL_MBOX_OP_SECURE_ERASE = 0x4401,
- CXL_MBOX_OP_GET_SECURITY_STATE = 0x4500,
- CXL_MBOX_OP_SET_PASSPHRASE = 0x4501,
- CXL_MBOX_OP_DISABLE_PASSPHRASE = 0x4502,
- CXL_MBOX_OP_UNLOCK = 0x4503,
- CXL_MBOX_OP_FREEZE_SECURITY = 0x4504,
- CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE = 0x4505,
- CXL_MBOX_OP_MAX = 0x10000
-};
-
#define DEFINE_CXL_CEL_UUID \
UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 0xb1, 0x62, \
0x3b, 0x3f, 0x17)
@@ -709,30 +662,6 @@ struct cxl_mbox_clear_poison {
u8 write_data[CXL_POISON_LEN_MULT];
} __packed;
-/**
- * struct cxl_mem_command - Driver representation of a memory device command
- * @info: Command information as it exists for the UAPI
- * @opcode: The actual bits used for the mailbox protocol
- * @flags: Set of flags effecting driver behavior.
- *
- * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag
- * will be enabled by the driver regardless of what hardware may have
- * advertised.
- *
- * The cxl_mem_command is the driver's internal representation of commands that
- * are supported by the driver. Some of these commands may not be supported by
- * the hardware. The driver will use @info to validate the fields passed in by
- * the user then submit the @opcode to the hardware.
- *
- * See struct cxl_command_info.
- */
-struct cxl_mem_command {
- struct cxl_command_info info;
- enum cxl_opcode opcode;
- u32 flags;
-#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0)
-};
-
#define CXL_PMEM_SEC_STATE_USER_PASS_SET 0x01
#define CXL_PMEM_SEC_STATE_MASTER_PASS_SET 0x02
#define CXL_PMEM_SEC_STATE_LOCKED 0x04
@@ -775,17 +704,7 @@ struct cxl_mbox_get_sup_feats_in {
__le16 reserved;
} __packed;
-struct cxl_feat_entry {
- uuid_t uuid;
- __le16 id;
- __le16 get_feat_size;
- __le16 set_feat_size;
- __le32 flags;
- u8 get_feat_ver;
- u8 set_feat_ver;
- __le16 effects;
- u8 reserved[18];
-} __packed;
+struct cxl_feat_entry;
struct cxl_mbox_get_sup_feats_out {
__le16 num_entries;
diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
index 22f62034c021..01f0771148e1 100644
--- a/drivers/fwctl/cxl/cxl.c
+++ b/drivers/fwctl/cxl/cxl.c
@@ -51,10 +51,133 @@ static void *cxlctl_info(struct fwctl_uctx *uctx, size_t *length)
return info;
}
+static bool cxlctl_validate_set_features(struct cxl_mailbox *cxl_mbox,
+ const struct fwctl_cxl_command *send_cmd,
+ enum fwctl_rpc_scope scope)
+{
+ struct cxl_feat_entry *feat;
+ bool found = false;
+ uuid_t uuid;
+ u16 mask;
+
+ if (send_cmd->in.size < sizeof(struct set_feature_input))
+ return false;
+
+ if (copy_from_user(&uuid, u64_to_user_ptr(send_cmd->in.payload),
+ sizeof(uuid)))
+ return false;
+
+ for (int i = 0; i < cxl_mbox->num_features; i++) {
+ feat = &cxl_mbox->entries[i];
+ if (uuid_equal(&uuid, &feat->uuid)) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found)
+ return false;
+
+ /* Currently no user background command support */
+ if (feat->effects & CXL_CMD_BACKGROUND)
+ return false;
+
+ mask = CXL_CMD_CONFIG_CHANGE_IMMEDIATE |
+ CXL_CMD_DATA_CHANGE_IMMEDIATE |
+ CXL_CMD_POLICY_CHANGE_IMMEDIATE |
+ CXL_CMD_LOG_CHANGE_IMMEDIATE;
+ if (feat->effects & mask && scope >= FWCTL_RPC_DEBUG_WRITE)
+ return true;
+
+ /* These effects supported for all scope */
+ if ((feat->effects & CXL_CMD_CONFIG_CHANGE_COLD_RESET ||
+ feat->effects & CXL_CMD_CONFIG_CHANGE_CONV_RESET) &&
+ scope >= FWCTL_RPC_DEBUG_READ_ONLY)
+ return true;
+
+ return false;
+}
+
+static bool cxlctl_validate_hw_cmds(struct cxl_mailbox *cxl_mbox,
+ const struct fwctl_cxl_command *send_cmd,
+ enum fwctl_rpc_scope scope)
+{
+ struct cxl_mem_command *cmd;
+
+ /*
+ * Only supporting feature commands.
+ */
+ if (!cxl_mbox->num_features)
+ return false;
+
+ cmd = cxl_get_mem_command(send_cmd->id);
+ if (!cmd)
+ return false;
+
+ if (test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
+ return false;
+
+ if (test_bit(cmd->info.id, cxl_mbox->exclusive_cmds))
+ return false;
+
+ switch (cmd->opcode) {
+ case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
+ case CXL_MBOX_OP_GET_FEATURE:
+ if (scope >= FWCTL_RPC_DEBUG_READ_ONLY)
+ return true;
+ break;
+ case CXL_MBOX_OP_SET_FEATURE:
+ return cxlctl_validate_set_features(cxl_mbox, send_cmd, scope);
+ default:
+ return false;
+ };
+
+ return false;
+}
+
+static bool cxlctl_validate_rpc(struct fwctl_uctx *uctx,
+ struct fwctl_rpc_cxl *rpc_in,
+ enum fwctl_rpc_scope scope)
+{
+ struct cxlctl_dev *cxlctl =
+ container_of(uctx->fwctl, struct cxlctl_dev, fwctl);
+
+ if (rpc_in->rpc_cmd != FWCTL_CXL_SEND_COMMAND)
+ return false;
+
+ return cxlctl_validate_hw_cmds(cxlctl->mbox, &rpc_in->send_cmd, scope);
+}
+
+static void *send_cxl_command(struct cxl_mailbox *cxl_mbox,
+ struct fwctl_cxl_command *send_cmd,
+ size_t *out_len)
+{
+ struct cxl_mbox_cmd mbox_cmd;
+ int rc;
+
+ rc = cxl_fwctl_send_cmd(cxl_mbox, send_cmd, &mbox_cmd, out_len);
+ if (rc)
+ return ERR_PTR(rc);
+
+ *out_len = mbox_cmd.size_out;
+
+ return mbox_cmd.payload_out;
+}
+
static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
- void *rpc_in, size_t in_len, size_t *out_len)
+ void *in, size_t in_len, size_t *out_len)
{
- /* Place holder */
+ struct cxlctl_dev *cxlctl =
+ container_of(uctx->fwctl, struct cxlctl_dev, fwctl);
+ struct cxl_mailbox *cxl_mbox = cxlctl->mbox;
+ struct fwctl_rpc_cxl *rpc_in = in;
+
+ if (!cxlctl_validate_rpc(uctx, rpc_in, scope))
+ return ERR_PTR(-EPERM);
+
+ if (rpc_in->rpc_cmd == FWCTL_CXL_SEND_COMMAND)
+ return send_cxl_command(cxl_mbox, &rpc_in->send_cmd, out_len);
+
return ERR_PTR(-EOPNOTSUPP);
}
diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
index 570864239b8f..b3c74f3da9a5 100644
--- a/include/linux/cxl/mailbox.h
+++ b/include/linux/cxl/mailbox.h
@@ -4,6 +4,7 @@
#define __CXL_MBOX_H__
#include <uapi/linux/cxl_mem.h>
+#include <uapi/fwctl/cxl.h>
#include <linux/auxiliary_bus.h>
/**
@@ -68,4 +69,104 @@ struct cxl_mailbox {
struct cxl_feat_entry *entries;
};
+enum cxl_opcode {
+ CXL_MBOX_OP_INVALID = 0x0000,
+ CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
+ CXL_MBOX_OP_GET_EVENT_RECORD = 0x0100,
+ CXL_MBOX_OP_CLEAR_EVENT_RECORD = 0x0101,
+ CXL_MBOX_OP_GET_EVT_INT_POLICY = 0x0102,
+ CXL_MBOX_OP_SET_EVT_INT_POLICY = 0x0103,
+ CXL_MBOX_OP_GET_FW_INFO = 0x0200,
+ CXL_MBOX_OP_TRANSFER_FW = 0x0201,
+ CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
+ CXL_MBOX_OP_GET_TIMESTAMP = 0x0300,
+ CXL_MBOX_OP_SET_TIMESTAMP = 0x0301,
+ CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
+ CXL_MBOX_OP_GET_LOG = 0x0401,
+ CXL_MBOX_OP_GET_LOG_CAPS = 0x0402,
+ CXL_MBOX_OP_CLEAR_LOG = 0x0403,
+ CXL_MBOX_OP_GET_SUP_LOG_SUBLIST = 0x0405,
+ CXL_MBOX_OP_GET_SUPPORTED_FEATURES = 0x0500,
+ CXL_MBOX_OP_GET_FEATURE = 0x0501,
+ CXL_MBOX_OP_SET_FEATURE = 0x0502,
+ CXL_MBOX_OP_IDENTIFY = 0x4000,
+ CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
+ CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
+ CXL_MBOX_OP_GET_LSA = 0x4102,
+ CXL_MBOX_OP_SET_LSA = 0x4103,
+ CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200,
+ CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201,
+ CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202,
+ CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203,
+ CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204,
+ CXL_MBOX_OP_GET_POISON = 0x4300,
+ CXL_MBOX_OP_INJECT_POISON = 0x4301,
+ CXL_MBOX_OP_CLEAR_POISON = 0x4302,
+ CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303,
+ CXL_MBOX_OP_SCAN_MEDIA = 0x4304,
+ CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305,
+ CXL_MBOX_OP_SANITIZE = 0x4400,
+ CXL_MBOX_OP_SECURE_ERASE = 0x4401,
+ CXL_MBOX_OP_GET_SECURITY_STATE = 0x4500,
+ CXL_MBOX_OP_SET_PASSPHRASE = 0x4501,
+ CXL_MBOX_OP_DISABLE_PASSPHRASE = 0x4502,
+ CXL_MBOX_OP_UNLOCK = 0x4503,
+ CXL_MBOX_OP_FREEZE_SECURITY = 0x4504,
+ CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE = 0x4505,
+ CXL_MBOX_OP_MAX = 0x10000
+};
+
+#define CXL_CMD_CONFIG_CHANGE_COLD_RESET BIT(0)
+#define CXL_CMD_CONFIG_CHANGE_IMMEDIATE BIT(1)
+#define CXL_CMD_DATA_CHANGE_IMMEDIATE BIT(2)
+#define CXL_CMD_POLICY_CHANGE_IMMEDIATE BIT(3)
+#define CXL_CMD_LOG_CHANGE_IMMEDIATE BIT(4)
+#define CXL_CMD_SECURITY_STATE_CHANGE BIT(5)
+#define CXL_CMD_BACKGROUND BIT(6)
+#define CXL_CMD_BGCMD_ABORT_SUPPORTED BIT(7)
+#define CXL_CMD_CONFIG_CHANGE_CONV_RESET (BIT(9) | BIT(10))
+#define CXL_CMD_CONFIG_CHANGE_CXL_RESET (BIT(9) | BIT(11))
+
+struct cxl_feat_entry {
+ uuid_t uuid;
+ __le16 id;
+ __le16 get_feat_size;
+ __le16 set_feat_size;
+ __le32 flags;
+ u8 get_feat_ver;
+ u8 set_feat_ver;
+ __le16 effects;
+ u8 reserved[18];
+} __packed;
+
+/**
+ * struct cxl_mem_command - Driver representation of a memory device command
+ * @info: Command information as it exists for the UAPI
+ * @opcode: The actual bits used for the mailbox protocol
+ * @flags: Set of flags effecting driver behavior.
+ *
+ * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag
+ * will be enabled by the driver regardless of what hardware may have
+ * advertised.
+ *
+ * The cxl_mem_command is the driver's internal representation of commands that
+ * are supported by the driver. Some of these commands may not be supported by
+ * the hardware. The driver will use @info to validate the fields passed in by
+ * the user then submit the @opcode to the hardware.
+ *
+ * See struct cxl_command_info.
+ */
+struct cxl_mem_command {
+ struct cxl_command_info info;
+ enum cxl_opcode opcode;
+ u32 flags;
+#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0)
+};
+
+struct cxl_mem_command *cxl_get_mem_command(u32 id);
+int cxl_fwctl_send_cmd(struct cxl_mailbox *cxl_mbox,
+ struct fwctl_cxl_command *fwctl_cmd,
+ struct cxl_mbox_cmd *mbox_cmd,
+ size_t *out_len);
+
#endif
diff --git a/include/uapi/fwctl/cxl.h b/include/uapi/fwctl/cxl.h
index d3a735f1fe4e..de8949a28473 100644
--- a/include/uapi/fwctl/cxl.h
+++ b/include/uapi/fwctl/cxl.h
@@ -22,4 +22,68 @@ struct fwctl_info_cxl {
__u32 uctx_caps;
};
+/*
+ * CXL spec r3.1 Table 8-101 Set Feature Input Payload
+ */
+struct set_feature_input {
+ uuid_t uuid;
+ __u32 flags;
+ __u16 offset;
+ __u8 version;
+ __u8 reserved[9];
+ __u8 data[];
+} __packed;
+
+/**
+ * struct cxl_send_command - Send a command to a memory device.
+ * @id: The command to send to the memory device. This must be one of the
+ * commands returned by the query command.
+ * @flags: Flags for the command (input).
+ * @raw: Special fields for raw commands
+ * @raw.opcode: Opcode passed to hardware when using the RAW command.
+ * @raw.rsvd: Must be zero.
+ * @rsvd: Must be zero.
+ * @retval: Return value from the memory device (output).
+ * @in: Parameters associated with input payload.
+ * @in.size: Size of the payload to provide to the device (input).
+ * @in.rsvd: Must be zero.
+ * @in.payload: Pointer to memory for payload input, payload is little endian.
+ *
+ * Output payload is defined with 'struct fwctl_rpc' and is the hardware output
+ */
+struct fwctl_cxl_command {
+ __u32 id;
+ __u32 flags;
+ union {
+ struct {
+ __u16 opcode;
+ __u16 rsvd;
+ } raw;
+ __u32 rsvd;
+ };
+
+ struct {
+ __u32 size;
+ __u32 rsvd;
+ __u64 payload;
+ } in;
+};
+
+/**
+ * struct fwctl_rpc_cxl - ioctl(FWCTL_RPC) input
+ */
+struct fwctl_rpc_cxl {
+ __u32 rpc_cmd;
+ __u32 payload_size;
+ __u32 version;
+ __u32 rsvd;
+ struct fwctl_cxl_command send_cmd;
+};
+
+struct fwctl_rpc_cxl_out {
+ __u32 retval;
+ __u32 rsvd;
+ __u8 payload[];
+};
+
#endif
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-07-18 21:32 ` [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands Dave Jiang
@ 2024-07-22 15:29 ` Jason Gunthorpe
2024-07-22 22:52 ` Dave Jiang
2024-07-29 11:29 ` Jonathan Cameron
1 sibling, 1 reply; 47+ messages in thread
From: Jason Gunthorpe @ 2024-07-22 15:29 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On Thu, Jul 18, 2024 at 02:32:28PM -0700, Dave Jiang wrote:
> diff --git a/include/uapi/fwctl/cxl.h b/include/uapi/fwctl/cxl.h
> index d3a735f1fe4e..de8949a28473 100644
> --- a/include/uapi/fwctl/cxl.h
> +++ b/include/uapi/fwctl/cxl.h
> @@ -22,4 +22,68 @@ struct fwctl_info_cxl {
> __u32 uctx_caps;
> };
>
> +/*
> + * CXL spec r3.1 Table 8-101 Set Feature Input Payload
> + */
> +struct set_feature_input {
> + uuid_t uuid;
> + __u32 flags;
> + __u16 offset;
> + __u8 version;
> + __u8 reserved[9];
> + __u8 data[];
> +} __packed;
This struct probably needs to be prefixed with fwcl_, but should it be
here I wonder or should the CXL defined structs be someplace else?
> +/**
> + * struct cxl_send_command - Send a command to a memory device.
> + * @id: The command to send to the memory device. This must be one of the
> + * commands returned by the query command.
> + * @flags: Flags for the command (input).
> + * @raw: Special fields for raw commands
> + * @raw.opcode: Opcode passed to hardware when using the RAW command.
> + * @raw.rsvd: Must be zero.
> + * @rsvd: Must be zero.
> + * @retval: Return value from the memory device (output).
> + * @in: Parameters associated with input payload.
> + * @in.size: Size of the payload to provide to the device (input).
> + * @in.rsvd: Must be zero.
> + * @in.payload: Pointer to memory for payload input, payload is little endian.
> + *
> + * Output payload is defined with 'struct fwctl_rpc' and is the hardware output
> + */
> +struct fwctl_cxl_command {
> + __u32 id;
> + __u32 flags;
> + union {
> + struct {
> + __u16 opcode;
> + __u16 rsvd;
> + } raw;
> + __u32 rsvd;
> + };
> +
> + struct {
> + __u32 size;
> + __u32 rsvd;
> + __u64 payload;
> + } in;
> +};
Why all the structs and unions?
> +/**
> + * struct fwctl_rpc_cxl - ioctl(FWCTL_RPC) input
> + */
> +struct fwctl_rpc_cxl {
> + __u32 rpc_cmd;
> + __u32 payload_size;
> + __u32 version;
> + __u32 rsvd;
> + struct fwctl_cxl_command send_cmd;
> +};
> +
> +struct fwctl_rpc_cxl_out {
> + __u32 retval;
> + __u32 rsvd;
> + __u8 payload[];
This might be better as
__aligned_u64 payload[]
Which will force a 8 byte alignment of the struct which will allow the
payload to be casted to aligned u64 reliably.
How do you know the length of payload?
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-07-22 15:29 ` Jason Gunthorpe
@ 2024-07-22 22:52 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-22 22:52 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On 7/22/24 8:29 AM, Jason Gunthorpe wrote:
> On Thu, Jul 18, 2024 at 02:32:28PM -0700, Dave Jiang wrote:
>> diff --git a/include/uapi/fwctl/cxl.h b/include/uapi/fwctl/cxl.h
>> index d3a735f1fe4e..de8949a28473 100644
>> --- a/include/uapi/fwctl/cxl.h
>> +++ b/include/uapi/fwctl/cxl.h
>> @@ -22,4 +22,68 @@ struct fwctl_info_cxl {
>> __u32 uctx_caps;
>> };
>>
>> +/*
>> + * CXL spec r3.1 Table 8-101 Set Feature Input Payload
>> + */
>> +struct set_feature_input {
>> + uuid_t uuid;
>> + __u32 flags;
>> + __u16 offset;
>> + __u8 version;
>> + __u8 reserved[9];
>> + __u8 data[];
>> +} __packed;
>
> This struct probably needs to be prefixed with fwcl_, but should it be
> here I wonder or should the CXL defined structs be someplace else?
>
It's the "set feature" mailbox input for user to fill out. So probably should be in uapi.
>> +/**
>> + * struct cxl_send_command - Send a command to a memory device.
>> + * @id: The command to send to the memory device. This must be one of the
>> + * commands returned by the query command.
>> + * @flags: Flags for the command (input).
>> + * @raw: Special fields for raw commands
>> + * @raw.opcode: Opcode passed to hardware when using the RAW command.
>> + * @raw.rsvd: Must be zero.
>> + * @rsvd: Must be zero.
>> + * @retval: Return value from the memory device (output).
>> + * @in: Parameters associated with input payload.
>> + * @in.size: Size of the payload to provide to the device (input).
>> + * @in.rsvd: Must be zero.
>> + * @in.payload: Pointer to memory for payload input, payload is little endian.
>> + *
>> + * Output payload is defined with 'struct fwctl_rpc' and is the hardware output
>> + */
>> +struct fwctl_cxl_command {
>> + __u32 id;
>> + __u32 flags;
>> + union {
>> + struct {
>> + __u16 opcode;
>> + __u16 rsvd;
>> + } raw;
>> + __u32 rsvd;
>> + };
>> +
>> + struct {
>> + __u32 size;
>> + __u32 rsvd;
>> + __u64 payload;
>> + } in;
>> +};
>
> Why all the structs and unions?
It was copied from 'struct cxl_send_command' for the current ioctl command. Will rework that.
>
>> +/**
>> + * struct fwctl_rpc_cxl - ioctl(FWCTL_RPC) input
>> + */
>> +struct fwctl_rpc_cxl {
>> + __u32 rpc_cmd;
>> + __u32 payload_size;
>> + __u32 version;
>> + __u32 rsvd;
>> + struct fwctl_cxl_command send_cmd;
>> +};
>> +
>> +struct fwctl_rpc_cxl_out {
>> + __u32 retval;
>> + __u32 rsvd;
>
>> + __u8 payload[];
>
> This might be better as
>
> __aligned_u64 payload[]
>
> Which will force a 8 byte alignment of the struct which will allow the
> payload to be casted to aligned u64 reliably.
ok will change.
>
> How do you know the length of payload?
Yes will change the rsvd field to size.
>
> Jason
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-07-18 21:32 ` [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands Dave Jiang
2024-07-22 15:29 ` Jason Gunthorpe
@ 2024-07-29 11:29 ` Jonathan Cameron
2024-11-13 15:41 ` Dave Jiang
1 sibling, 1 reply; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-29 11:29 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:28 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> fwctl provides a fwctl_ops->fw_rpc() callback in order to issue ioctls
> to a device. The cxl fwctl driver will start by supporting the CXL
> feature commands: Get Supported Features, Get Feature, and Set Feature.
I'll come back to this in reply to the cover letter, but this is
problematic because some of those features will almost certainly be
covered by standard kernel drivers and if Set Feature is enabled
those drivers will need to be hardened against the state mysteriously
changing under them.
This corner is the bit that worries me most about fwctl in general.
Anyhow, one for the cover letter / policy discussion not deep
in the patch series.
>
> The fw_rpc() callback provides 'enum fwctl_rpc_scope' parameter where
> it indicates the security scope of the call. The Get Supported Features
> and Get Feature calls can be executed with the scope of
> FWCTL_RPC_DEBUG_READ_ONLY. The Set Feature call is gated by the effects
> of the feature reported by Get Supported Features call for the specific
> feature.
Break the moves of error codes etc out as a precursor no-op patch to
make review of the real guts of this easier.
Trivial comment inline.
> diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
> index 22f62034c021..01f0771148e1 100644
> --- a/drivers/fwctl/cxl/cxl.c
> +++ b/drivers/fwctl/cxl/cxl.c
> @@ -51,10 +51,133 @@ static void *cxlctl_info(struct fwctl_uctx *uctx, size_t *length)
> return info;
> }
> +static bool cxlctl_validate_hw_cmds(struct cxl_mailbox *cxl_mbox,
> + const struct fwctl_cxl_command *send_cmd,
> + enum fwctl_rpc_scope scope)
> +{
> + struct cxl_mem_command *cmd;
> +
> + /*
> + * Only supporting feature commands.
> + */
> + if (!cxl_mbox->num_features)
> + return false;
> +
> + cmd = cxl_get_mem_command(send_cmd->id);
> + if (!cmd)
> + return false;
> +
> + if (test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
> + return false;
> +
> + if (test_bit(cmd->info.id, cxl_mbox->exclusive_cmds))
> + return false;
> +
> + switch (cmd->opcode) {
> + case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
> + case CXL_MBOX_OP_GET_FEATURE:
> + if (scope >= FWCTL_RPC_DEBUG_READ_ONLY)
> + return true;
> + break;
return false here.
> + case CXL_MBOX_OP_SET_FEATURE:
> + return cxlctl_validate_set_features(cxl_mbox, send_cmd, scope);
> + default:
> + return false;
> + };
> +
> + return false;
And drop this.
> +}
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-07-29 11:29 ` Jonathan Cameron
@ 2024-11-13 15:41 ` Dave Jiang
2024-11-19 11:41 ` Jonathan Cameron
0 siblings, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-11-13 15:41 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On 7/29/24 4:29 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:28 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> fwctl provides a fwctl_ops->fw_rpc() callback in order to issue ioctls
>> to a device. The cxl fwctl driver will start by supporting the CXL
>> feature commands: Get Supported Features, Get Feature, and Set Feature.
>
> I'll come back to this in reply to the cover letter, but this is
> problematic because some of those features will almost certainly be
> covered by standard kernel drivers and if Set Feature is enabled
> those drivers will need to be hardened against the state mysteriously
> changing under them.
Do we need to have an exclusive_features mask just like exclusive_cmds to block kernel managed features like RAS? That may be the best way to deal with things happening behind kernel's back.
>
> This corner is the bit that worries me most about fwctl in general.
> Anyhow, one for the cover letter / policy discussion not deep
> in the patch series.
Need to add a documentation patch and maybe we can put policies and rules in there?
>
>
>>
>> The fw_rpc() callback provides 'enum fwctl_rpc_scope' parameter where
>> it indicates the security scope of the call. The Get Supported Features
>> and Get Feature calls can be executed with the scope of
>> FWCTL_RPC_DEBUG_READ_ONLY. The Set Feature call is gated by the effects
>> of the feature reported by Get Supported Features call for the specific
>> feature.
>
> Break the moves of error codes etc out as a precursor no-op patch to
> make review of the real guts of this easier.
ok
>
> Trivial comment inline.
>> diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
>> index 22f62034c021..01f0771148e1 100644
>> --- a/drivers/fwctl/cxl/cxl.c
>> +++ b/drivers/fwctl/cxl/cxl.c
>> @@ -51,10 +51,133 @@ static void *cxlctl_info(struct fwctl_uctx *uctx, size_t *length)
>> return info;
>> }
>
>> +static bool cxlctl_validate_hw_cmds(struct cxl_mailbox *cxl_mbox,
>> + const struct fwctl_cxl_command *send_cmd,
>> + enum fwctl_rpc_scope scope)
>> +{
>> + struct cxl_mem_command *cmd;
>> +
>> + /*
>> + * Only supporting feature commands.
>> + */
>> + if (!cxl_mbox->num_features)
>> + return false;
>> +
>> + cmd = cxl_get_mem_command(send_cmd->id);
>> + if (!cmd)
>> + return false;
>> +
>> + if (test_bit(cmd->info.id, cxl_mbox->enabled_cmds))
>> + return false;
>> +
>> + if (test_bit(cmd->info.id, cxl_mbox->exclusive_cmds))
>> + return false;
>> +
>> + switch (cmd->opcode) {
>> + case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
>> + case CXL_MBOX_OP_GET_FEATURE:
>> + if (scope >= FWCTL_RPC_DEBUG_READ_ONLY)
>> + return true;
>> + break;
>
> return false here.
ok
>
>> + case CXL_MBOX_OP_SET_FEATURE:
>> + return cxlctl_validate_set_features(cxl_mbox, send_cmd, scope);
>> + default:
>> + return false;
>> + };
>> +
>> + return false;
>
> And drop this.
ok
>
>> +}
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
2024-11-13 15:41 ` Dave Jiang
@ 2024-11-19 11:41 ` Jonathan Cameron
0 siblings, 0 replies; 47+ messages in thread
From: Jonathan Cameron @ 2024-11-19 11:41 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Wed, 13 Nov 2024 08:41:15 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> On 7/29/24 4:29 AM, Jonathan Cameron wrote:
> > On Thu, 18 Jul 2024 14:32:28 -0700
> > Dave Jiang <dave.jiang@intel.com> wrote:
> >
> >> fwctl provides a fwctl_ops->fw_rpc() callback in order to issue ioctls
> >> to a device. The cxl fwctl driver will start by supporting the CXL
> >> feature commands: Get Supported Features, Get Feature, and Set Feature.
> >
> > I'll come back to this in reply to the cover letter, but this is
> > problematic because some of those features will almost certainly be
> > covered by standard kernel drivers and if Set Feature is enabled
> > those drivers will need to be hardened against the state mysteriously
> > changing under them.
>
> Do we need to have an exclusive_features mask just like exclusive_cmds to block kernel managed features like RAS? That may be the best way to deal with things happening behind kernel's back.
Agreed. Something along those lines should work.
My understandin is we are fine 'breaking' backwards compatibility in
FWCTL interfaces if we decide later that something isn't safe for it to access.
Mind you we can probably make a reasonable starting guess + restrict anything
in the non-vendor ranges that isn't yet defined.
Things get messier for defacto standards if any of them surface as we may
have kernel support upstream for those in the future but not initially.
>
> >
> > This corner is the bit that worries me most about fwctl in general.
> > Anyhow, one for the cover letter / policy discussion not deep
> > in the patch series.
>
> Need to add a documentation patch and maybe we can put policies and rules in there?
Even better.
Jonathan
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc()
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (9 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 10/13] fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-22 15:24 ` Jason Gunthorpe
2024-07-29 11:48 ` Jonathan Cameron
2024-07-18 21:32 ` [RFC PATCH 12/13] cxl/test: Add Get Feature support to cxl_test Dave Jiang
` (4 subsequent siblings)
15 siblings, 2 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add a software command through the ->fw_rpc() in order for the user to
retrieve information about the commands that are supported. In this
instance only 3 commands are supported: Get Supported Features, Get
Feature, and Set Feature.
The expected flow of operation is to send the call first with 0 set
to the n_commands parameter to indicate query of total commands
available. And then a second call provides the number of commands
to retrieve with the appropriate amount of memory allocated to store
information about the commands.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
drivers/cxl/core/mbox.c | 80 +++++++++++++++++++++++++++++++++++++
drivers/fwctl/cxl/cxl.c | 42 ++++++++++++++++---
include/linux/cxl/mailbox.h | 3 ++
include/uapi/fwctl/cxl.h | 5 ++-
4 files changed, 124 insertions(+), 6 deletions(-)
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index d77817e347d8..91be8cfbc2bb 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -655,6 +655,86 @@ int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
return 0;
}
+static bool fwctl_supported_commands(u16 opcode)
+{
+ switch (opcode) {
+ case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
+ case CXL_MBOX_OP_GET_FEATURE:
+ case CXL_MBOX_OP_SET_FEATURE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static int get_fwctl_supported_commands(void)
+{
+ struct cxl_mem_command *cmd;
+ int cmds = 0;
+
+ cxl_for_each_cmd(cmd) {
+ if (fwctl_supported_commands(cmd->opcode))
+ cmds++;
+ }
+
+ return cmds;
+}
+
+void *cxl_query_cmd_from_fwctl(struct cxl_mailbox *cxl_mbox,
+ struct cxl_mem_query_commands *q,
+ size_t *out_len)
+{
+ u32 n_commands = q->n_commands;
+ struct cxl_mem_command *cmd;
+ size_t output_size;
+ int commands;
+ int j = 0;
+
+ commands = get_fwctl_supported_commands();
+ if (n_commands > commands)
+ return ERR_PTR(-EINVAL);
+
+ output_size = sizeof(struct cxl_mem_query_commands) +
+ n_commands * sizeof(struct cxl_command_info);
+
+ struct cxl_mem_query_commands *qout __free(kvfree) =
+ kvzalloc(output_size, GFP_KERNEL);
+
+ if (!qout)
+ return ERR_PTR(-ENOMEM);
+
+ *out_len = output_size;
+ if (n_commands == 0) {
+ qout->n_commands = commands;
+ return no_free_ptr(qout);
+ }
+
+ qout->n_commands = n_commands;
+
+ /*
+ * otherwise, return min(n_commands, total commands) cxl_command_info
+ * structures.
+ */
+ cxl_for_each_cmd(cmd) {
+ struct cxl_command_info info = cmd->info;
+
+ if (fwctl_supported_commands(cmd->opcode)) {
+ if (test_bit(info.id, cxl_mbox->enabled_cmds))
+ info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
+ if (test_bit(info.id, cxl_mbox->exclusive_cmds))
+ info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
+
+ memcpy(&qout->commands[j++], &info, sizeof(info));
+ }
+
+ if (j == n_commands)
+ break;
+ }
+
+ return no_free_ptr(qout);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_query_cmd_from_fwctl, CXL);
+
/**
* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
* @mailbox: The mailbox context for the operation.
diff --git a/drivers/fwctl/cxl/cxl.c b/drivers/fwctl/cxl/cxl.c
index 01f0771148e1..8836a806763f 100644
--- a/drivers/fwctl/cxl/cxl.c
+++ b/drivers/fwctl/cxl/cxl.c
@@ -135,6 +135,24 @@ static bool cxlctl_validate_hw_cmds(struct cxl_mailbox *cxl_mbox,
return false;
}
+static bool cxlctl_validate_query_commands(struct fwctl_rpc_cxl *rpc_in)
+{
+ int cmds;
+
+ if (rpc_in->payload_size < sizeof(rpc_in->query))
+ return false;
+
+ cmds = rpc_in->query.n_commands;
+ if (cmds) {
+ int cmds_size = rpc_in->payload_size - sizeof(rpc_in->query);
+
+ if (cmds != cmds_size / sizeof(struct cxl_command_info))
+ return false;
+ }
+
+ return true;
+}
+
static bool cxlctl_validate_rpc(struct fwctl_uctx *uctx,
struct fwctl_rpc_cxl *rpc_in,
enum fwctl_rpc_scope scope)
@@ -142,10 +160,17 @@ static bool cxlctl_validate_rpc(struct fwctl_uctx *uctx,
struct cxlctl_dev *cxlctl =
container_of(uctx->fwctl, struct cxlctl_dev, fwctl);
- if (rpc_in->rpc_cmd != FWCTL_CXL_SEND_COMMAND)
+ switch (rpc_in->rpc_cmd) {
+ case FWCTL_CXL_QUERY_COMMANDS:
+ return cxlctl_validate_query_commands(rpc_in);
+
+ case FWCTL_CXL_SEND_COMMAND:
+ return cxlctl_validate_hw_cmds(cxlctl->mbox,
+ &rpc_in->send_cmd, scope);
+
+ default:
return false;
-
- return cxlctl_validate_hw_cmds(cxlctl->mbox, &rpc_in->send_cmd, scope);
+ }
}
static void *send_cxl_command(struct cxl_mailbox *cxl_mbox,
@@ -175,10 +200,17 @@ static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
if (!cxlctl_validate_rpc(uctx, rpc_in, scope))
return ERR_PTR(-EPERM);
- if (rpc_in->rpc_cmd == FWCTL_CXL_SEND_COMMAND)
+ switch (rpc_in->rpc_cmd) {
+ case FWCTL_CXL_QUERY_COMMANDS:
+ return cxl_query_cmd_from_fwctl(cxl_mbox, &rpc_in->query,
+ out_len);
+
+ case FWCTL_CXL_SEND_COMMAND:
return send_cxl_command(cxl_mbox, &rpc_in->send_cmd, out_len);
- return ERR_PTR(-EOPNOTSUPP);
+ default:
+ return ERR_PTR(-EOPNOTSUPP);
+ }
}
static const struct fwctl_ops cxlctl_ops = {
diff --git a/include/linux/cxl/mailbox.h b/include/linux/cxl/mailbox.h
index b3c74f3da9a5..13b5bb6e5bc3 100644
--- a/include/linux/cxl/mailbox.h
+++ b/include/linux/cxl/mailbox.h
@@ -168,5 +168,8 @@ int cxl_fwctl_send_cmd(struct cxl_mailbox *cxl_mbox,
struct fwctl_cxl_command *fwctl_cmd,
struct cxl_mbox_cmd *mbox_cmd,
size_t *out_len);
+void *cxl_query_cmd_from_fwctl(struct cxl_mailbox *cxl_mbox,
+ struct cxl_mem_query_commands *q,
+ size_t *out_len);
#endif
diff --git a/include/uapi/fwctl/cxl.h b/include/uapi/fwctl/cxl.h
index de8949a28473..a439f497b889 100644
--- a/include/uapi/fwctl/cxl.h
+++ b/include/uapi/fwctl/cxl.h
@@ -77,7 +77,10 @@ struct fwctl_rpc_cxl {
__u32 payload_size;
__u32 version;
__u32 rsvd;
- struct fwctl_cxl_command send_cmd;
+ union {
+ struct cxl_mem_query_commands query;
+ struct fwctl_cxl_command send_cmd;
+ };
};
struct fwctl_rpc_cxl_out {
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc()
2024-07-18 21:32 ` [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc() Dave Jiang
@ 2024-07-22 15:24 ` Jason Gunthorpe
2024-07-22 23:23 ` Dave Jiang
2024-07-29 11:48 ` Jonathan Cameron
1 sibling, 1 reply; 47+ messages in thread
From: Jason Gunthorpe @ 2024-07-22 15:24 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On Thu, Jul 18, 2024 at 02:32:29PM -0700, Dave Jiang wrote:
> Add a software command through the ->fw_rpc() in order for the user to
> retrieve information about the commands that are supported. In this
> instance only 3 commands are supported: Get Supported Features, Get
> Feature, and Set Feature.
I wonder if we should try to have a seperate API for this, knowing
what command numbers are supported seems sort of general to me?
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc()
2024-07-22 15:24 ` Jason Gunthorpe
@ 2024-07-22 23:23 ` Dave Jiang
2024-08-08 12:56 ` Jason Gunthorpe
0 siblings, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-07-22 23:23 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On 7/22/24 8:24 AM, Jason Gunthorpe wrote:
> On Thu, Jul 18, 2024 at 02:32:29PM -0700, Dave Jiang wrote:
>> Add a software command through the ->fw_rpc() in order for the user to
>> retrieve information about the commands that are supported. In this
>> instance only 3 commands are supported: Get Supported Features, Get
>> Feature, and Set Feature.
>
> I wonder if we should try to have a seperate API for this, knowing
> what command numbers are supported seems sort of general to me?
Probably a reasonable thing to add. Will you add something in the fwctl v3 or should I try to add something in my series for fwctl?
Wonder if we should also return some sort of fwctl commonly defined effects field for each command.
>
> Jason
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc()
2024-07-22 23:23 ` Dave Jiang
@ 2024-08-08 12:56 ` Jason Gunthorpe
0 siblings, 0 replies; 47+ messages in thread
From: Jason Gunthorpe @ 2024-08-08 12:56 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On Mon, Jul 22, 2024 at 04:23:14PM -0700, Dave Jiang wrote:
>
>
> On 7/22/24 8:24 AM, Jason Gunthorpe wrote:
> > On Thu, Jul 18, 2024 at 02:32:29PM -0700, Dave Jiang wrote:
> >> Add a software command through the ->fw_rpc() in order for the user to
> >> retrieve information about the commands that are supported. In this
> >> instance only 3 commands are supported: Get Supported Features, Get
> >> Feature, and Set Feature.
> >
> > I wonder if we should try to have a seperate API for this, knowing
> > what command numbers are supported seems sort of general to me?
>
> Probably a reasonable thing to add. Will you add something in the
> fwctl v3 or should I try to add something in my series for fwctl?
>
> Wonder if we should also return some sort of fwctl commonly defined
> effects field for each command.
I'm not sure I fully understand what CXL would like here, including
Jonathan's remakrs too. Can you give some short draft?
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc()
2024-07-18 21:32 ` [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc() Dave Jiang
2024-07-22 15:24 ` Jason Gunthorpe
@ 2024-07-29 11:48 ` Jonathan Cameron
1 sibling, 0 replies; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-29 11:48 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose
On Thu, 18 Jul 2024 14:32:29 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> Add a software command through the ->fw_rpc() in order for the user to
> retrieve information about the commands that are supported. In this
> instance only 3 commands are supported: Get Supported Features, Get
> Feature, and Set Feature.
>
> The expected flow of operation is to send the call first with 0 set
> to the n_commands parameter to indicate query of total commands
> available. And then a second call provides the number of commands
> to retrieve with the appropriate amount of memory allocated to store
> information about the commands.
We will need another layer of querying to query what features
are supported for each type of access. Get is probably always reasonable
in some cases we may want to block Set even with taint as it may provide
a nasty footgun.
Jonathan
^ permalink raw reply [flat|nested] 47+ messages in thread
* [RFC PATCH 12/13] cxl/test: Add Get Feature support to cxl_test
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (10 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 11/13] fwctl/cxl: Add query commands software command for ->fw_rpc() Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-18 21:32 ` [RFC PATCH 13/13] cxl/test: Add Set " Dave Jiang
` (3 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add emulation of Get Feature command to cxl_test. The feature for
device patrol scrub is returned by the emulation code. This is
the only feature currently supported by cxl_test. It returns
the information for the device patrol scrub feature.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
tools/testing/cxl/test/mem.c | 107 +++++++++++++++++++++++++++++++++++
1 file changed, 107 insertions(+)
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index c2e04dbc8af6..c779640cb610 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -48,6 +48,10 @@ static struct cxl_cel_entry mock_cel[] = {
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_SUPPORTED_FEATURES),
.effect = CXL_CMD_EFFECT_NONE,
},
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_GET_FEATURE),
+ .effect = CXL_CMD_EFFECT_NONE,
+ },
{
.opcode = cpu_to_le16(CXL_MBOX_OP_IDENTIFY),
.effect = CXL_CMD_EFFECT_NONE,
@@ -149,6 +153,16 @@ struct mock_event_store {
u32 ev_status;
};
+struct scrub_patrol_read_feat {
+ u8 cap;
+#define SCRUB_HOURS_CYCLE_DEFAULT 6
+#define SCRUB_HOURS_CYCLE_MASK GENMASK(7, 0)
+#define SCRUB_SMALLEST_HOURS_DEFAULT 2
+#define SCRUB_SMALLEST_HOURS_MASK GENMASK(15, 8)
+ __le16 cycle_support;
+ u8 flags;
+} __packed;
+
struct cxl_mockmem_data {
void *lsa;
void *fw;
@@ -165,6 +179,7 @@ struct cxl_mockmem_data {
u8 event_buf[SZ_4K];
u64 timestamp;
unsigned long sanitize_timeout;
+ struct scrub_patrol_read_feat patrol_feat;
};
static struct mock_event_log *event_find_log(struct device *dev, int log_type)
@@ -1354,6 +1369,82 @@ static void fill_feature_device_patrol_scrub_control(struct cxl_feat_entry *feat
feat->effects = cpu_to_le16(0x202);
}
+struct get_feat_input {
+ uuid_t uuid;
+ __le16 offset;
+ __le16 count;
+ u8 selection;
+} __packed;
+
+enum scrub_patrol_selection {
+ SCRUB_PATROL_CURRENT_VALUE = 0,
+ SCRUB_PATROL_DEFAULT_VALUE = 1,
+ SCRUB_PATROL_SAVED_VALUE = 2,
+};
+
+static int mock_get_patrol_scrub(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct scrub_patrol_read_feat *output = cmd->payload_out;
+ struct get_feat_input *input = cmd->payload_in;
+ struct scrub_patrol_read_feat feat = { 0 };
+ u16 offset = le16_to_cpu(input->offset);
+ u16 count = le16_to_cpu(input->count);
+ u16 support;
+ u8 *ptr;
+
+ if (offset > sizeof(*output)) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ if (offset + count > sizeof(*output)) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ /*
+ * CXL Spec r3.1 Table 8-206
+ * Saved selection is not supported
+ */
+ switch (input->selection) {
+ case SCRUB_PATROL_SAVED_VALUE:
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EOPNOTSUPP;
+ case SCRUB_PATROL_DEFAULT_VALUE:
+ support = FIELD_PREP(SCRUB_HOURS_CYCLE_MASK,
+ SCRUB_HOURS_CYCLE_DEFAULT) |
+ FIELD_PREP(SCRUB_SMALLEST_HOURS_MASK,
+ SCRUB_SMALLEST_HOURS_DEFAULT);
+ feat.cycle_support = cpu_to_le16(support);
+ ptr = (u8 *)&feat + offset;
+ break;
+ case SCRUB_PATROL_CURRENT_VALUE:
+ ptr = (u8 *)&mdata->patrol_feat + offset;
+ break;
+ default:
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ memcpy((u8 *)output + offset, ptr, count);
+
+ return 0;
+}
+
+static int mock_get_feature(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct get_feat_input *input = cmd->payload_in;
+
+ if (uuid_equal(&input->uuid, &DEV_PATROL_SCRUB_FEAT_UUID))
+ return mock_get_patrol_scrub(mdata, cmd);
+
+ cmd->return_code = CXL_MBOX_CMD_RC_UNSUPPORTED;
+
+ return -EOPNOTSUPP;
+}
+
static int mock_get_supported_features(struct cxl_mockmem_data *mdata,
struct cxl_mbox_cmd *cmd)
{
@@ -1486,6 +1577,9 @@ static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
rc = mock_get_supported_features(mdata, cmd);
break;
+ case CXL_MBOX_OP_GET_FEATURE:
+ rc = mock_get_feature(mdata, cmd);
+ break;
default:
break;
}
@@ -1522,6 +1616,18 @@ static ssize_t event_trigger_store(struct device *dev,
}
static DEVICE_ATTR_WO(event_trigger);
+static void cxl_mock_mem_set_default_feat(struct cxl_mockmem_data *mdata)
+{
+ u16 support;
+
+ support = FIELD_PREP(SCRUB_HOURS_CYCLE_MASK,
+ SCRUB_HOURS_CYCLE_DEFAULT) |
+ FIELD_PREP(SCRUB_SMALLEST_HOURS_MASK,
+ SCRUB_SMALLEST_HOURS_DEFAULT);
+
+ mdata->patrol_feat.cycle_support = cpu_to_le16(support);
+}
+
static int cxl_mock_mem_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
@@ -1609,6 +1715,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
return rc;
cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
+ cxl_mock_mem_set_default_feat(mdata);
return 0;
}
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* [RFC PATCH 13/13] cxl/test: Add Set Feature support to cxl_test
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (11 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 12/13] cxl/test: Add Get Feature support to cxl_test Dave Jiang
@ 2024-07-18 21:32 ` Dave Jiang
2024-07-19 6:23 ` [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Alejandro Lucero Palau
` (2 subsequent siblings)
15 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-18 21:32 UTC (permalink / raw)
To: linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
Add emulation to support Set Feature mailbox command to cxl_test.
The only feature supported is the device patrol scrub feature. The
set feature allows activation of patrol scrub for the cxl_test
emulated device. The command does not support partial data transfer
even though the spec allows it. This restriction is to reduce complexity
of the emulation given the patrol scrub feature is very minimal.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
tools/testing/cxl/test/mem.c | 89 ++++++++++++++++++++++++++++++++++++
1 file changed, 89 insertions(+)
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index c779640cb610..c2018d894719 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -52,6 +52,10 @@ static struct cxl_cel_entry mock_cel[] = {
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_FEATURE),
.effect = CXL_CMD_EFFECT_NONE,
},
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_SET_FEATURE),
+ .effect = cpu_to_le16(EFFECT(CONF_CHANGE_IMMEDIATE)),
+ },
{
.opcode = cpu_to_le16(CXL_MBOX_OP_IDENTIFY),
.effect = CXL_CMD_EFFECT_NONE,
@@ -1445,6 +1449,88 @@ static int mock_get_feature(struct cxl_mockmem_data *mdata,
return -EOPNOTSUPP;
}
+struct scrub_feat_data {
+ u8 cycle;
+#define SCRUB_DATA_F_ENABLE BIT(0)
+ u8 flags;
+};
+#define SCRUB_DATA_CYCLE_OFFSET 0
+#define SCRUB_DATA_FLAGS_OFFSET 1
+
+struct set_feat_input {
+ uuid_t uuid;
+ __le32 flags;
+ __le16 offset;
+ u8 version;
+ u8 rsvd[9];
+ u8 data[];
+} __packed;
+
+enum set_feature_flags {
+ SET_FEATURE_F_FULL_XFER = 0,
+ SET_FEATURE_F_INIT_XFER,
+ SET_FEATURE_F_CONT_XFER,
+ SET_FEATURE_F_FIN_XFER,
+ SET_FEATURE_F_ABORT_XFER,
+};
+
+#define SET_FEATURE_F_ACTION_MASK GENMASK(2, 0)
+
+static int mock_set_patrol_scrub(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct set_feat_input *input = cmd->payload_in;
+ struct scrub_feat_data *data;
+ u32 action;
+
+ action = FIELD_GET(SET_FEATURE_F_ACTION_MASK,
+ le32_to_cpu(input->flags));
+ /*
+ * While it is spec compliant to support other set actions, it is not
+ * necessary to add the complication in the emulation currently. Reject
+ * anything besides full xfer.
+ */
+ if (action != SET_FEATURE_F_FULL_XFER) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ /* Offset should be reserved when doing full transfer */
+ if (input->offset) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ data = (struct scrub_feat_data *)input->data;
+ /* Does not support changing cycle, should be reserved */
+ if (data->cycle) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+
+ /* Check reserved fields */
+ if (data->flags & ~SCRUB_DATA_F_ENABLE) {
+ cmd->return_code = CXL_MBOX_CMD_RC_INPUT;
+ return -EINVAL;
+ }
+ mdata->patrol_feat.flags = data->flags & SCRUB_DATA_F_ENABLE;
+
+ return 0;
+}
+
+static int mock_set_feature(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct set_feat_input *input = cmd->payload_in;
+
+ if (uuid_equal(&input->uuid, &DEV_PATROL_SCRUB_FEAT_UUID))
+ return mock_set_patrol_scrub(mdata, cmd);
+
+ cmd->return_code = CXL_MBOX_CMD_RC_UNSUPPORTED;
+
+ return -EOPNOTSUPP;
+}
+
static int mock_get_supported_features(struct cxl_mockmem_data *mdata,
struct cxl_mbox_cmd *cmd)
{
@@ -1580,6 +1666,9 @@ static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox,
case CXL_MBOX_OP_GET_FEATURE:
rc = mock_get_feature(mdata, cmd);
break;
+ case CXL_MBOX_OP_SET_FEATURE:
+ rc = mock_set_feature(mdata, cmd);
+ break;
default:
break;
}
--
2.45.2
^ permalink raw reply related [flat|nested] 47+ messages in thread* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (12 preceding siblings ...)
2024-07-18 21:32 ` [RFC PATCH 13/13] cxl/test: Add Set " Dave Jiang
@ 2024-07-19 6:23 ` Alejandro Lucero Palau
2024-07-19 15:48 ` Dave Jiang
2024-07-22 15:32 ` Jason Gunthorpe
2024-07-29 12:05 ` Jonathan Cameron
15 siblings, 1 reply; 47+ messages in thread
From: Alejandro Lucero Palau @ 2024-07-19 6:23 UTC (permalink / raw)
To: Dave Jiang, linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
On 7/18/24 22:32, Dave Jiang wrote:
> This series add support for CXL feature commands using the FWCTL framework [1].
> The code is untested and I'm looking for architectural and implementation feedback.
> While CXL currently has a chardev for user ioctls to send some mailbox
> commands to a memory device, the fwctl framework provides more security policies
> that can be a potential vehicle to move CXL ioctl path to that.
>
> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
> "Set Feature" commands are implemented. The "get" commands under the
> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
> on the effect of the feature. All mailbox commands for CXL provides an effects
> table that describes the effects of a command when performed on the device.
> For CXL features, there is also an effects field that describes the effects
> a feature write operation has on the device per feature. The security policy
> is checked against this feature specific effects field. Looking for discussion
> on matching the CXL spec defined effects with the FWCTL security policy.
>
> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>
> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>
> ---
>
> Dave Jiang (13):
> cxl: Move mailbox related bits to the same context
> cxl: Fix comment regarding cxl_query_cmd() return data
> cxl: Refactor user ioctl command path from mds to mailbox
> cxl: Add Get Supported Features command for kernel usage
> cxl/test: Add Get Supported Features mailbox command support
> cxl: Add Get Feature command support
> cxl: Add Set Feature command support
> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
> fwctl/cxl: Add support for get driver information
> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
> fwctl/cxl: Add query commands software command for ->fw_rpc()
> cxl/test: Add Get Feature support to cxl_test
> cxl/test: Add Set Feature support to cxl_test
>
> MAINTAINERS | 8 +
> drivers/cxl/core/core.h | 9 +-
> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
> drivers/cxl/core/memdev.c | 78 +++++---
> drivers/cxl/cxlmem.h | 141 +++------------
> drivers/cxl/pci.c | 68 ++++---
> drivers/cxl/pmem.c | 10 +-
> drivers/cxl/security.c | 18 +-
> drivers/fwctl/Kconfig | 9 +
> drivers/fwctl/Makefile | 1 +
> drivers/fwctl/cxl/Makefile | 4 +
> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
> include/uapi/fwctl/cxl.h | 92 ++++++++++
> include/uapi/fwctl/fwctl.h | 1 +
> include/uapi/linux/cxl_mem.h | 3 +
> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
> 17 files changed, 1515 insertions(+), 275 deletions(-)
>
I can not see any documentation change in that file list.
I would say for something like what this patchset does, it is mandatory.
Isn't it?
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-19 6:23 ` [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Alejandro Lucero Palau
@ 2024-07-19 15:48 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-07-19 15:48 UTC (permalink / raw)
To: Alejandro Lucero Palau, linux-cxl
Cc: dan.j.williams, ira.weiny, vishal.l.verma, alison.schofield,
Jonathan.Cameron, dave, jgg, shiju.jose
On 7/18/24 11:23 PM, Alejandro Lucero Palau wrote:
>
> On 7/18/24 22:32, Dave Jiang wrote:
>> This series add support for CXL feature commands using the FWCTL framework [1].
>> The code is untested and I'm looking for architectural and implementation feedback.
>> While CXL currently has a chardev for user ioctls to send some mailbox
>> commands to a memory device, the fwctl framework provides more security policies
>> that can be a potential vehicle to move CXL ioctl path to that.
>>
>> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
>> "Set Feature" commands are implemented. The "get" commands under the
>> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
>> on the effect of the feature. All mailbox commands for CXL provides an effects
>> table that describes the effects of a command when performed on the device.
>> For CXL features, there is also an effects field that describes the effects
>> a feature write operation has on the device per feature. The security policy
>> is checked against this feature specific effects field. Looking for discussion
>> on matching the CXL spec defined effects with the FWCTL security policy.
>>
>> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>>
>> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>>
>> ---
>>
>> Dave Jiang (13):
>> cxl: Move mailbox related bits to the same context
>> cxl: Fix comment regarding cxl_query_cmd() return data
>> cxl: Refactor user ioctl command path from mds to mailbox
>> cxl: Add Get Supported Features command for kernel usage
>> cxl/test: Add Get Supported Features mailbox command support
>> cxl: Add Get Feature command support
>> cxl: Add Set Feature command support
>> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
>> fwctl/cxl: Add support for get driver information
>> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
>> fwctl/cxl: Add query commands software command for ->fw_rpc()
>> cxl/test: Add Get Feature support to cxl_test
>> cxl/test: Add Set Feature support to cxl_test
>>
>> MAINTAINERS | 8 +
>> drivers/cxl/core/core.h | 9 +-
>> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
>> drivers/cxl/core/memdev.c | 78 +++++---
>> drivers/cxl/cxlmem.h | 141 +++------------
>> drivers/cxl/pci.c | 68 ++++---
>> drivers/cxl/pmem.c | 10 +-
>> drivers/cxl/security.c | 18 +-
>> drivers/fwctl/Kconfig | 9 +
>> drivers/fwctl/Makefile | 1 +
>> drivers/fwctl/cxl/Makefile | 4 +
>> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
>> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
>> include/uapi/fwctl/cxl.h | 92 ++++++++++
>> include/uapi/fwctl/fwctl.h | 1 +
>> include/uapi/linux/cxl_mem.h | 3 +
>> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
>> 17 files changed, 1515 insertions(+), 275 deletions(-)
>>
>
> I can not see any documentation change in that file list.
>
> I would say for something like what this patchset does, it is mandatory. Isn't it?
Yeah I will write some documentation to detail the new ioctl API. Thanks for the reminder.
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (13 preceding siblings ...)
2024-07-19 6:23 ` [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Alejandro Lucero Palau
@ 2024-07-22 15:32 ` Jason Gunthorpe
2024-07-29 12:05 ` Jonathan Cameron
15 siblings, 0 replies; 47+ messages in thread
From: Jason Gunthorpe @ 2024-07-22 15:32 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, Jonathan.Cameron, dave, shiju.jose
On Thu, Jul 18, 2024 at 02:32:18PM -0700, Dave Jiang wrote:
> This series add support for CXL feature commands using the FWCTL framework [1].
> The code is untested and I'm looking for architectural and implementation feedback.
> While CXL currently has a chardev for user ioctls to send some mailbox
> commands to a memory device, the fwctl framework provides more security policies
> that can be a potential vehicle to move CXL ioctl path to that.
This looks pretty good, I think it shows my general feeling that it
might be better to have these kinds of things in a structured
framework with a documented security policy then ad-hoc in ioctls with
every subsystem..
Thanks,
Jason
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-18 21:32 [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl Dave Jiang
` (14 preceding siblings ...)
2024-07-22 15:32 ` Jason Gunthorpe
@ 2024-07-29 12:05 ` Jonathan Cameron
2024-08-06 16:44 ` Dave Jiang
2024-11-13 0:17 ` Dave Jiang
15 siblings, 2 replies; 47+ messages in thread
From: Jonathan Cameron @ 2024-07-29 12:05 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose, Borislav Petkov,
Tony Luck, James Morse, Mauro Carvalho Chehab, Robert Richter
On Thu, 18 Jul 2024 14:32:18 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> This series add support for CXL feature commands using the FWCTL framework [1].
> The code is untested and I'm looking for architectural and implementation feedback.
> While CXL currently has a chardev for user ioctls to send some mailbox
> commands to a memory device, the fwctl framework provides more security policies
> that can be a potential vehicle to move CXL ioctl path to that.
>
> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
> "Set Feature" commands are implemented. The "get" commands under the
> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
> on the effect of the feature. All mailbox commands for CXL provides an effects
> table that describes the effects of a command when performed on the device.
> For CXL features, there is also an effects field that describes the effects
> a feature write operation has on the device per feature. The security policy
> is checked against this feature specific effects field. Looking for discussion
> on matching the CXL spec defined effects with the FWCTL security policy.
Hi Dave,
Great to have this code.
My main concern is that, once a feature is exposed by fwctl, it becomes ABI.
Even with the taint, does that mean we can't remove it later? We
may well have userspace code using fwctl that will break kernel driver
support.
As you probably know, for the scrub control you are using as an example,
there is an ongoing effort to generalize that across multiple subsystems.
It's a convenient test as a simple get/set feature in CXL, so maybe
I'm reading too much into that choice.
One piece of good feedback we got on that generalization effort was that
this sort of RAS control should all be in one place (we were proposing
a RAS control subsystem at the times, separate from EDAC). The key reason
being to ensure it gets sufficient review by experts in the RAS aspects
(rather than CXL or other implementation).
Shiju's latest series puts it all in /sys/bus/edac/
https://lore.kernel.org/linux-cxl/20240726160556.2079-1-shiju.jose@huawei.com/T/#t
Even if we make the driver safe to someone else messing with the state under it
(so basically make it stateless) we can't make the same guarantees for any
user space code on top of that interface. Can we rely on only one path ever
being used? I'm not sure.
We could in theory decide to expose all the stuff proposed for the EDAC / rascontrol
as fwctl only, but that's going to get nasty as next set of features we plan to
implement are memory repair related and in at least some cases those require the
memory to be offlined if you don't want nasty side effects. I don't think there
will be appetite for that sort of thing via fwctl.
There are a bunch of other get/set features in the CXL spec and I expect to see
more over time so I think this will be an ongoing discussion.
Jonathan
+CC Edac maintainers etc, as they may have views on using fwctl for this stuff.
>
> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>
> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>
> ---
>
> Dave Jiang (13):
> cxl: Move mailbox related bits to the same context
> cxl: Fix comment regarding cxl_query_cmd() return data
> cxl: Refactor user ioctl command path from mds to mailbox
> cxl: Add Get Supported Features command for kernel usage
> cxl/test: Add Get Supported Features mailbox command support
> cxl: Add Get Feature command support
> cxl: Add Set Feature command support
> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
> fwctl/cxl: Add support for get driver information
> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
> fwctl/cxl: Add query commands software command for ->fw_rpc()
> cxl/test: Add Get Feature support to cxl_test
> cxl/test: Add Set Feature support to cxl_test
>
> MAINTAINERS | 8 +
> drivers/cxl/core/core.h | 9 +-
> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
> drivers/cxl/core/memdev.c | 78 +++++---
> drivers/cxl/cxlmem.h | 141 +++------------
> drivers/cxl/pci.c | 68 ++++---
> drivers/cxl/pmem.c | 10 +-
> drivers/cxl/security.c | 18 +-
> drivers/fwctl/Kconfig | 9 +
> drivers/fwctl/Makefile | 1 +
> drivers/fwctl/cxl/Makefile | 4 +
> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
> include/uapi/fwctl/cxl.h | 92 ++++++++++
> include/uapi/fwctl/fwctl.h | 1 +
> include/uapi/linux/cxl_mem.h | 3 +
> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
> 17 files changed, 1515 insertions(+), 275 deletions(-)
^ permalink raw reply [flat|nested] 47+ messages in thread* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-29 12:05 ` Jonathan Cameron
@ 2024-08-06 16:44 ` Dave Jiang
2024-11-13 0:17 ` Dave Jiang
1 sibling, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-08-06 16:44 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose, Borislav Petkov,
Tony Luck, James Morse, Mauro Carvalho Chehab, Robert Richter
On 7/29/24 5:05 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:18 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> This series add support for CXL feature commands using the FWCTL framework [1].
>> The code is untested and I'm looking for architectural and implementation feedback.
>> While CXL currently has a chardev for user ioctls to send some mailbox
>> commands to a memory device, the fwctl framework provides more security policies
>> that can be a potential vehicle to move CXL ioctl path to that.
>>
>> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
>> "Set Feature" commands are implemented. The "get" commands under the
>> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
>> on the effect of the feature. All mailbox commands for CXL provides an effects
>> table that describes the effects of a command when performed on the device.
>> For CXL features, there is also an effects field that describes the effects
>> a feature write operation has on the device per feature. The security policy
>> is checked against this feature specific effects field. Looking for discussion
>> on matching the CXL spec defined effects with the FWCTL security policy.
>
> Hi Dave,
>
> Great to have this code.
>
> My main concern is that, once a feature is exposed by fwctl, it becomes ABI.
> Even with the taint, does that mean we can't remove it later? We
> may well have userspace code using fwctl that will break kernel driver
> support.
>
> As you probably know, for the scrub control you are using as an example,
> there is an ongoing effort to generalize that across multiple subsystems.
> It's a convenient test as a simple get/set feature in CXL, so maybe
> I'm reading too much into that choice.
>
> One piece of good feedback we got on that generalization effort was that
> this sort of RAS control should all be in one place (we were proposing
> a RAS control subsystem at the times, separate from EDAC). The key reason
> being to ensure it gets sufficient review by experts in the RAS aspects
> (rather than CXL or other implementation).
>
> Shiju's latest series puts it all in /sys/bus/edac/
> https://lore.kernel.org/linux-cxl/20240726160556.2079-1-shiju.jose@huawei.com/T/#t
I didn't see Shiju's patches until I created the RFC. For v1 I just wanted to post upstream to show intent and get some discussion going. I did let Shiju know that I'll pull in his code in a later rev. I expect a lot of churn with this code as we discuss.
It does seem like we may need a per feature (write) list/mask that is exclusive to kernel such as scrub? Also it looks like I need to find a different cxl spec defined feature to use for testing fwctl.
>
> Even if we make the driver safe to someone else messing with the state under it
> (so basically make it stateless) we can't make the same guarantees for any
> user space code on top of that interface. Can we rely on only one path ever
> being used? I'm not sure.
>
> We could in theory decide to expose all the stuff proposed for the EDAC / rascontrol
> as fwctl only, but that's going to get nasty as next set of features we plan to
> implement are memory repair related and in at least some cases those require the
> memory to be offlined if you don't want nasty side effects. I don't think there
> will be appetite for that sort of thing via fwctl.
>
> There are a bunch of other get/set features in the CXL spec and I expect to see
> more over time so I think this will be an ongoing discussion.
>
> Jonathan
>
> +CC Edac maintainers etc, as they may have views on using fwctl for this stuff.
>
>
>
>>
>> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>>
>> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>>
>> ---
>>
>> Dave Jiang (13):
>> cxl: Move mailbox related bits to the same context
>> cxl: Fix comment regarding cxl_query_cmd() return data
>> cxl: Refactor user ioctl command path from mds to mailbox
>> cxl: Add Get Supported Features command for kernel usage
>> cxl/test: Add Get Supported Features mailbox command support
>> cxl: Add Get Feature command support
>> cxl: Add Set Feature command support
>> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
>> fwctl/cxl: Add support for get driver information
>> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
>> fwctl/cxl: Add query commands software command for ->fw_rpc()
>> cxl/test: Add Get Feature support to cxl_test
>> cxl/test: Add Set Feature support to cxl_test
>>
>> MAINTAINERS | 8 +
>> drivers/cxl/core/core.h | 9 +-
>> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
>> drivers/cxl/core/memdev.c | 78 +++++---
>> drivers/cxl/cxlmem.h | 141 +++------------
>> drivers/cxl/pci.c | 68 ++++---
>> drivers/cxl/pmem.c | 10 +-
>> drivers/cxl/security.c | 18 +-
>> drivers/fwctl/Kconfig | 9 +
>> drivers/fwctl/Makefile | 1 +
>> drivers/fwctl/cxl/Makefile | 4 +
>> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
>> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
>> include/uapi/fwctl/cxl.h | 92 ++++++++++
>> include/uapi/fwctl/fwctl.h | 1 +
>> include/uapi/linux/cxl_mem.h | 3 +
>> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
>> 17 files changed, 1515 insertions(+), 275 deletions(-)
>
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-07-29 12:05 ` Jonathan Cameron
2024-08-06 16:44 ` Dave Jiang
@ 2024-11-13 0:17 ` Dave Jiang
2024-11-19 11:43 ` Jonathan Cameron
1 sibling, 1 reply; 47+ messages in thread
From: Dave Jiang @ 2024-11-13 0:17 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose, Borislav Petkov,
Tony Luck, James Morse, Mauro Carvalho Chehab, Robert Richter
On 7/29/24 5:05 AM, Jonathan Cameron wrote:
> On Thu, 18 Jul 2024 14:32:18 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> This series add support for CXL feature commands using the FWCTL framework [1].
>> The code is untested and I'm looking for architectural and implementation feedback.
>> While CXL currently has a chardev for user ioctls to send some mailbox
>> commands to a memory device, the fwctl framework provides more security policies
>> that can be a potential vehicle to move CXL ioctl path to that.
>>
>> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
>> "Set Feature" commands are implemented. The "get" commands under the
>> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
>> on the effect of the feature. All mailbox commands for CXL provides an effects
>> table that describes the effects of a command when performed on the device.
>> For CXL features, there is also an effects field that describes the effects
>> a feature write operation has on the device per feature. The security policy
>> is checked against this feature specific effects field. Looking for discussion
>> on matching the CXL spec defined effects with the FWCTL security policy.
>
> Hi Dave,
>
> Great to have this code.
>
> My main concern is that, once a feature is exposed by fwctl, it becomes ABI.
> Even with the taint, does that mean we can't remove it later? We
> may well have userspace code using fwctl that will break kernel driver
> support.
Maybe we can sanitize the return of the get features command on the kernel side and block what we don't want the user to see. That way we can theoretically "remove" things that's within the kernel's control right?
DJ
>
> As you probably know, for the scrub control you are using as an example,
> there is an ongoing effort to generalize that across multiple subsystems.
> It's a convenient test as a simple get/set feature in CXL, so maybe
> I'm reading too much into that choice.
>
> One piece of good feedback we got on that generalization effort was that
> this sort of RAS control should all be in one place (we were proposing
> a RAS control subsystem at the times, separate from EDAC). The key reason
> being to ensure it gets sufficient review by experts in the RAS aspects
> (rather than CXL or other implementation).
>
> Shiju's latest series puts it all in /sys/bus/edac/
> https://lore.kernel.org/linux-cxl/20240726160556.2079-1-shiju.jose@huawei.com/T/#t
>
> Even if we make the driver safe to someone else messing with the state under it
> (so basically make it stateless) we can't make the same guarantees for any
> user space code on top of that interface. Can we rely on only one path ever
> being used? I'm not sure.
>
> We could in theory decide to expose all the stuff proposed for the EDAC / rascontrol
> as fwctl only, but that's going to get nasty as next set of features we plan to
> implement are memory repair related and in at least some cases those require the
> memory to be offlined if you don't want nasty side effects. I don't think there
> will be appetite for that sort of thing via fwctl.
>
> There are a bunch of other get/set features in the CXL spec and I expect to see
> more over time so I think this will be an ongoing discussion.
>
> Jonathan
>
> +CC Edac maintainers etc, as they may have views on using fwctl for this stuff.
>
>
>
>>
>> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>>
>> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>>
>> ---
>>
>> Dave Jiang (13):
>> cxl: Move mailbox related bits to the same context
>> cxl: Fix comment regarding cxl_query_cmd() return data
>> cxl: Refactor user ioctl command path from mds to mailbox
>> cxl: Add Get Supported Features command for kernel usage
>> cxl/test: Add Get Supported Features mailbox command support
>> cxl: Add Get Feature command support
>> cxl: Add Set Feature command support
>> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
>> fwctl/cxl: Add support for get driver information
>> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
>> fwctl/cxl: Add query commands software command for ->fw_rpc()
>> cxl/test: Add Get Feature support to cxl_test
>> cxl/test: Add Set Feature support to cxl_test
>>
>> MAINTAINERS | 8 +
>> drivers/cxl/core/core.h | 9 +-
>> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
>> drivers/cxl/core/memdev.c | 78 +++++---
>> drivers/cxl/cxlmem.h | 141 +++------------
>> drivers/cxl/pci.c | 68 ++++---
>> drivers/cxl/pmem.c | 10 +-
>> drivers/cxl/security.c | 18 +-
>> drivers/fwctl/Kconfig | 9 +
>> drivers/fwctl/Makefile | 1 +
>> drivers/fwctl/cxl/Makefile | 4 +
>> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
>> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
>> include/uapi/fwctl/cxl.h | 92 ++++++++++
>> include/uapi/fwctl/fwctl.h | 1 +
>> include/uapi/linux/cxl_mem.h | 3 +
>> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
>> 17 files changed, 1515 insertions(+), 275 deletions(-)
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-11-13 0:17 ` Dave Jiang
@ 2024-11-19 11:43 ` Jonathan Cameron
2024-11-19 15:58 ` Dave Jiang
0 siblings, 1 reply; 47+ messages in thread
From: Jonathan Cameron @ 2024-11-19 11:43 UTC (permalink / raw)
To: Dave Jiang
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose, Borislav Petkov,
Tony Luck, James Morse, Mauro Carvalho Chehab, Robert Richter
On Tue, 12 Nov 2024 17:17:11 -0700
Dave Jiang <dave.jiang@intel.com> wrote:
> On 7/29/24 5:05 AM, Jonathan Cameron wrote:
> > On Thu, 18 Jul 2024 14:32:18 -0700
> > Dave Jiang <dave.jiang@intel.com> wrote:
> >
> >> This series add support for CXL feature commands using the FWCTL framework [1].
> >> The code is untested and I'm looking for architectural and implementation feedback.
> >> While CXL currently has a chardev for user ioctls to send some mailbox
> >> commands to a memory device, the fwctl framework provides more security policies
> >> that can be a potential vehicle to move CXL ioctl path to that.
> >>
> >> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
> >> "Set Feature" commands are implemented. The "get" commands under the
> >> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
> >> on the effect of the feature. All mailbox commands for CXL provides an effects
> >> table that describes the effects of a command when performed on the device.
> >> For CXL features, there is also an effects field that describes the effects
> >> a feature write operation has on the device per feature. The security policy
> >> is checked against this feature specific effects field. Looking for discussion
> >> on matching the CXL spec defined effects with the FWCTL security policy.
> >
> > Hi Dave,
> >
> > Great to have this code.
> >
> > My main concern is that, once a feature is exposed by fwctl, it becomes ABI.
> > Even with the taint, does that mean we can't remove it later? We
> > may well have userspace code using fwctl that will break kernel driver
> > support.
>
> Maybe we can sanitize the return of the get features command on the kernel side and block what we don't want the user to see. That way we can theoretically "remove" things that's within the kernel's control right?
Hmm. That sort of works, though I think it still would conventionally be
ABI breakage if we made a feature disappear between kernel versions.
Hopefully we are find doing that in the corner case that we don't have
visibility of a future need to remove a feature from what get feature
reports.
J
>
> DJ
>
>
> >
> > As you probably know, for the scrub control you are using as an example,
> > there is an ongoing effort to generalize that across multiple subsystems.
> > It's a convenient test as a simple get/set feature in CXL, so maybe
> > I'm reading too much into that choice.
> >
> > One piece of good feedback we got on that generalization effort was that
> > this sort of RAS control should all be in one place (we were proposing
> > a RAS control subsystem at the times, separate from EDAC). The key reason
> > being to ensure it gets sufficient review by experts in the RAS aspects
> > (rather than CXL or other implementation).
> >
> > Shiju's latest series puts it all in /sys/bus/edac/
> > https://lore.kernel.org/linux-cxl/20240726160556.2079-1-shiju.jose@huawei.com/T/#t
> >
> > Even if we make the driver safe to someone else messing with the state under it
> > (so basically make it stateless) we can't make the same guarantees for any
> > user space code on top of that interface. Can we rely on only one path ever
> > being used? I'm not sure.
> >
> > We could in theory decide to expose all the stuff proposed for the EDAC / rascontrol
> > as fwctl only, but that's going to get nasty as next set of features we plan to
> > implement are memory repair related and in at least some cases those require the
> > memory to be offlined if you don't want nasty side effects. I don't think there
> > will be appetite for that sort of thing via fwctl.
> >
> > There are a bunch of other get/set features in the CXL spec and I expect to see
> > more over time so I think this will be an ongoing discussion.
> >
> > Jonathan
> >
> > +CC Edac maintainers etc, as they may have views on using fwctl for this stuff.
> >
> >
> >
> >>
> >> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
> >>
> >> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
> >>
> >> ---
> >>
> >> Dave Jiang (13):
> >> cxl: Move mailbox related bits to the same context
> >> cxl: Fix comment regarding cxl_query_cmd() return data
> >> cxl: Refactor user ioctl command path from mds to mailbox
> >> cxl: Add Get Supported Features command for kernel usage
> >> cxl/test: Add Get Supported Features mailbox command support
> >> cxl: Add Get Feature command support
> >> cxl: Add Set Feature command support
> >> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
> >> fwctl/cxl: Add support for get driver information
> >> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
> >> fwctl/cxl: Add query commands software command for ->fw_rpc()
> >> cxl/test: Add Get Feature support to cxl_test
> >> cxl/test: Add Set Feature support to cxl_test
> >>
> >> MAINTAINERS | 8 +
> >> drivers/cxl/core/core.h | 9 +-
> >> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
> >> drivers/cxl/core/memdev.c | 78 +++++---
> >> drivers/cxl/cxlmem.h | 141 +++------------
> >> drivers/cxl/pci.c | 68 ++++---
> >> drivers/cxl/pmem.c | 10 +-
> >> drivers/cxl/security.c | 18 +-
> >> drivers/fwctl/Kconfig | 9 +
> >> drivers/fwctl/Makefile | 1 +
> >> drivers/fwctl/cxl/Makefile | 4 +
> >> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
> >> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
> >> include/uapi/fwctl/cxl.h | 92 ++++++++++
> >> include/uapi/fwctl/fwctl.h | 1 +
> >> include/uapi/linux/cxl_mem.h | 3 +
> >> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
> >> 17 files changed, 1515 insertions(+), 275 deletions(-)
> >
>
>
^ permalink raw reply [flat|nested] 47+ messages in thread
* Re: [RFC PATCH 0/13] fwctl/cxl: Add CXL feature commands support via fwctl
2024-11-19 11:43 ` Jonathan Cameron
@ 2024-11-19 15:58 ` Dave Jiang
0 siblings, 0 replies; 47+ messages in thread
From: Dave Jiang @ 2024-11-19 15:58 UTC (permalink / raw)
To: Jonathan Cameron
Cc: linux-cxl, dan.j.williams, ira.weiny, vishal.l.verma,
alison.schofield, dave, jgg, shiju.jose, Borislav Petkov,
Tony Luck, James Morse, Mauro Carvalho Chehab, Robert Richter
On 11/19/24 4:43 AM, Jonathan Cameron wrote:
> On Tue, 12 Nov 2024 17:17:11 -0700
> Dave Jiang <dave.jiang@intel.com> wrote:
>
>> On 7/29/24 5:05 AM, Jonathan Cameron wrote:
>>> On Thu, 18 Jul 2024 14:32:18 -0700
>>> Dave Jiang <dave.jiang@intel.com> wrote:
>>>
>>>> This series add support for CXL feature commands using the FWCTL framework [1].
>>>> The code is untested and I'm looking for architectural and implementation feedback.
>>>> While CXL currently has a chardev for user ioctls to send some mailbox
>>>> commands to a memory device, the fwctl framework provides more security policies
>>>> that can be a potential vehicle to move CXL ioctl path to that.
>>>>
>>>> For this RFC, the mailbox commands "Get Supported Features", "Get Feature", and
>>>> "Set Feature" commands are implemented. The "get" commands under the
>>>> FWCTL_RPC_DEBUG_READ_ONLY policy, the "set" command checks the policy depending
>>>> on the effect of the feature. All mailbox commands for CXL provides an effects
>>>> table that describes the effects of a command when performed on the device.
>>>> For CXL features, there is also an effects field that describes the effects
>>>> a feature write operation has on the device per feature. The security policy
>>>> is checked against this feature specific effects field. Looking for discussion
>>>> on matching the CXL spec defined effects with the FWCTL security policy.
>>>
>>> Hi Dave,
>>>
>>> Great to have this code.
>>>
>>> My main concern is that, once a feature is exposed by fwctl, it becomes ABI.
>>> Even with the taint, does that mean we can't remove it later? We
>>> may well have userspace code using fwctl that will break kernel driver
>>> support.
>>
>> Maybe we can sanitize the return of the get features command on the kernel side and block what we don't want the user to see. That way we can theoretically "remove" things that's within the kernel's control right?
>
> Hmm. That sort of works, though I think it still would conventionally be
> ABI breakage if we made a feature disappear between kernel versions.
>
> Hopefully we are find doing that in the corner case that we don't have
> visibility of a future need to remove a feature from what get feature
> reports.
Please take a look at the implementation in v2 and see if that would work. To me given that the features are discovery based and we filter the output of "get supported features" command via the kernel, user app should only issue command after verified via discovery and not hard code support.
>
> J
>>
>> DJ
>>
>>
>>>
>>> As you probably know, for the scrub control you are using as an example,
>>> there is an ongoing effort to generalize that across multiple subsystems.
>>> It's a convenient test as a simple get/set feature in CXL, so maybe
>>> I'm reading too much into that choice.
>>>
>>> One piece of good feedback we got on that generalization effort was that
>>> this sort of RAS control should all be in one place (we were proposing
>>> a RAS control subsystem at the times, separate from EDAC). The key reason
>>> being to ensure it gets sufficient review by experts in the RAS aspects
>>> (rather than CXL or other implementation).
>>>
>>> Shiju's latest series puts it all in /sys/bus/edac/
>>> https://lore.kernel.org/linux-cxl/20240726160556.2079-1-shiju.jose@huawei.com/T/#t
>>>
>>> Even if we make the driver safe to someone else messing with the state under it
>>> (so basically make it stateless) we can't make the same guarantees for any
>>> user space code on top of that interface. Can we rely on only one path ever
>>> being used? I'm not sure.
>>>
>>> We could in theory decide to expose all the stuff proposed for the EDAC / rascontrol
>>> as fwctl only, but that's going to get nasty as next set of features we plan to
>>> implement are memory repair related and in at least some cases those require the
>>> memory to be offlined if you don't want nasty side effects. I don't think there
>>> will be appetite for that sort of thing via fwctl.
>>>
>>> There are a bunch of other get/set features in the CXL spec and I expect to see
>>> more over time so I think this will be an ongoing discussion.
>>>
>>> Jonathan
>>>
>>> +CC Edac maintainers etc, as they may have views on using fwctl for this stuff.
>>>
>>>
>>>
>>>>
>>>> The code is based off of the latest FWCTL series [1] posted by Jason on top of v6.10.
>>>>
>>>> [1]: https://lore.kernel.org/linux-cxl/20240624161802.1b7c962d@kernel.org/T/#t
>>>>
>>>> ---
>>>>
>>>> Dave Jiang (13):
>>>> cxl: Move mailbox related bits to the same context
>>>> cxl: Fix comment regarding cxl_query_cmd() return data
>>>> cxl: Refactor user ioctl command path from mds to mailbox
>>>> cxl: Add Get Supported Features command for kernel usage
>>>> cxl/test: Add Get Supported Features mailbox command support
>>>> cxl: Add Get Feature command support
>>>> cxl: Add Set Feature command support
>>>> fwctl/cxl: Add driver for CXL mailbox for handling CXL features commands
>>>> fwctl/cxl: Add support for get driver information
>>>> fwctl/cxl: Add support for fwctl RPC command to enable CXL feature commands
>>>> fwctl/cxl: Add query commands software command for ->fw_rpc()
>>>> cxl/test: Add Get Feature support to cxl_test
>>>> cxl/test: Add Set Feature support to cxl_test
>>>>
>>>> MAINTAINERS | 8 +
>>>> drivers/cxl/core/core.h | 9 +-
>>>> drivers/cxl/core/mbox.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
>>>> drivers/cxl/core/memdev.c | 78 +++++---
>>>> drivers/cxl/cxlmem.h | 141 +++------------
>>>> drivers/cxl/pci.c | 68 ++++---
>>>> drivers/cxl/pmem.c | 10 +-
>>>> drivers/cxl/security.c | 18 +-
>>>> drivers/fwctl/Kconfig | 9 +
>>>> drivers/fwctl/Makefile | 1 +
>>>> drivers/fwctl/cxl/Makefile | 4 +
>>>> drivers/fwctl/cxl/cxl.c | 274 ++++++++++++++++++++++++++++
>>>> include/linux/cxl/mailbox.h | 175 ++++++++++++++++++
>>>> include/uapi/fwctl/cxl.h | 92 ++++++++++
>>>> include/uapi/fwctl/fwctl.h | 1 +
>>>> include/uapi/linux/cxl_mem.h | 3 +
>>>> tools/testing/cxl/test/mem.c | 292 +++++++++++++++++++++++++++++-
>>>> 17 files changed, 1515 insertions(+), 275 deletions(-)
>>>
>>
>>
>
^ permalink raw reply [flat|nested] 47+ messages in thread