* [PATCH 1/9] hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output payload of identify memory device command
[not found] <20230720203708.25825-1-nifan@outlook.com>
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 2/9] hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative and mailbox command support nifan
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Based on CXL spec 3.0 Table 8-94 (Identify Memory Device Output
Payload), dynamic capacity event log size should be part of
output of the Identify command.
Add dc_event_log_size to the output payload for the host to get the info.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index ad7a6116e4..b013e30314 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -21,6 +21,8 @@
#include "sysemu/hostmem.h"
#define CXL_CAPACITY_MULTIPLIER (256 * MiB)
+/* Experimental value: dynamic capacity event log size */
+#define CXL_DC_EVENT_LOG_SIZE 8
/*
* How to add a new command, example. The command set FOO, with cmd BAR.
@@ -519,8 +521,9 @@ static CXLRetCode cmd_identify_memory_device(struct cxl_cmd *cmd,
uint16_t inject_poison_limit;
uint8_t poison_caps;
uint8_t qos_telemetry_caps;
+ uint16_t dc_event_log_size;
} QEMU_PACKED *id;
- QEMU_BUILD_BUG_ON(sizeof(*id) != 0x43);
+ QEMU_BUILD_BUG_ON(sizeof(*id) != 0x45);
CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
CXLType3Class *cvc = CXL_TYPE3_GET_CLASS(ct3d);
@@ -543,6 +546,7 @@ static CXLRetCode cmd_identify_memory_device(struct cxl_cmd *cmd,
st24_le_p(id->poison_list_max_mer, 256);
/* No limit - so limited by main poison record limit */
stw_le_p(&id->inject_poison_limit, 0);
+ stw_le_p(&id->dc_event_log_size, CXL_DC_EVENT_LOG_SIZE);
*len = sizeof(*id);
return CXL_MBOX_SUCCESS;
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 2/9] hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative and mailbox command support
[not found] <20230720203708.25825-1-nifan@outlook.com>
2023-07-20 20:37 ` [PATCH 1/9] hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output payload of identify memory device command nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 3/9] include/hw/cxl/cxl_device: Rename mem_size as static_mem_size for type3 memory devices nifan
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Per cxl spec 3.0, add dynamic capacity region representative based on
Table 8-126 and extend the cxl type3 device definition to include dc region
information. Also, based on info in 8.2.9.8.9.1, add 'Get Dynamic Capacity
Configuration' mailbox support.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 69 +++++++++++++++++++++++++++++++++++++
include/hw/cxl/cxl_device.h | 16 +++++++++
2 files changed, 85 insertions(+)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index b013e30314..41e43ec231 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -81,6 +81,8 @@ enum {
#define GET_POISON_LIST 0x0
#define INJECT_POISON 0x1
#define CLEAR_POISON 0x2
+ DCD_CONFIG = 0x48, /*r3.0: 8.2.9.8.9*/
+ #define GET_DC_CONFIG 0x0
PHYSICAL_SWITCH = 0x51
#define IDENTIFY_SWITCH_DEVICE 0x0
};
@@ -939,6 +941,71 @@ static CXLRetCode cmd_media_clear_poison(struct cxl_cmd *cmd,
return CXL_MBOX_SUCCESS;
}
+/*
+ * cxl spec 3.0: 8.2.9.8.9.1
+ * Get Dynamic Capacity Configuration
+ **/
+static CXLRetCode cmd_dcd_get_dyn_cap_config(struct cxl_cmd *cmd,
+ CXLDeviceState *cxl_dstate,
+ uint16_t *len)
+{
+ struct get_dyn_cap_config_in_pl {
+ uint8_t region_cnt;
+ uint8_t start_region_id;
+ } QEMU_PACKED;
+
+ struct get_dyn_cap_config_out_pl {
+ uint8_t num_regions;
+ uint8_t rsvd1[7];
+ struct {
+ uint64_t base;
+ uint64_t decode_len;
+ uint64_t region_len;
+ uint64_t block_size;
+ uint32_t dsmadhandle;
+ uint8_t flags;
+ uint8_t rsvd2[3];
+ } QEMU_PACKED records[];
+ } QEMU_PACKED;
+
+ struct get_dyn_cap_config_in_pl *in = (void *)cmd->payload;
+ struct get_dyn_cap_config_out_pl *out = (void *)cmd->payload;
+ struct CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev,
+ cxl_dstate);
+ uint16_t record_count = 0, i;
+ uint16_t out_pl_len;
+ uint8_t start_region_id = in->start_region_id;
+
+ if (start_region_id >= ct3d->dc.num_regions) {
+ return CXL_MBOX_INVALID_INPUT;
+ }
+
+ record_count = MIN(ct3d->dc.num_regions - in->start_region_id,
+ in->region_cnt);
+
+ out_pl_len = sizeof(*out) + record_count * sizeof(out->records[0]);
+ assert(out_pl_len <= CXL_MAILBOX_MAX_PAYLOAD_SIZE);
+
+ memset(out, 0, out_pl_len);
+ out->num_regions = record_count;
+ for (i = 0; i < record_count; i++) {
+ stq_le_p(&out->records[i].base,
+ ct3d->dc.regions[start_region_id + i].base);
+ stq_le_p(&out->records[i].decode_len,
+ ct3d->dc.regions[start_region_id + i].decode_len);
+ stq_le_p(&out->records[i].region_len,
+ ct3d->dc.regions[start_region_id + i].len);
+ stq_le_p(&out->records[i].block_size,
+ ct3d->dc.regions[start_region_id + i].block_size);
+ stl_le_p(&out->records[i].dsmadhandle,
+ ct3d->dc.regions[start_region_id + i].dsmadhandle);
+ out->records[i].flags = ct3d->dc.regions[start_region_id + i].flags;
+ }
+
+ *len = out_pl_len;
+ return CXL_MBOX_SUCCESS;
+}
+
#define IMMEDIATE_CONFIG_CHANGE (1 << 1)
#define IMMEDIATE_DATA_CHANGE (1 << 2)
#define IMMEDIATE_POLICY_CHANGE (1 << 3)
@@ -977,6 +1044,8 @@ static struct cxl_cmd cxl_cmd_set[256][256] = {
cmd_media_inject_poison, 8, 0 },
[MEDIA_AND_POISON][CLEAR_POISON] = { "MEDIA_AND_POISON_CLEAR_POISON",
cmd_media_clear_poison, 72, 0 },
+ [DCD_CONFIG][GET_DC_CONFIG] = { "DCD_GET_DC_CONFIG",
+ cmd_dcd_get_dyn_cap_config, 2, 0 },
};
static struct cxl_cmd cxl_cmd_set_sw[256][256] = {
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index cd7f28dba8..bf564f4a0b 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -382,6 +382,17 @@ typedef struct CXLPoison {
typedef QLIST_HEAD(, CXLPoison) CXLPoisonList;
#define CXL_POISON_LIST_LIMIT 256
+#define DCD_MAX_REGION_NUM 8
+
+typedef struct CXLDCD_Region {
+ uint64_t base;
+ uint64_t decode_len; /* in multiples of 256MB */
+ uint64_t len;
+ uint64_t block_size;
+ uint32_t dsmadhandle;
+ uint8_t flags;
+} CXLDCD_Region;
+
struct CXLType3Dev {
/* Private */
PCIDevice parent_obj;
@@ -413,6 +424,11 @@ struct CXLType3Dev {
unsigned int poison_list_cnt;
bool poison_list_overflowed;
uint64_t poison_list_overflow_ts;
+
+ struct dynamic_capacity {
+ uint8_t num_regions; /* 0-8 regions */
+ struct CXLDCD_Region regions[DCD_MAX_REGION_NUM];
+ } dc;
};
#define TYPE_CXL_TYPE3 "cxl-type3"
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 3/9] include/hw/cxl/cxl_device: Rename mem_size as static_mem_size for type3 memory devices
[not found] <20230720203708.25825-1-nifan@outlook.com>
2023-07-20 20:37 ` [PATCH 1/9] hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output payload of identify memory device command nifan
2023-07-20 20:37 ` [PATCH 2/9] hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative and mailbox command support nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 4/9] hw/mem/cxl_type3: Add support to create DC regions to " nifan
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Rename mem_size as static_mem_size for type3 memdev to cover static RAM and
pmem capacity, preparing for the introduction of dynamic capacity to support
dynamic capacity devices.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 5 +++--
hw/mem/cxl_type3.c | 8 ++++----
include/hw/cxl/cxl_device.h | 2 +-
3 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 41e43ec231..59d57ae245 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -540,7 +540,8 @@ static CXLRetCode cmd_identify_memory_device(struct cxl_cmd *cmd,
snprintf(id->fw_revision, 0x10, "BWFW VERSION %02d", 0);
- stq_le_p(&id->total_capacity, cxl_dstate->mem_size / CXL_CAPACITY_MULTIPLIER);
+ stq_le_p(&id->total_capacity,
+ cxl_dstate->static_mem_size / CXL_CAPACITY_MULTIPLIER);
stq_le_p(&id->persistent_capacity, cxl_dstate->pmem_size / CXL_CAPACITY_MULTIPLIER);
stq_le_p(&id->volatile_capacity, cxl_dstate->vmem_size / CXL_CAPACITY_MULTIPLIER);
stl_le_p(&id->lsa_size, cvc->get_lsa_size(ct3d));
@@ -879,7 +880,7 @@ static CXLRetCode cmd_media_clear_poison(struct cxl_cmd *cmd,
struct clear_poison_pl *in = (void *)cmd->payload;
dpa = ldq_le_p(&in->dpa);
- if (dpa + CXL_CACHE_LINE_SIZE > cxl_dstate->mem_size) {
+ if (dpa + CXL_CACHE_LINE_SIZE > cxl_dstate->static_mem_size) {
return CXL_MBOX_INVALID_PA;
}
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 862107c5ef..237b544b9c 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -748,7 +748,7 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
}
address_space_init(&ct3d->hostvmem_as, vmr, v_name);
ct3d->cxl_dstate.vmem_size = memory_region_size(vmr);
- ct3d->cxl_dstate.mem_size += memory_region_size(vmr);
+ ct3d->cxl_dstate.static_mem_size += memory_region_size(vmr);
g_free(v_name);
}
@@ -771,7 +771,7 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
}
address_space_init(&ct3d->hostpmem_as, pmr, p_name);
ct3d->cxl_dstate.pmem_size = memory_region_size(pmr);
- ct3d->cxl_dstate.mem_size += memory_region_size(pmr);
+ ct3d->cxl_dstate.static_mem_size += memory_region_size(pmr);
g_free(p_name);
}
@@ -984,7 +984,7 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
return -EINVAL;
}
- if (*dpa_offset > ct3d->cxl_dstate.mem_size) {
+ if (*dpa_offset > ct3d->cxl_dstate.static_mem_size) {
return -EINVAL;
}
@@ -1142,7 +1142,7 @@ static bool set_cacheline(CXLType3Dev *ct3d, uint64_t dpa_offset, uint8_t *data)
return false;
}
- if (dpa_offset + CXL_CACHE_LINE_SIZE > ct3d->cxl_dstate.mem_size) {
+ if (dpa_offset + CXL_CACHE_LINE_SIZE > ct3d->cxl_dstate.static_mem_size) {
return false;
}
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index bf564f4a0b..a32ee6d6ba 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -209,7 +209,7 @@ typedef struct cxl_device_state {
} timestamp;
/* memory region size, HDM */
- uint64_t mem_size;
+ uint64_t static_mem_size;
uint64_t pmem_size;
uint64_t vmem_size;
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/9] hw/mem/cxl_type3: Add support to create DC regions to type3 memory devices
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (2 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 3/9] include/hw/cxl/cxl_device: Rename mem_size as static_mem_size for type3 memory devices nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 5/9] hw/mem/cxl_type3: Add host backend and address space handling for DC regions nifan
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
With the change, when setting up memory for type3 memory device, we can
create DC regions
A property 'num-dc-regions' is added to ct3_props to allow users to pass the
number of DC regions to create. To make it easier, other region parameters
like region base, length, and block size are hard coded. If needed,
these parameters can be added easily.
With the change, we can create DC regions with proper kernel side
support as below:
region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
echo "dc0" >/sys/bus/cxl/devices/decoder2.0/mode
echo 0x40000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
echo 0x40000000 > /sys/bus/cxl/devices/$region/size
echo "decoder2.0" > /sys/bus/cxl/devices/$region/target0
echo 1 > /sys/bus/cxl/devices/$region/commit
echo $region > /sys/bus/cxl/drivers/cxl_region/bind
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/mem/cxl_type3.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 237b544b9c..27b5920f7d 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -707,6 +707,34 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
}
}
+/*
+ * Create a dc region to test "Get Dynamic Capacity Configuration" command.
+ */
+static int cxl_create_dc_regions(CXLType3Dev *ct3d)
+{
+ int i;
+ uint64_t region_base = (ct3d->hostvmem ? ct3d->hostvmem->size : 0)
+ + (ct3d->hostpmem ? ct3d->hostpmem->size : 0);
+ uint64_t region_len = (uint64_t)2 * 1024 * 1024 * 1024;
+ uint64_t decode_len = 4; /* 4*256MB */
+ uint64_t blk_size = 2 * 1024 * 1024;
+ struct CXLDCD_Region *region;
+
+ for (i = 0; i < ct3d->dc.num_regions; i++) {
+ region = &ct3d->dc.regions[i];
+ region->base = region_base;
+ region->decode_len = decode_len;
+ region->len = region_len;
+ region->block_size = blk_size;
+ /* dsmad_handle is set when creating cdat table entries */
+ region->flags = 0;
+
+ region_base += region->len;
+ }
+
+ return 0;
+}
+
static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
{
DeviceState *ds = DEVICE(ct3d);
@@ -775,6 +803,10 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
g_free(p_name);
}
+ if (cxl_create_dc_regions(ct3d)) {
+ return false;
+ }
+
return true;
}
@@ -1062,6 +1094,7 @@ static Property ct3_props[] = {
DEFINE_PROP_UINT64("sn", CXLType3Dev, sn, UI64_NULL),
DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename),
DEFINE_PROP_UINT16("spdm", CXLType3Dev, spdm_port, 0),
+ DEFINE_PROP_UINT8("num-dc-regions", CXLType3Dev, dc.num_regions, 0),
DEFINE_PROP_END_OF_LIST(),
};
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 5/9] hw/mem/cxl_type3: Add host backend and address space handling for DC regions
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (3 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 4/9] hw/mem/cxl_type3: Add support to create DC regions to " nifan
@ 2023-07-20 20:37 ` nifan
2023-07-24 16:29 ` Nathan Fontenot
2023-07-20 20:37 ` [PATCH 6/9] hw/mem/cxl_type3: Add DC extent list representative and get DC extent list mailbox support nifan
` (3 subsequent siblings)
8 siblings, 1 reply; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Add (file/memory backed) host backend, all the dynamic capacity regions
will share a single, large enough host backend. Set up address space for
DC regions to support read/write operations to dynamic capacity for DCD.
With the change, following supports are added:
1. add a new property to type3 device "nonvolatile-dc-memdev" to point to host
memory backend for dynamic capacity;
2. add namespace for dynamic capacity for read/write support;
3. create cdat entries for each dynamic capacity region;
4. fix dvsec range registers to include DC regions.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 19 +++-
hw/mem/cxl_type3.c | 203 +++++++++++++++++++++++++++++-------
include/hw/cxl/cxl_device.h | 4 +
3 files changed, 185 insertions(+), 41 deletions(-)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 59d57ae245..c497298a1d 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -388,9 +388,11 @@ static CXLRetCode cmd_firmware_update_get_info(struct cxl_cmd *cmd,
char fw_rev4[0x10];
} QEMU_PACKED *fw_info;
QEMU_BUILD_BUG_ON(sizeof(*fw_info) != 0x50);
+ CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
if ((cxl_dstate->vmem_size < CXL_CAPACITY_MULTIPLIER) ||
- (cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER)) {
+ (cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER) ||
+ (ct3d->dc.total_capacity < CXL_CAPACITY_MULTIPLIER)) {
return CXL_MBOX_INTERNAL_ERROR;
}
@@ -531,7 +533,8 @@ static CXLRetCode cmd_identify_memory_device(struct cxl_cmd *cmd,
CXLType3Class *cvc = CXL_TYPE3_GET_CLASS(ct3d);
if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
- (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
+ (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER)) ||
+ (!QEMU_IS_ALIGNED(ct3d->dc.total_capacity, CXL_CAPACITY_MULTIPLIER))) {
return CXL_MBOX_INTERNAL_ERROR;
}
@@ -566,9 +569,11 @@ static CXLRetCode cmd_ccls_get_partition_info(struct cxl_cmd *cmd,
uint64_t next_pmem;
} QEMU_PACKED *part_info = (void *)cmd->payload;
QEMU_BUILD_BUG_ON(sizeof(*part_info) != 0x20);
+ CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
- (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
+ (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER)) ||
+ (!QEMU_IS_ALIGNED(ct3d->dc.total_capacity, CXL_CAPACITY_MULTIPLIER))) {
return CXL_MBOX_INTERNAL_ERROR;
}
@@ -880,7 +885,13 @@ static CXLRetCode cmd_media_clear_poison(struct cxl_cmd *cmd,
struct clear_poison_pl *in = (void *)cmd->payload;
dpa = ldq_le_p(&in->dpa);
- if (dpa + CXL_CACHE_LINE_SIZE > cxl_dstate->static_mem_size) {
+ if (dpa + CXL_CACHE_LINE_SIZE >= cxl_dstate->static_mem_size
+ && ct3d->dc.num_regions == 0) {
+ return CXL_MBOX_INVALID_PA;
+ }
+
+ if (ct3d->dc.num_regions && dpa + CXL_CACHE_LINE_SIZE >=
+ cxl_dstate->static_mem_size + ct3d->dc.total_capacity) {
return CXL_MBOX_INVALID_PA;
}
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 27b5920f7d..af1d919be3 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -20,6 +20,7 @@
#include "hw/pci/spdm.h"
#define DWORD_BYTE 4
+#define CXL_CAPACITY_MULTIPLIER (256 * MiB)
/* Default CDAT entries for a memory region */
enum {
@@ -33,8 +34,8 @@ enum {
};
static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
- int dsmad_handle, MemoryRegion *mr,
- bool is_pmem, uint64_t dpa_base)
+ int dsmad_handle, uint8_t flags,
+ uint64_t dpa_base, uint64_t size)
{
g_autofree CDATDsmas *dsmas = NULL;
g_autofree CDATDslbis *dslbis0 = NULL;
@@ -53,9 +54,9 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
.length = sizeof(*dsmas),
},
.DSMADhandle = dsmad_handle,
- .flags = is_pmem ? CDAT_DSMAS_FLAG_NV : 0,
+ .flags = flags,
.DPA_base = dpa_base,
- .DPA_length = memory_region_size(mr),
+ .DPA_length = size,
};
/* For now, no memory side cache, plausiblish numbers */
@@ -137,9 +138,9 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
* NV: Reserved - the non volatile from DSMAS matters
* V: EFI_MEMORY_SP
*/
- .EFI_memory_type_attr = is_pmem ? 2 : 1,
+ .EFI_memory_type_attr = flags ? 2 : 1,
.DPA_offset = 0,
- .DPA_length = memory_region_size(mr),
+ .DPA_length = size,
};
/* Header always at start of structure */
@@ -158,21 +159,28 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
g_autofree CDATSubHeader **table = NULL;
CXLType3Dev *ct3d = priv;
MemoryRegion *volatile_mr = NULL, *nonvolatile_mr = NULL;
+ MemoryRegion *dc_mr = NULL;
int dsmad_handle = 0;
int cur_ent = 0;
int len = 0;
int rc, i;
+ uint64_t vmr_size = 0, pmr_size = 0;
- if (!ct3d->hostpmem && !ct3d->hostvmem) {
+ if (!ct3d->hostpmem && !ct3d->hostvmem && !ct3d->dc.num_regions) {
return 0;
}
+ if (ct3d->hostpmem && ct3d->hostvmem && ct3d->dc.host_dc) {
+ warn_report("The device has static ram and pmem and dynamic capacity");
+ }
+
if (ct3d->hostvmem) {
volatile_mr = host_memory_backend_get_memory(ct3d->hostvmem);
if (!volatile_mr) {
return -EINVAL;
}
len += CT3_CDAT_NUM_ENTRIES;
+ vmr_size = volatile_mr->size;
}
if (ct3d->hostpmem) {
@@ -181,6 +189,19 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
return -EINVAL;
}
len += CT3_CDAT_NUM_ENTRIES;
+ pmr_size = nonvolatile_mr->size;
+ }
+
+ if (ct3d->dc.num_regions) {
+ if (ct3d->dc.host_dc) {
+ dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
+ if (!dc_mr) {
+ return -EINVAL;
+ }
+ len += CT3_CDAT_NUM_ENTRIES * ct3d->dc.num_regions;
+ } else {
+ return -EINVAL;
+ }
}
table = g_malloc0(len * sizeof(*table));
@@ -190,8 +211,8 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
/* Now fill them in */
if (volatile_mr) {
- rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++, volatile_mr,
- false, 0);
+ rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++,
+ 0, 0, vmr_size);
if (rc < 0) {
return rc;
}
@@ -200,14 +221,37 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
if (nonvolatile_mr) {
rc = ct3_build_cdat_entries_for_mr(&(table[cur_ent]), dsmad_handle++,
- nonvolatile_mr, true,
- (volatile_mr ?
- memory_region_size(volatile_mr) : 0));
+ CDAT_DSMAS_FLAG_NV, vmr_size, pmr_size);
if (rc < 0) {
goto error_cleanup;
}
cur_ent += CT3_CDAT_NUM_ENTRIES;
}
+
+ if (dc_mr) {
+ uint64_t region_base = vmr_size + pmr_size;
+
+ /*
+ * Currently we create cdat entries for each region, should we only
+ * create dsmas table instead??
+ * We assume all dc regions are non-volatile for now.
+ *
+ */
+ for (i = 0; i < ct3d->dc.num_regions; i++) {
+ rc = ct3_build_cdat_entries_for_mr(&(table[cur_ent])
+ , dsmad_handle++
+ , CDAT_DSMAS_FLAG_NV | CDAT_DSMAS_FLAG_DYNAMIC_CAP
+ , region_base, ct3d->dc.regions[i].len);
+ if (rc < 0) {
+ goto error_cleanup;
+ }
+ ct3d->dc.regions[i].dsmadhandle = dsmad_handle - 1;
+
+ cur_ent += CT3_CDAT_NUM_ENTRIES;
+ region_base += ct3d->dc.regions[i].len;
+ }
+ }
+
assert(len == cur_ent);
*cdat_table = g_steal_pointer(&table);
@@ -435,11 +479,24 @@ static void build_dvsecs(CXLType3Dev *ct3d)
range2_size_hi = ct3d->hostpmem->size >> 32;
range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostpmem->size & 0xF0000000);
+ } else if (ct3d->dc.host_dc) {
+ range2_size_hi = ct3d->dc.host_dc->size >> 32;
+ range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
+ (ct3d->dc.host_dc->size & 0xF0000000);
}
- } else {
+ } else if (ct3d->hostpmem) {
range1_size_hi = ct3d->hostpmem->size >> 32;
range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostpmem->size & 0xF0000000);
+ if (ct3d->dc.host_dc) {
+ range2_size_hi = ct3d->dc.host_dc->size >> 32;
+ range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
+ (ct3d->dc.host_dc->size & 0xF0000000);
+ }
+ } else {
+ range1_size_hi = ct3d->dc.host_dc->size >> 32;
+ range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
+ (ct3d->dc.host_dc->size & 0xF0000000);
}
dvsec = (uint8_t *)&(CXLDVSECDevice){
@@ -708,7 +765,8 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
}
/*
- * Create a dc region to test "Get Dynamic Capacity Configuration" command.
+ * Create dc regions.
+ * TODO: region parameters are hard coded, may need to change in the future.
*/
static int cxl_create_dc_regions(CXLType3Dev *ct3d)
{
@@ -739,7 +797,8 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
{
DeviceState *ds = DEVICE(ct3d);
- if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem) {
+ if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem
+ && !ct3d->dc.num_regions) {
error_setg(errp, "at least one memdev property must be set");
return false;
} else if (ct3d->hostmem && ct3d->hostpmem) {
@@ -807,6 +866,50 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
return false;
}
+ ct3d->dc.total_capacity = 0;
+ if (ct3d->dc.host_dc) {
+ MemoryRegion *dc_mr;
+ char *dc_name;
+ uint64_t total_region_size = 0;
+ int i;
+
+ dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
+ if (!dc_mr) {
+ error_setg(errp, "dynamic capacity must have backing device");
+ return false;
+ }
+ /* FIXME: set dc as nonvolatile for now */
+ memory_region_set_nonvolatile(dc_mr, true);
+ memory_region_set_enabled(dc_mr, true);
+ host_memory_backend_set_mapped(ct3d->dc.host_dc, true);
+ if (ds->id) {
+ dc_name = g_strdup_printf("cxl-dcd-dpa-dc-space:%s", ds->id);
+ } else {
+ dc_name = g_strdup("cxl-dcd-dpa-dc-space");
+ }
+ address_space_init(&ct3d->dc.host_dc_as, dc_mr, dc_name);
+
+ for (i = 0; i < ct3d->dc.num_regions; i++) {
+ total_region_size += ct3d->dc.regions[i].len;
+ }
+ /* Make sure the host backend is large enough to cover all dc range */
+ if (total_region_size > memory_region_size(dc_mr)) {
+ error_setg(errp,
+ "too small host backend size, increase to %lu MiB or more",
+ total_region_size / 1024 / 1024);
+ return false;
+ }
+
+ if (dc_mr->size % CXL_CAPACITY_MULTIPLIER != 0) {
+ error_setg(errp, "DC region size is unaligned to %lx",
+ CXL_CAPACITY_MULTIPLIER);
+ return false;
+ }
+
+ ct3d->dc.total_capacity = total_region_size;
+ g_free(dc_name);
+ }
+
return true;
}
@@ -916,6 +1019,9 @@ err_release_cdat:
err_free_special_ops:
g_free(regs->special_ops);
err_address_space_free:
+ if (ct3d->dc.host_dc) {
+ address_space_destroy(&ct3d->dc.host_dc_as);
+ }
if (ct3d->hostpmem) {
address_space_destroy(&ct3d->hostpmem_as);
}
@@ -935,6 +1041,9 @@ static void ct3_exit(PCIDevice *pci_dev)
cxl_doe_cdat_release(cxl_cstate);
spdm_sock_fini(ct3d->doe_spdm.socket);
g_free(regs->special_ops);
+ if (ct3d->dc.host_dc) {
+ address_space_destroy(&ct3d->dc.host_dc_as);
+ }
if (ct3d->hostpmem) {
address_space_destroy(&ct3d->hostpmem_as);
}
@@ -999,16 +1108,24 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
AddressSpace **as,
uint64_t *dpa_offset)
{
- MemoryRegion *vmr = NULL, *pmr = NULL;
+ MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
+ uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
if (ct3d->hostvmem) {
vmr = host_memory_backend_get_memory(ct3d->hostvmem);
+ vmr_size = memory_region_size(vmr);
}
if (ct3d->hostpmem) {
pmr = host_memory_backend_get_memory(ct3d->hostpmem);
+ pmr_size = memory_region_size(pmr);
+ }
+ if (ct3d->dc.host_dc) {
+ dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
+ /* Do we want dc_size to be dc_mr->size or not?? */
+ dc_size = ct3d->dc.total_capacity;
}
- if (!vmr && !pmr) {
+ if (!vmr && !pmr && !dc_mr) {
return -ENODEV;
}
@@ -1016,19 +1133,19 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
return -EINVAL;
}
- if (*dpa_offset > ct3d->cxl_dstate.static_mem_size) {
+ if ((*dpa_offset >= vmr_size + pmr_size + dc_size) ||
+ (*dpa_offset >= vmr_size + pmr_size && ct3d->dc.num_regions == 0)) {
return -EINVAL;
}
- if (vmr) {
- if (*dpa_offset < memory_region_size(vmr)) {
- *as = &ct3d->hostvmem_as;
- } else {
- *as = &ct3d->hostpmem_as;
- *dpa_offset -= memory_region_size(vmr);
- }
- } else {
+ if (*dpa_offset < vmr_size) {
+ *as = &ct3d->hostvmem_as;
+ } else if (*dpa_offset < vmr_size + pmr_size) {
*as = &ct3d->hostpmem_as;
+ *dpa_offset -= vmr_size;
+ } else {
+ *as = &ct3d->dc.host_dc_as;
+ *dpa_offset -= (vmr_size + pmr_size);
}
return 0;
@@ -1095,6 +1212,8 @@ static Property ct3_props[] = {
DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename),
DEFINE_PROP_UINT16("spdm", CXLType3Dev, spdm_port, 0),
DEFINE_PROP_UINT8("num-dc-regions", CXLType3Dev, dc.num_regions, 0),
+ DEFINE_PROP_LINK("nonvolatile-dc-memdev", CXLType3Dev, dc.host_dc,
+ TYPE_MEMORY_BACKEND, HostMemoryBackend *),
DEFINE_PROP_END_OF_LIST(),
};
@@ -1161,33 +1280,43 @@ static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size,
static bool set_cacheline(CXLType3Dev *ct3d, uint64_t dpa_offset, uint8_t *data)
{
- MemoryRegion *vmr = NULL, *pmr = NULL;
+ MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
AddressSpace *as;
+ uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
if (ct3d->hostvmem) {
vmr = host_memory_backend_get_memory(ct3d->hostvmem);
+ vmr_size = memory_region_size(vmr);
}
if (ct3d->hostpmem) {
pmr = host_memory_backend_get_memory(ct3d->hostpmem);
+ pmr_size = memory_region_size(pmr);
}
+ if (ct3d->dc.host_dc) {
+ dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
+ dc_size = ct3d->dc.total_capacity;
+ }
- if (!vmr && !pmr) {
+ if (!vmr && !pmr && !dc_mr) {
return false;
}
- if (dpa_offset + CXL_CACHE_LINE_SIZE > ct3d->cxl_dstate.static_mem_size) {
+ if (dpa_offset >= vmr_size + pmr_size + dc_size) {
+ return false;
+ }
+ if (dpa_offset + CXL_CACHE_LINE_SIZE >= vmr_size + pmr_size
+ && ct3d->dc.num_regions == 0) {
return false;
}
- if (vmr) {
- if (dpa_offset < memory_region_size(vmr)) {
- as = &ct3d->hostvmem_as;
- } else {
- as = &ct3d->hostpmem_as;
- dpa_offset -= memory_region_size(vmr);
- }
- } else {
+ if (dpa_offset < vmr_size) {
+ as = &ct3d->hostvmem_as;
+ } else if (dpa_offset < vmr_size + pmr_size) {
as = &ct3d->hostpmem_as;
+ dpa_offset -= vmr->size;
+ } else {
+ as = &ct3d->dc.host_dc_as;
+ dpa_offset -= (vmr_size + pmr_size);
}
address_space_write(as, dpa_offset, MEMTXATTRS_UNSPECIFIED, &data,
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index a32ee6d6ba..ddb24271a8 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -426,6 +426,10 @@ struct CXLType3Dev {
uint64_t poison_list_overflow_ts;
struct dynamic_capacity {
+ HostMemoryBackend *host_dc;
+ AddressSpace host_dc_as;
+ uint64_t total_capacity; /* 256M aligned */
+
uint8_t num_regions; /* 0-8 regions */
struct CXLDCD_Region regions[DCD_MAX_REGION_NUM];
} dc;
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 5/9] hw/mem/cxl_type3: Add host backend and address space handling for DC regions
2023-07-20 20:37 ` [PATCH 5/9] hw/mem/cxl_type3: Add host backend and address space handling for DC regions nifan
@ 2023-07-24 16:29 ` Nathan Fontenot
0 siblings, 0 replies; 10+ messages in thread
From: Nathan Fontenot @ 2023-07-24 16:29 UTC (permalink / raw)
To: nifan, qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni
On 7/20/23 15:37, nifan@outlook.com wrote:
> From: Fan Ni <nifan@outlook.com>
>
> Add (file/memory backed) host backend, all the dynamic capacity regions
> will share a single, large enough host backend. Set up address space for
> DC regions to support read/write operations to dynamic capacity for DCD.
>
> With the change, following supports are added:
> 1. add a new property to type3 device "nonvolatile-dc-memdev" to point to host
> memory backend for dynamic capacity;
> 2. add namespace for dynamic capacity for read/write support;
> 3. create cdat entries for each dynamic capacity region;
> 4. fix dvsec range registers to include DC regions.
>
> Signed-off-by: Fan Ni <fan.ni@samsung.com>
> ---
> hw/cxl/cxl-mailbox-utils.c | 19 +++-
> hw/mem/cxl_type3.c | 203 +++++++++++++++++++++++++++++-------
> include/hw/cxl/cxl_device.h | 4 +
> 3 files changed, 185 insertions(+), 41 deletions(-)
>
> diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
> index 59d57ae245..c497298a1d 100644
> --- a/hw/cxl/cxl-mailbox-utils.c
> +++ b/hw/cxl/cxl-mailbox-utils.c
> @@ -388,9 +388,11 @@ static CXLRetCode cmd_firmware_update_get_info(struct cxl_cmd *cmd,
> char fw_rev4[0x10];
> } QEMU_PACKED *fw_info;
> QEMU_BUILD_BUG_ON(sizeof(*fw_info) != 0x50);
> + CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
>
> if ((cxl_dstate->vmem_size < CXL_CAPACITY_MULTIPLIER) ||
> - (cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER)) {
> + (cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER) ||
> + (ct3d->dc.total_capacity < CXL_CAPACITY_MULTIPLIER)) {
> return CXL_MBOX_INTERNAL_ERROR;
> }
>
> @@ -531,7 +533,8 @@ static CXLRetCode cmd_identify_memory_device(struct cxl_cmd *cmd,
> CXLType3Class *cvc = CXL_TYPE3_GET_CLASS(ct3d);
>
> if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
> - (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
> + (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER)) ||
> + (!QEMU_IS_ALIGNED(ct3d->dc.total_capacity, CXL_CAPACITY_MULTIPLIER))) {
> return CXL_MBOX_INTERNAL_ERROR;
> }
>
> @@ -566,9 +569,11 @@ static CXLRetCode cmd_ccls_get_partition_info(struct cxl_cmd *cmd,
> uint64_t next_pmem;
> } QEMU_PACKED *part_info = (void *)cmd->payload;
> QEMU_BUILD_BUG_ON(sizeof(*part_info) != 0x20);
> + CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
>
> if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
> - (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
> + (!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER)) ||
> + (!QEMU_IS_ALIGNED(ct3d->dc.total_capacity, CXL_CAPACITY_MULTIPLIER))) {
> return CXL_MBOX_INTERNAL_ERROR;
> }
>
> @@ -880,7 +885,13 @@ static CXLRetCode cmd_media_clear_poison(struct cxl_cmd *cmd,
> struct clear_poison_pl *in = (void *)cmd->payload;
>
> dpa = ldq_le_p(&in->dpa);
> - if (dpa + CXL_CACHE_LINE_SIZE > cxl_dstate->static_mem_size) {
> + if (dpa + CXL_CACHE_LINE_SIZE >= cxl_dstate->static_mem_size
> + && ct3d->dc.num_regions == 0) {
> + return CXL_MBOX_INVALID_PA;
> + }
> +
> + if (ct3d->dc.num_regions && dpa + CXL_CACHE_LINE_SIZE >=
> + cxl_dstate->static_mem_size + ct3d->dc.total_capacity) {
> return CXL_MBOX_INVALID_PA;
> }
>
> diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> index 27b5920f7d..af1d919be3 100644
> --- a/hw/mem/cxl_type3.c
> +++ b/hw/mem/cxl_type3.c
> @@ -20,6 +20,7 @@
> #include "hw/pci/spdm.h"
>
> #define DWORD_BYTE 4
> +#define CXL_CAPACITY_MULTIPLIER (256 * MiB)
>
> /* Default CDAT entries for a memory region */
> enum {
> @@ -33,8 +34,8 @@ enum {
> };
>
> static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
> - int dsmad_handle, MemoryRegion *mr,
> - bool is_pmem, uint64_t dpa_base)
> + int dsmad_handle, uint8_t flags,
> + uint64_t dpa_base, uint64_t size)
> {
> g_autofree CDATDsmas *dsmas = NULL;
> g_autofree CDATDslbis *dslbis0 = NULL;
> @@ -53,9 +54,9 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
> .length = sizeof(*dsmas),
> },
> .DSMADhandle = dsmad_handle,
> - .flags = is_pmem ? CDAT_DSMAS_FLAG_NV : 0,
> + .flags = flags,
> .DPA_base = dpa_base,
> - .DPA_length = memory_region_size(mr),
> + .DPA_length = size,
> };
>
> /* For now, no memory side cache, plausiblish numbers */
> @@ -137,9 +138,9 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
> * NV: Reserved - the non volatile from DSMAS matters
> * V: EFI_MEMORY_SP
> */
> - .EFI_memory_type_attr = is_pmem ? 2 : 1,
> + .EFI_memory_type_attr = flags ? 2 : 1,
> .DPA_offset = 0,
> - .DPA_length = memory_region_size(mr),
> + .DPA_length = size,
> };
>
> /* Header always at start of structure */
> @@ -158,21 +159,28 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
> g_autofree CDATSubHeader **table = NULL;
> CXLType3Dev *ct3d = priv;
> MemoryRegion *volatile_mr = NULL, *nonvolatile_mr = NULL;
> + MemoryRegion *dc_mr = NULL;
> int dsmad_handle = 0;
> int cur_ent = 0;
> int len = 0;
> int rc, i;
> + uint64_t vmr_size = 0, pmr_size = 0;
>
> - if (!ct3d->hostpmem && !ct3d->hostvmem) {
> + if (!ct3d->hostpmem && !ct3d->hostvmem && !ct3d->dc.num_regions) {
> return 0;
> }
>
> + if (ct3d->hostpmem && ct3d->hostvmem && ct3d->dc.host_dc) {
> + warn_report("The device has static ram and pmem and dynamic capacity");
> + }
> +
> if (ct3d->hostvmem) {
> volatile_mr = host_memory_backend_get_memory(ct3d->hostvmem);
> if (!volatile_mr) {
> return -EINVAL;
> }
> len += CT3_CDAT_NUM_ENTRIES;
> + vmr_size = volatile_mr->size;
> }
>
> if (ct3d->hostpmem) {
> @@ -181,6 +189,19 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
> return -EINVAL;
> }
> len += CT3_CDAT_NUM_ENTRIES;
> + pmr_size = nonvolatile_mr->size;
> + }
> +
> + if (ct3d->dc.num_regions) {
> + if (ct3d->dc.host_dc) {
> + dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
> + if (!dc_mr) {
> + return -EINVAL;
> + }
> + len += CT3_CDAT_NUM_ENTRIES * ct3d->dc.num_regions;
> + } else {
> + return -EINVAL;
> + }
> }
>
> table = g_malloc0(len * sizeof(*table));
> @@ -190,8 +211,8 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
>
> /* Now fill them in */
> if (volatile_mr) {
> - rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++, volatile_mr,
> - false, 0);
> + rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++,
> + 0, 0, vmr_size);
> if (rc < 0) {
> return rc;
> }
> @@ -200,14 +221,37 @@ static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
>
> if (nonvolatile_mr) {
> rc = ct3_build_cdat_entries_for_mr(&(table[cur_ent]), dsmad_handle++,
> - nonvolatile_mr, true,
> - (volatile_mr ?
> - memory_region_size(volatile_mr) : 0));
> + CDAT_DSMAS_FLAG_NV, vmr_size, pmr_size);
> if (rc < 0) {
> goto error_cleanup;
> }
> cur_ent += CT3_CDAT_NUM_ENTRIES;
> }
> +
> + if (dc_mr) {
> + uint64_t region_base = vmr_size + pmr_size;
> +
> + /*
> + * Currently we create cdat entries for each region, should we only
> + * create dsmas table instead??
> + * We assume all dc regions are non-volatile for now.
> + *
> + */
> + for (i = 0; i < ct3d->dc.num_regions; i++) {
> + rc = ct3_build_cdat_entries_for_mr(&(table[cur_ent])
> + , dsmad_handle++
> + , CDAT_DSMAS_FLAG_NV | CDAT_DSMAS_FLAG_DYNAMIC_CAP
> + , region_base, ct3d->dc.regions[i].len);
> + if (rc < 0) {
> + goto error_cleanup;
> + }
> + ct3d->dc.regions[i].dsmadhandle = dsmad_handle - 1;
> +
> + cur_ent += CT3_CDAT_NUM_ENTRIES;
> + region_base += ct3d->dc.regions[i].len;
> + }
> + }
> +
> assert(len == cur_ent);
>
> *cdat_table = g_steal_pointer(&table);
> @@ -435,11 +479,24 @@ static void build_dvsecs(CXLType3Dev *ct3d)
> range2_size_hi = ct3d->hostpmem->size >> 32;
> range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
> (ct3d->hostpmem->size & 0xF0000000);
> + } else if (ct3d->dc.host_dc) {
> + range2_size_hi = ct3d->dc.host_dc->size >> 32;
> + range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
> + (ct3d->dc.host_dc->size & 0xF0000000);
> }
> - } else {
> + } else if (ct3d->hostpmem) {
> range1_size_hi = ct3d->hostpmem->size >> 32;
> range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
> (ct3d->hostpmem->size & 0xF0000000);
> + if (ct3d->dc.host_dc) {
> + range2_size_hi = ct3d->dc.host_dc->size >> 32;
> + range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
> + (ct3d->dc.host_dc->size & 0xF0000000);
> + }
> + } else {
> + range1_size_hi = ct3d->dc.host_dc->size >> 32;
> + range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
> + (ct3d->dc.host_dc->size & 0xF0000000);
> }
>
> dvsec = (uint8_t *)&(CXLDVSECDevice){
> @@ -708,7 +765,8 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
> }
>
> /*
> - * Create a dc region to test "Get Dynamic Capacity Configuration" command.
> + * Create dc regions.
> + * TODO: region parameters are hard coded, may need to change in the future.
> */
> static int cxl_create_dc_regions(CXLType3Dev *ct3d)
> {
> @@ -739,7 +797,8 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
> {
> DeviceState *ds = DEVICE(ct3d);
>
> - if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem) {
> + if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem
> + && !ct3d->dc.num_regions) {
> error_setg(errp, "at least one memdev property must be set");
> return false;
> } else if (ct3d->hostmem && ct3d->hostpmem) {
> @@ -807,6 +866,50 @@ static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
> return false;
> }
>
> + ct3d->dc.total_capacity = 0;
> + if (ct3d->dc.host_dc) {
> + MemoryRegion *dc_mr;
> + char *dc_name;
> + uint64_t total_region_size = 0;
> + int i;
> +
> + dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
> + if (!dc_mr) {
> + error_setg(errp, "dynamic capacity must have backing device");
> + return false;
> + }
> + /* FIXME: set dc as nonvolatile for now */
> + memory_region_set_nonvolatile(dc_mr, true);
> + memory_region_set_enabled(dc_mr, true);
> + host_memory_backend_set_mapped(ct3d->dc.host_dc, true);
> + if (ds->id) {
> + dc_name = g_strdup_printf("cxl-dcd-dpa-dc-space:%s", ds->id);
> + } else {
> + dc_name = g_strdup("cxl-dcd-dpa-dc-space");
> + }
> + address_space_init(&ct3d->dc.host_dc_as, dc_mr, dc_name);
> +
> + for (i = 0; i < ct3d->dc.num_regions; i++) {
> + total_region_size += ct3d->dc.regions[i].len;
> + }
> + /* Make sure the host backend is large enough to cover all dc range */
> + if (total_region_size > memory_region_size(dc_mr)) {
> + error_setg(errp,
> + "too small host backend size, increase to %lu MiB or more",
> + total_region_size / 1024 / 1024);
> + return false;
> + }
> +
> + if (dc_mr->size % CXL_CAPACITY_MULTIPLIER != 0) {
> + error_setg(errp, "DC region size is unaligned to %lx",
> + CXL_CAPACITY_MULTIPLIER);
> + return false;
> + }
> +
> + ct3d->dc.total_capacity = total_region_size;
> + g_free(dc_name);
Shouldn't this be free'ed in the cases where we return false also?
Looks like you could put the gfree() call right after the address_space_init()
call.
-Nathan Fontenot
> + }
> +
> return true;
> }
>
> @@ -916,6 +1019,9 @@ err_release_cdat:
> err_free_special_ops:
> g_free(regs->special_ops);
> err_address_space_free:
> + if (ct3d->dc.host_dc) {
> + address_space_destroy(&ct3d->dc.host_dc_as);
> + }
> if (ct3d->hostpmem) {
> address_space_destroy(&ct3d->hostpmem_as);
> }
> @@ -935,6 +1041,9 @@ static void ct3_exit(PCIDevice *pci_dev)
> cxl_doe_cdat_release(cxl_cstate);
> spdm_sock_fini(ct3d->doe_spdm.socket);
> g_free(regs->special_ops);
> + if (ct3d->dc.host_dc) {
> + address_space_destroy(&ct3d->dc.host_dc_as);
> + }
> if (ct3d->hostpmem) {
> address_space_destroy(&ct3d->hostpmem_as);
> }
> @@ -999,16 +1108,24 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
> AddressSpace **as,
> uint64_t *dpa_offset)
> {
> - MemoryRegion *vmr = NULL, *pmr = NULL;
> + MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
> + uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
>
> if (ct3d->hostvmem) {
> vmr = host_memory_backend_get_memory(ct3d->hostvmem);
> + vmr_size = memory_region_size(vmr);
> }
> if (ct3d->hostpmem) {
> pmr = host_memory_backend_get_memory(ct3d->hostpmem);
> + pmr_size = memory_region_size(pmr);
> + }
> + if (ct3d->dc.host_dc) {
> + dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
> + /* Do we want dc_size to be dc_mr->size or not?? */
> + dc_size = ct3d->dc.total_capacity;
> }
>
> - if (!vmr && !pmr) {
> + if (!vmr && !pmr && !dc_mr) {
> return -ENODEV;
> }
>
> @@ -1016,19 +1133,19 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
> return -EINVAL;
> }
>
> - if (*dpa_offset > ct3d->cxl_dstate.static_mem_size) {
> + if ((*dpa_offset >= vmr_size + pmr_size + dc_size) ||
> + (*dpa_offset >= vmr_size + pmr_size && ct3d->dc.num_regions == 0)) {
> return -EINVAL;
> }
>
> - if (vmr) {
> - if (*dpa_offset < memory_region_size(vmr)) {
> - *as = &ct3d->hostvmem_as;
> - } else {
> - *as = &ct3d->hostpmem_as;
> - *dpa_offset -= memory_region_size(vmr);
> - }
> - } else {
> + if (*dpa_offset < vmr_size) {
> + *as = &ct3d->hostvmem_as;
> + } else if (*dpa_offset < vmr_size + pmr_size) {
> *as = &ct3d->hostpmem_as;
> + *dpa_offset -= vmr_size;
> + } else {
> + *as = &ct3d->dc.host_dc_as;
> + *dpa_offset -= (vmr_size + pmr_size);
> }
>
> return 0;
> @@ -1095,6 +1212,8 @@ static Property ct3_props[] = {
> DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename),
> DEFINE_PROP_UINT16("spdm", CXLType3Dev, spdm_port, 0),
> DEFINE_PROP_UINT8("num-dc-regions", CXLType3Dev, dc.num_regions, 0),
> + DEFINE_PROP_LINK("nonvolatile-dc-memdev", CXLType3Dev, dc.host_dc,
> + TYPE_MEMORY_BACKEND, HostMemoryBackend *),
> DEFINE_PROP_END_OF_LIST(),
> };
>
> @@ -1161,33 +1280,43 @@ static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size,
>
> static bool set_cacheline(CXLType3Dev *ct3d, uint64_t dpa_offset, uint8_t *data)
> {
> - MemoryRegion *vmr = NULL, *pmr = NULL;
> + MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
> AddressSpace *as;
> + uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
>
> if (ct3d->hostvmem) {
> vmr = host_memory_backend_get_memory(ct3d->hostvmem);
> + vmr_size = memory_region_size(vmr);
> }
> if (ct3d->hostpmem) {
> pmr = host_memory_backend_get_memory(ct3d->hostpmem);
> + pmr_size = memory_region_size(pmr);
> }
> + if (ct3d->dc.host_dc) {
> + dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
> + dc_size = ct3d->dc.total_capacity;
> + }
>
> - if (!vmr && !pmr) {
> + if (!vmr && !pmr && !dc_mr) {
> return false;
> }
>
> - if (dpa_offset + CXL_CACHE_LINE_SIZE > ct3d->cxl_dstate.static_mem_size) {
> + if (dpa_offset >= vmr_size + pmr_size + dc_size) {
> + return false;
> + }
> + if (dpa_offset + CXL_CACHE_LINE_SIZE >= vmr_size + pmr_size
> + && ct3d->dc.num_regions == 0) {
> return false;
> }
>
> - if (vmr) {
> - if (dpa_offset < memory_region_size(vmr)) {
> - as = &ct3d->hostvmem_as;
> - } else {
> - as = &ct3d->hostpmem_as;
> - dpa_offset -= memory_region_size(vmr);
> - }
> - } else {
> + if (dpa_offset < vmr_size) {
> + as = &ct3d->hostvmem_as;
> + } else if (dpa_offset < vmr_size + pmr_size) {
> as = &ct3d->hostpmem_as;
> + dpa_offset -= vmr->size;
> + } else {
> + as = &ct3d->dc.host_dc_as;
> + dpa_offset -= (vmr_size + pmr_size);
> }
>
> address_space_write(as, dpa_offset, MEMTXATTRS_UNSPECIFIED, &data,
> diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
> index a32ee6d6ba..ddb24271a8 100644
> --- a/include/hw/cxl/cxl_device.h
> +++ b/include/hw/cxl/cxl_device.h
> @@ -426,6 +426,10 @@ struct CXLType3Dev {
> uint64_t poison_list_overflow_ts;
>
> struct dynamic_capacity {
> + HostMemoryBackend *host_dc;
> + AddressSpace host_dc_as;
> + uint64_t total_capacity; /* 256M aligned */
> +
> uint8_t num_regions; /* 0-8 regions */
> struct CXLDCD_Region regions[DCD_MAX_REGION_NUM];
> } dc;
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 6/9] hw/mem/cxl_type3: Add DC extent list representative and get DC extent list mailbox support
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (4 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 5/9] hw/mem/cxl_type3: Add host backend and address space handling for DC regions nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 7/9] hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release dynamic capacity response nifan
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Add dynamic capacity extent list representative to the definition of
CXLType3Dev and add get DC extent list mailbox command per
CXL.spec.3.0:.8.2.9.8.9.2.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 71 +++++++++++++++++++++++++++++++++++++
hw/mem/cxl_type3.c | 1 +
include/hw/cxl/cxl_device.h | 23 ++++++++++++
3 files changed, 95 insertions(+)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index c497298a1d..754ab68b78 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -83,6 +83,7 @@ enum {
#define CLEAR_POISON 0x2
DCD_CONFIG = 0x48, /*r3.0: 8.2.9.8.9*/
#define GET_DC_CONFIG 0x0
+ #define GET_DYN_CAP_EXT_LIST 0x1
PHYSICAL_SWITCH = 0x51
#define IDENTIFY_SWITCH_DEVICE 0x0
};
@@ -1018,6 +1019,73 @@ static CXLRetCode cmd_dcd_get_dyn_cap_config(struct cxl_cmd *cmd,
return CXL_MBOX_SUCCESS;
}
+/*
+ * cxl spec 3.0: 8.2.9.8.9.2
+ * Get Dynamic Capacity Extent List (Opcode 4810h)
+ */
+static CXLRetCode cmd_dcd_get_dyn_cap_ext_list(struct cxl_cmd *cmd,
+ CXLDeviceState *cxl_dstate,
+ uint16_t *len)
+{
+ struct get_dyn_cap_ext_list_in_pl {
+ uint32_t extent_cnt;
+ uint32_t start_extent_id;
+ } QEMU_PACKED;
+
+ struct get_dyn_cap_ext_list_out_pl {
+ uint32_t count;
+ uint32_t total_extents;
+ uint32_t generation_num;
+ uint8_t rsvd[4];
+ CXLDCExtent_raw records[];
+ } QEMU_PACKED;
+
+ struct get_dyn_cap_ext_list_in_pl *in = (void *)cmd->payload;
+ struct get_dyn_cap_ext_list_out_pl *out = (void *)cmd->payload;
+ struct CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev,
+ cxl_dstate);
+ uint16_t record_count = 0, i = 0, record_done = 0;
+ CXLDCDExtentList *extent_list = &ct3d->dc.extents;
+ CXLDCD_Extent *ent;
+ uint16_t out_pl_len;
+ uint32_t start_extent_id = in->start_extent_id;
+
+ if (start_extent_id > ct3d->dc.total_extent_count) {
+ return CXL_MBOX_INVALID_INPUT;
+ }
+
+ record_count = MIN(in->extent_cnt,
+ ct3d->dc.total_extent_count - start_extent_id);
+
+ out_pl_len = sizeof(*out) + record_count * sizeof(out->records[0]);
+ /* May need more processing here in the future */
+ assert(out_pl_len <= CXL_MAILBOX_MAX_PAYLOAD_SIZE);
+
+ memset(out, 0, out_pl_len);
+ stl_le_p(&out->count, record_count);
+ stl_le_p(&out->total_extents, ct3d->dc.total_extent_count);
+ stl_le_p(&out->generation_num, ct3d->dc.ext_list_gen_seq);
+
+ if (record_count > 0) {
+ QTAILQ_FOREACH(ent, extent_list, node) {
+ if (i++ < start_extent_id) {
+ continue;
+ }
+ stq_le_p(&out->records[record_done].start_dpa, ent->start_dpa);
+ stq_le_p(&out->records[record_done].len, ent->len);
+ memcpy(&out->records[record_done].tag, ent->tag, 0x10);
+ stw_le_p(&out->records[record_done].shared_seq, ent->shared_seq);
+ record_done++;
+ if (record_done == record_count) {
+ break;
+ }
+ }
+ }
+
+ *len = out_pl_len;
+ return CXL_MBOX_SUCCESS;
+}
+
#define IMMEDIATE_CONFIG_CHANGE (1 << 1)
#define IMMEDIATE_DATA_CHANGE (1 << 2)
#define IMMEDIATE_POLICY_CHANGE (1 << 3)
@@ -1058,6 +1126,9 @@ static struct cxl_cmd cxl_cmd_set[256][256] = {
cmd_media_clear_poison, 72, 0 },
[DCD_CONFIG][GET_DC_CONFIG] = { "DCD_GET_DC_CONFIG",
cmd_dcd_get_dyn_cap_config, 2, 0 },
+ [DCD_CONFIG][GET_DYN_CAP_EXT_LIST] = {
+ "DCD_GET_DYNAMIC_CAPACITY_EXTENT_LIST", cmd_dcd_get_dyn_cap_ext_list,
+ 8, 0 },
};
static struct cxl_cmd cxl_cmd_set_sw[256][256] = {
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index af1d919be3..608063ac52 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -789,6 +789,7 @@ static int cxl_create_dc_regions(CXLType3Dev *ct3d)
region_base += region->len;
}
+ QTAILQ_INIT(&ct3d->dc.extents);
return 0;
}
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index ddb24271a8..a9cfe4e904 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -384,6 +384,25 @@ typedef QLIST_HEAD(, CXLPoison) CXLPoisonList;
#define DCD_MAX_REGION_NUM 8
+typedef struct CXLDCD_Extent_raw {
+ uint64_t start_dpa;
+ uint64_t len;
+ uint8_t tag[0x10];
+ uint16_t shared_seq;
+ uint8_t rsvd[0x6];
+} QEMU_PACKED CXLDCExtent_raw;
+
+typedef struct CXLDCD_Extent {
+ uint64_t start_dpa;
+ uint64_t len;
+ uint8_t tag[0x10];
+ uint16_t shared_seq;
+ uint8_t rsvd[0x6];
+
+ QTAILQ_ENTRY(CXLDCD_Extent) node;
+} CXLDCD_Extent;
+typedef QTAILQ_HEAD(, CXLDCD_Extent) CXLDCDExtentList;
+
typedef struct CXLDCD_Region {
uint64_t base;
uint64_t decode_len; /* in multiples of 256MB */
@@ -432,6 +451,10 @@ struct CXLType3Dev {
uint8_t num_regions; /* 0-8 regions */
struct CXLDCD_Region regions[DCD_MAX_REGION_NUM];
+ CXLDCDExtentList extents;
+
+ uint32_t total_extent_count;
+ uint32_t ext_list_gen_seq;
} dc;
};
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 7/9] hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release dynamic capacity response
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (5 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 6/9] hw/mem/cxl_type3: Add DC extent list representative and get DC extent list mailbox support nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents nifan
2023-07-20 20:37 ` [PATCH 9/9] hw/mem/cxl_type3: Add dpa range validation for accesses to dc regions nifan
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Per CXL spec 3.0, two mailbox commands are implemented:
Add Dynamic Capacity Response (Opcode 4802h) 8.2.9.8.9.3, and
Release Dynamic Capacity (Opcode 4803h) 8.2.9.8.9.4.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/cxl/cxl-mailbox-utils.c | 253 ++++++++++++++++++++++++++++++++++++
include/hw/cxl/cxl_device.h | 3 +-
2 files changed, 255 insertions(+), 1 deletion(-)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 754ab68b78..d547385ba7 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -84,6 +84,8 @@ enum {
DCD_CONFIG = 0x48, /*r3.0: 8.2.9.8.9*/
#define GET_DC_CONFIG 0x0
#define GET_DYN_CAP_EXT_LIST 0x1
+ #define ADD_DYN_CAP_RSP 0x2
+ #define RELEASE_DYN_CAP 0x3
PHYSICAL_SWITCH = 0x51
#define IDENTIFY_SWITCH_DEVICE 0x0
};
@@ -1086,6 +1088,251 @@ static CXLRetCode cmd_dcd_get_dyn_cap_ext_list(struct cxl_cmd *cmd,
return CXL_MBOX_SUCCESS;
}
+/*
+ * Check whether the bits at addr between [nr, nr+size) are all set,
+ * return 1 if all 1s, else return 0
+ */
+static inline int test_bits(const unsigned long *addr, int nr, int size)
+{
+ unsigned long res = find_next_zero_bit(addr, size + nr, nr);
+
+ return (res >= nr + size) ? 1 : 0;
+}
+
+/*
+ * Find dynamic capacity region id based on dpa range [dpa, dpa+len)
+ */
+static uint8_t find_region_id(struct CXLType3Dev *dev, uint64_t dpa,
+ uint64_t len)
+{
+ int8_t i = dev->dc.num_regions - 1;
+
+ while (i > 0 && dpa < dev->dc.regions[i].base) {
+ i--;
+ }
+
+ if (dpa < dev->dc.regions[i].base
+ || dpa + len > dev->dc.regions[i].base + dev->dc.regions[i].len) {
+ return dev->dc.num_regions;
+ }
+
+ return i;
+}
+
+static void insert_extent_to_extent_list(CXLDCDExtentList *list, uint64_t dpa,
+ uint64_t len, uint8_t *tag, uint16_t shared_seq)
+{
+ CXLDCD_Extent *extent;
+ extent = g_new0(CXLDCD_Extent, 1);
+ extent->start_dpa = dpa;
+ extent->len = len;
+ if (tag) {
+ memcpy(extent->tag, tag, 0x10);
+ } else {
+ memset(extent->tag, 0, 0x10);
+ }
+ extent->shared_seq = shared_seq;
+
+ QTAILQ_INSERT_TAIL(list, extent, node);
+}
+
+typedef struct updated_dc_extent_list_in_pl {
+ uint32_t num_entries_updated;
+ uint8_t rsvd[4];
+ struct { /* r3.0: Table 8-130 */
+ uint64_t start_dpa;
+ uint64_t len;
+ uint8_t rsvd[8];
+ } QEMU_PACKED updated_entries[];
+} QEMU_PACKED updated_dc_extent_list_in_pl;
+
+/*
+ * The function only check the input extent list against itself.
+ */
+static CXLRetCode detect_malformed_extent_list(CXLType3Dev *dev,
+ const updated_dc_extent_list_in_pl *in)
+{
+ unsigned long *blk_bitmap;
+ uint64_t min_block_size = dev->dc.regions[0].block_size;
+ struct CXLDCD_Region *region = &dev->dc.regions[0];
+ uint32_t i;
+ uint64_t dpa, len;
+ uint8_t rid;
+ CXLRetCode ret;
+
+ for (i = 1; i < dev->dc.num_regions; i++) {
+ region = &dev->dc.regions[i];
+ if (min_block_size > region->block_size) {
+ min_block_size = region->block_size;
+ }
+ }
+
+ blk_bitmap = bitmap_new((region->len + region->base
+ - dev->dc.regions[0].base) / min_block_size);
+
+ for (i = 0; i < in->num_entries_updated; i++) {
+ dpa = in->updated_entries[i].start_dpa;
+ len = in->updated_entries[i].len;
+
+ rid = find_region_id(dev, dpa, len);
+ if (rid == dev->dc.num_regions) {
+ ret = CXL_MBOX_INVALID_PA;
+ goto out;
+ }
+
+ region = &dev->dc.regions[rid];
+ if (dpa % region->block_size || len % region->block_size) {
+ ret = CXL_MBOX_INVALID_EXTENT_LIST;
+ goto out;
+ }
+ /* the dpa range already covered by some other extents in the list */
+ if (test_bits(blk_bitmap, dpa / min_block_size, len / min_block_size)) {
+ ret = CXL_MBOX_INVALID_EXTENT_LIST;
+ goto out;
+ }
+ bitmap_set(blk_bitmap, dpa / min_block_size, len / min_block_size);
+ }
+
+ ret = CXL_MBOX_SUCCESS;
+
+out:
+ g_free(blk_bitmap);
+ return ret;
+}
+
+/*
+ * cxl spec 3.0: 8.2.9.8.9.3
+ * Add Dynamic Capacity Response (opcode 4802h)
+ * Assume an extent is added only after the response is processed successfully
+ * TODO: for better extent list validation, a better solution would be
+ * maintaining a pending extent list and use it to verify the extent list in
+ * the response.
+ */
+static CXLRetCode cmd_dcd_add_dyn_cap_rsp(struct cxl_cmd *cmd,
+ CXLDeviceState *cxl_dstate, uint16_t *len_unused)
+{
+ updated_dc_extent_list_in_pl *in = (void *)cmd->payload;
+ struct CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev,
+ cxl_dstate);
+ CXLDCDExtentList *extent_list = &ct3d->dc.extents;
+ CXLDCD_Extent *ent;
+ uint32_t i;
+ uint64_t dpa, len;
+ CXLRetCode ret;
+
+ if (in->num_entries_updated == 0) {
+ ret = CXL_MBOX_SUCCESS;
+ goto out;
+ }
+
+ ret = detect_malformed_extent_list(ct3d, in);
+ if (ret != CXL_MBOX_SUCCESS) {
+ goto out;
+ }
+
+ for (i = 0; i < in->num_entries_updated; i++) {
+ dpa = in->updated_entries[i].start_dpa;
+ len = in->updated_entries[i].len;
+
+ /*
+ * Check if the DPA range of the to-be-added extent overlaps with
+ * existing extent list maintained by the device.
+ */
+ QTAILQ_FOREACH(ent, extent_list, node) {
+ if (ent->start_dpa == dpa && ent->len == len) {
+ ret = CXL_MBOX_INVALID_PA;
+ goto out;
+ } else if (ent->start_dpa <= dpa
+ && dpa + len <= ent->start_dpa + ent->len) {
+ ret = CXL_MBOX_INVALID_PA;
+ goto out;
+ } else if ((dpa < ent->start_dpa + ent->len
+ && dpa + len > ent->start_dpa + ent->len)
+ || (dpa < ent->start_dpa && dpa + len > ent->start_dpa)) {
+ ret = CXL_MBOX_INVALID_PA;
+ goto out;
+ }
+ }
+
+ /*
+ * TODO: add a pending extent list based on event log record and verify
+ * the input response
+ */
+
+ insert_extent_to_extent_list(extent_list, dpa, len, NULL, 0);
+ }
+ ret = CXL_MBOX_SUCCESS;
+
+out:
+ return ret;
+}
+
+/*
+ * Spec 3.0: 8.2.9.8.9.4
+ * Release Dynamic Capacity (opcode 4803h)
+ **/
+static CXLRetCode cmd_dcd_release_dyn_cap(struct cxl_cmd *cmd,
+ CXLDeviceState *cxl_dstate,
+ uint16_t *len_unused)
+{
+ updated_dc_extent_list_in_pl *in = (void *)cmd->payload;
+ struct CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev,
+ cxl_dstate);
+ CXLDCDExtentList *extent_list = &ct3d->dc.extents;
+ CXLDCD_Extent *ent;
+ uint32_t i;
+ uint64_t dpa, len;
+ CXLRetCode ret;
+
+ if (in->num_entries_updated == 0) {
+ return CXL_MBOX_INVALID_INPUT;
+ }
+
+ ret = detect_malformed_extent_list(ct3d, in);
+ if (ret != CXL_MBOX_SUCCESS) {
+ return ret;
+ }
+
+ for (i = 0; i < in->num_entries_updated; i++) {
+ dpa = in->updated_entries[i].start_dpa;
+ len = in->updated_entries[i].len;
+
+ QTAILQ_FOREACH(ent, extent_list, node) {
+ if (ent->start_dpa == dpa && ent->len == len) {
+ break;
+ } else if (ent->start_dpa < dpa
+ && dpa + len <= ent->start_dpa + ent->len) {
+ /* remove partial extent */
+ uint64_t len1 = dpa - ent->start_dpa;
+ uint64_t len2 = ent->start_dpa + ent->len - dpa - len;
+
+ if (len1) {
+ insert_extent_to_extent_list(extent_list, ent->start_dpa,
+ len1, NULL, 0);
+ }
+ if (len2) {
+ insert_extent_to_extent_list(extent_list, dpa + len, len2,
+ NULL, 0);
+ }
+ break;
+ } else if ((dpa < ent->start_dpa + ent->len
+ && dpa + len > ent->start_dpa + ent->len)
+ || (dpa < ent->start_dpa && dpa + len > ent->start_dpa))
+ return CXL_MBOX_INVALID_EXTENT_LIST;
+ }
+
+ if (ent) {
+ QTAILQ_REMOVE(extent_list, ent, node);
+ g_free(ent);
+ } else {
+ /* Try to remove a non-existing extent */
+ return CXL_MBOX_INVALID_PA;
+ }
+ }
+
+ return CXL_MBOX_SUCCESS;
+}
+
#define IMMEDIATE_CONFIG_CHANGE (1 << 1)
#define IMMEDIATE_DATA_CHANGE (1 << 2)
#define IMMEDIATE_POLICY_CHANGE (1 << 3)
@@ -1129,6 +1376,12 @@ static struct cxl_cmd cxl_cmd_set[256][256] = {
[DCD_CONFIG][GET_DYN_CAP_EXT_LIST] = {
"DCD_GET_DYNAMIC_CAPACITY_EXTENT_LIST", cmd_dcd_get_dyn_cap_ext_list,
8, 0 },
+ [DCD_CONFIG][ADD_DYN_CAP_RSP] = {
+ "ADD_DCD_DYNAMIC_CAPACITY_RESPONSE", cmd_dcd_add_dyn_cap_rsp,
+ ~0, IMMEDIATE_DATA_CHANGE },
+ [DCD_CONFIG][RELEASE_DYN_CAP] = {
+ "RELEASE_DCD_DYNAMIC_CAPACITY", cmd_dcd_release_dyn_cap,
+ ~0, IMMEDIATE_DATA_CHANGE },
};
static struct cxl_cmd cxl_cmd_set_sw[256][256] = {
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index a9cfe4e904..5bf1dd4024 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -130,7 +130,8 @@ typedef enum {
CXL_MBOX_INCORRECT_PASSPHRASE = 0x14,
CXL_MBOX_UNSUPPORTED_MAILBOX = 0x15,
CXL_MBOX_INVALID_PAYLOAD_LENGTH = 0x16,
- CXL_MBOX_MAX = 0x17
+ CXL_MBOX_INVALID_EXTENT_LIST = 0x1E, /* cxl r3.0: Table 8-34*/
+ CXL_MBOX_MAX = 0x1F
} CXLRetCode;
struct cxl_cmd;
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (6 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 7/9] hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release dynamic capacity response nifan
@ 2023-07-20 20:37 ` nifan
2023-07-20 20:37 ` [PATCH 9/9] hw/mem/cxl_type3: Add dpa range validation for accesses to dc regions nifan
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Since fabric manager emulation is not supported yet, the change implements
the functions to add/release dynamic capacity extents as QMP interfaces.
1. Add dynamic capacity extents:
For example, the command to add two continuous extents (each is 128MB long)
to region 0 (starting at dpa offset 0 and 128MB) looks like below:
{ "execute": "qmp_capabilities" }
{ "execute": "cxl-add-dynamic-capacity-event",
"arguments": {
"path": "/machine/peripheral/cxl-dcd0",
"extents": [
{
"region-id": 0,
"dpa": 0,
"len": 128
},
{
"region-id": 0,
"dpa": 128,
"len": 128
}
]
}
}
2. Release dynamic capacity extents:
For example, the command to release an extent of size 128MB from region 0
(starting at dpa offset 128MB) look like below:
{ "execute": "cxl-release-dynamic-capacity-event",
"arguments": {
"path": "/machine/peripheral/cxl-dcd0",
"extents": [
{
"region-id": 0,
"dpa": 128,
"len": 128
}
]
}
}
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/mem/cxl_type3.c | 145 ++++++++++++++++++++++++++++++++++++
hw/mem/cxl_type3_stubs.c | 6 ++
include/hw/cxl/cxl_events.h | 16 ++++
qapi/cxl.json | 49 ++++++++++++
4 files changed, 216 insertions(+)
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 608063ac52..cb1f9182e6 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -1811,6 +1811,151 @@ void qmp_cxl_inject_memory_module_event(const char *path, CxlEventLog log,
}
}
+static const QemuUUID dynamic_capacity_uuid = {
+ .data = UUID(0xca95afa7, 0xf183, 0x4018, 0x8c, 0x2f,
+ 0x95, 0x26, 0x8e, 0x10, 0x1a, 0x2a),
+};
+
+/*
+ * cxl r3.0: Table 8-47
+ * 00h: add capacity
+ * 01h: release capacity
+ * 02h: forced capacity release
+ * 03h: region configuration updated
+ * 04h: Add capacity response
+ * 05h: capacity released
+ */
+enum DC_Event_Type {
+ DC_EVENT_ADD_CAPACITY,
+ DC_EVENT_RELEASE_CAPACITY,
+ DC_EVENT_FORCED_RELEASE_CAPACITY,
+ DC_EVENT_REGION_CONFIG_UPDATED,
+ DC_EVENT_ADD_CAPACITY_RSP,
+ DC_EVENT_CAPACITY_RELEASED,
+ DC_EVENT_NUM
+};
+
+#define MEM_BLK_SIZE_MB 128
+static void qmp_cxl_process_dynamic_capacity_event(const char *path,
+ CxlEventLog log, enum DC_Event_Type type,
+ uint16_t hid, CXLDCExtentRecordList *records, Error **errp)
+{
+ Object *obj = object_resolve_path(path, NULL);
+ CXLEventDynamicCapacity dCap;
+ CXLEventRecordHdr *hdr = &dCap.hdr;
+ CXLDeviceState *cxlds;
+ CXLType3Dev *dcd;
+ uint8_t flags = 1 << CXL_EVENT_TYPE_INFO;
+ uint32_t num_extents = 0;
+ CXLDCExtentRecordList *list = records;
+ CXLDCExtent_raw *extents;
+ uint64_t dpa, len;
+ uint8_t rid;
+ int i;
+
+ if (!obj) {
+ error_setg(errp, "Unable to resolve path");
+ return;
+ }
+ if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
+ error_setg(errp, "Path not point to a valid CXL type3 device");
+ return;
+ }
+
+ dcd = CXL_TYPE3(obj);
+ cxlds = &dcd->cxl_dstate;
+ memset(&dCap, 0, sizeof(dCap));
+
+ if (!dcd->dc.num_regions) {
+ error_setg(errp, "No dynamic capacity support from the device");
+ return;
+ }
+
+ while (list) {
+ dpa = list->value->dpa * 1024 * 1024;
+ len = list->value->len * 1024 * 1024;
+ rid = list->value->region_id;
+
+ if (rid >= dcd->dc.num_regions) {
+ error_setg(errp, "region id is too large");
+ return;
+ }
+
+ if (dpa % dcd->dc.regions[rid].block_size
+ || len % dcd->dc.regions[rid].block_size) {
+ error_setg(errp, "dpa or len is not aligned to region block size");
+ return;
+ }
+
+ if (dpa + len > dcd->dc.regions[rid].decode_len * 256 * 1024 * 1024) {
+ error_setg(errp, "extent range is beyond the region end");
+ return;
+ }
+
+ num_extents++;
+ list = list->next;
+ }
+
+ i = 0;
+ list = records;
+ extents = g_new0(CXLDCExtent_raw, num_extents);
+ while (list) {
+ dpa = list->value->dpa * 1024 * 1024;
+ len = list->value->len * 1024 * 1024;
+ rid = list->value->region_id;
+
+ extents[i].start_dpa = dpa + dcd->dc.regions[rid].base;
+ extents[i].len = len;
+ memset(extents[i].tag, 0, 0x10);
+ extents[i].shared_seq = 0;
+
+ list = list->next;
+ i++;
+ }
+
+ /*
+ * 8.2.9.1.5
+ * All Dynamic Capacity event records shall set the Event Record
+ * Severity field in the Common Event Record Format to Informational
+ * Event. All Dynamic Capacity related events shall be logged in the
+ * Dynamic Capacity Event Log.
+ */
+ cxl_assign_event_header(hdr, &dynamic_capacity_uuid, flags, sizeof(dCap),
+ cxl_device_get_timestamp(&dcd->cxl_dstate));
+
+ dCap.type = type;
+ stw_le_p(&dCap.host_id, hid);
+ /* only valid for DC_REGION_CONFIG_UPDATED event */
+ dCap.updated_region_id = rid;
+ for (i = 0; i < num_extents; i++) {
+ memcpy(&dCap.dynamic_capacity_extent, &extents[i]
+ , sizeof(CXLDCExtent_raw));
+
+ if (cxl_event_insert(cxlds, CXL_EVENT_TYPE_DYNAMIC_CAP,
+ (CXLEventRecordRaw *)&dCap)) {
+ cxl_event_irq_assert(dcd);
+ }
+ }
+
+ g_free(extents);
+}
+
+void qmp_cxl_add_dynamic_capacity_event(const char *path,
+ struct CXLDCExtentRecordList *records,
+ Error **errp)
+{
+ qmp_cxl_process_dynamic_capacity_event(path, CXL_EVENT_LOG_INFORMATIONAL,
+ DC_EVENT_ADD_CAPACITY, 0, records, errp);
+}
+
+void qmp_cxl_release_dynamic_capacity_event(const char *path,
+ struct CXLDCExtentRecordList *records,
+ Error **errp)
+{
+ qmp_cxl_process_dynamic_capacity_event(path, CXL_EVENT_LOG_INFORMATIONAL,
+ DC_EVENT_RELEASE_CAPACITY, 0, records, errp);
+}
+
static void ct3_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
diff --git a/hw/mem/cxl_type3_stubs.c b/hw/mem/cxl_type3_stubs.c
index f3e4a9fa72..482229f3bd 100644
--- a/hw/mem/cxl_type3_stubs.c
+++ b/hw/mem/cxl_type3_stubs.c
@@ -56,3 +56,9 @@ void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type,
{
error_setg(errp, "CXL Type 3 support is not compiled in");
}
+
+void qmp_cxl_add_dynamic_capacity_event(const char *path,
+ struct CXLDCExtentRecordList *records, Error **errp) {}
+
+void qmp_cxl_release_dynamic_capacity_event(const char *path,
+ struct CXLDCExtentRecordList *records, Error **errp) {}
diff --git a/include/hw/cxl/cxl_events.h b/include/hw/cxl/cxl_events.h
index 089ba2091f..3baf745f8d 100644
--- a/include/hw/cxl/cxl_events.h
+++ b/include/hw/cxl/cxl_events.h
@@ -165,4 +165,20 @@ typedef struct CXLEventMemoryModule {
uint8_t reserved[0x3d];
} QEMU_PACKED CXLEventMemoryModule;
+/*
+ * Dynamic Capacity Event Record
+ * CXL Rev 3.0 Section 8.2.9.2.1.5: Table 8-47
+ * All fields little endian.
+ */
+typedef struct CXLEventDynamicCapacity {
+ CXLEventRecordHdr hdr;
+ uint8_t type;
+ uint8_t reserved1;
+ uint16_t host_id;
+ uint8_t updated_region_id;
+ uint8_t reserved2[3];
+ uint8_t dynamic_capacity_extent[0x28]; /* defined in cxl_device.h */
+ uint8_t reserved[0x20];
+} QEMU_PACKED CXLEventDynamicCapacity;
+
#endif /* CXL_EVENTS_H */
diff --git a/qapi/cxl.json b/qapi/cxl.json
index 05c560cfe5..fb04ec4c41 100644
--- a/qapi/cxl.json
+++ b/qapi/cxl.json
@@ -369,3 +369,52 @@
##
{'command': 'cxl-inject-correctable-error',
'data': {'path': 'str', 'type': 'CxlCorErrorType'}}
+
+##
+# @CXLDCExtentRecord:
+#
+# Record of a single extent to add/release
+#
+# @region-id: id of the region where the extent to add/release
+# @dpa: start dpa (in MiB) of the extent, related to region base address
+# @len: extent size (in MiB)
+#
+# Since: 8.0
+##
+{ 'struct': 'CXLDCExtentRecord',
+ 'data': {
+ 'region-id': 'uint8',
+ 'dpa':'uint64',
+ 'len': 'uint64'
+ }
+}
+
+##
+# @cxl-add-dynamic-capacity-event:
+#
+# Command to add dynamic capacity extent event
+#
+# @path: CXL DCD canonical QOM path
+# @extents: Extents to add
+#
+##
+{ 'command': 'cxl-add-dynamic-capacity-event',
+ 'data': { 'path': 'str',
+ 'extents': [ 'CXLDCExtentRecord' ]
+ }
+}
+
+##
+# @cxl-release-dynamic-capacity-event:
+#
+# Command to release dynamic capacity extent event
+#
+# @path: CXL DCD canonical QOM path
+# @extents: Extents to release
+#
+##
+{ 'command': 'cxl-release-dynamic-capacity-event',
+ 'data': { 'path': 'str',
+ 'extents': [ 'CXLDCExtentRecord' ]
+ }
+}
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 9/9] hw/mem/cxl_type3: Add dpa range validation for accesses to dc regions
[not found] <20230720203708.25825-1-nifan@outlook.com>
` (7 preceding siblings ...)
2023-07-20 20:37 ` [PATCH 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents nifan
@ 2023-07-20 20:37 ` nifan
8 siblings, 0 replies; 10+ messages in thread
From: nifan @ 2023-07-20 20:37 UTC (permalink / raw)
To: qemu-devel
Cc: jonathan.cameron, linux-cxl, gregory.price, hchkuo, cbrowy,
ira.weiny, dan.j.williams, a.manzanares, dave, nmtadam.samsung,
Fan Ni, Fan Ni
From: Fan Ni <nifan@outlook.com>
Not all dpa range in the dc regions is valid to access until an extent
covering the range has been added. Add a bitmap for each region to
record whether a dc block in the region has been backed by dc extent.
For the bitmap, a bit in the bitmap represents a dc block. When a dc
extent is added, all the bits of the blocks in the extent will be set,
which will be cleared when the extent is released.
Signed-off-by: Fan Ni <fan.ni@samsung.com>
---
hw/mem/cxl_type3.c | 155 ++++++++++++++++++++++++++++++++++++
include/hw/cxl/cxl_device.h | 1 +
2 files changed, 156 insertions(+)
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index cb1f9182e6..e673287804 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -787,13 +787,37 @@ static int cxl_create_dc_regions(CXLType3Dev *ct3d)
/* dsmad_handle is set when creating cdat table entries */
region->flags = 0;
+ region->blk_bitmap = bitmap_new(region->len / region->block_size);
+ if (!region->blk_bitmap) {
+ break;
+ }
+
region_base += region->len;
}
+
+ if (i < ct3d->dc.num_regions) {
+ while (--i >= 0) {
+ g_free(ct3d->dc.regions[i].blk_bitmap);
+ }
+ return -1;
+ }
+
QTAILQ_INIT(&ct3d->dc.extents);
return 0;
}
+static void cxl_destroy_dc_regions(CXLType3Dev *ct3d)
+{
+ int i;
+ struct CXLDCD_Region *region;
+
+ for (i = 0; i < ct3d->dc.num_regions; i++) {
+ region = &ct3d->dc.regions[i];
+ g_free(region->blk_bitmap);
+ }
+}
+
static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
{
DeviceState *ds = DEVICE(ct3d);
@@ -1021,6 +1045,7 @@ err_free_special_ops:
g_free(regs->special_ops);
err_address_space_free:
if (ct3d->dc.host_dc) {
+ cxl_destroy_dc_regions(ct3d);
address_space_destroy(&ct3d->dc.host_dc_as);
}
if (ct3d->hostpmem) {
@@ -1043,6 +1068,7 @@ static void ct3_exit(PCIDevice *pci_dev)
spdm_sock_fini(ct3d->doe_spdm.socket);
g_free(regs->special_ops);
if (ct3d->dc.host_dc) {
+ cxl_destroy_dc_regions(ct3d);
address_space_destroy(&ct3d->dc.host_dc_as);
}
if (ct3d->hostpmem) {
@@ -1053,6 +1079,110 @@ static void ct3_exit(PCIDevice *pci_dev)
}
}
+/*
+ * This function will marked the dpa range [dpa, dap + len) to be backed and
+ * accessible, this happens when a dc extent is added and accepted by the
+ * host.
+ */
+static void set_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
+ uint64_t len)
+{
+ int i;
+ CXLDCD_Region *region = &ct3d->dc.regions[0];
+
+ if (dpa < region->base
+ || dpa >= region->base + ct3d->dc.total_capacity)
+ return;
+
+ /*
+ * spec 3.0 9.13.3: Regions are used in increasing-DPA order, with
+ * Region 0 being used for the lowest DPA of Dynamic Capacity and
+ * Region 7 for the highest DPA.
+ * So we check from the last region to find where the dpa belongs.
+ * access across multiple regions is not allowed.
+ **/
+ for (i = ct3d->dc.num_regions - 1; i >= 0; i--) {
+ region = &ct3d->dc.regions[i];
+ if (dpa >= region->base) {
+ break;
+ }
+ }
+
+ bitmap_set(region->blk_bitmap, (dpa - region->base) / region->block_size,
+ len / region->block_size);
+}
+
+/*
+ * This function check whether a dpa range [dpa, dpa + len) has been backed
+ * with dc extents, used when validating read/write to dc regions
+ */
+static bool test_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
+ uint64_t len)
+{
+ int i;
+ CXLDCD_Region *region = &ct3d->dc.regions[0];
+ uint64_t nbits;
+ long nr;
+
+ if (dpa < region->base
+ || dpa >= region->base + ct3d->dc.total_capacity)
+ return false;
+
+ /*
+ * spec 3.0 9.13.3: Regions are used in increasing-DPA order, with
+ * Region 0 being used for the lowest DPA of Dynamic Capacity and
+ * Region 7 for the highest DPA.
+ * So we check from the last region to find where the dpa belongs.
+ * access across multiple regions is not allowed.
+ */
+ for (i = ct3d->dc.num_regions - 1; i >= 0; i--) {
+ region = &ct3d->dc.regions[i];
+ if (dpa >= region->base) {
+ break;
+ }
+ }
+
+ nr = (dpa - region->base) / region->block_size;
+ nbits = len / region->block_size;
+ return find_next_zero_bit(region->blk_bitmap, nbits, nr) >= nr + nbits;
+}
+
+/*
+ * This function will marked the dpa range [dpa, dap + len) to be unbacked and
+ * inaccessible, this happens when a dc extent is added and accepted by the
+ * host.
+ */
+static void clear_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
+ uint64_t len)
+{
+ int i;
+ CXLDCD_Region *region = &ct3d->dc.regions[0];
+ uint64_t nbits;
+ long nr;
+
+ if (dpa < region->base
+ || dpa >= region->base + ct3d->dc.total_capacity)
+ return;
+
+ /*
+ * spec 3.0 9.13.3: Regions are used in increasing-DPA order, with
+ * Region 0 being used for the lowest DPA of Dynamic Capacity and
+ * Region 7 for the highest DPA.
+ * So we check from the last region to find where the dpa belongs.
+ * access across multiple regions is not allowed.
+ */
+ for (i = ct3d->dc.num_regions - 1; i >= 0; i--) {
+ region = &ct3d->dc.regions[i];
+ if (dpa >= region->base) {
+ break;
+ }
+ }
+
+ nr = (dpa - region->base) / region->block_size;
+ nbits = len / region->block_size;
+ bitmap_clear(region->blk_bitmap, nr, nbits);
+}
+
static bool cxl_type3_dpa(CXLType3Dev *ct3d, hwaddr host_addr, uint64_t *dpa)
{
uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers;
@@ -1145,6 +1275,10 @@ static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
*as = &ct3d->hostpmem_as;
*dpa_offset -= vmr_size;
} else {
+ if (!test_region_block_backed(ct3d, *dpa_offset, size)) {
+ return -ENODEV;
+ }
+
*as = &ct3d->dc.host_dc_as;
*dpa_offset -= (vmr_size + pmr_size);
}
@@ -1938,6 +2072,27 @@ static void qmp_cxl_process_dynamic_capacity_event(const char *path,
}
g_free(extents);
+
+ /* Another choice is to do the set/clear after getting mailbox response*/
+ list = records;
+ while (list) {
+ dpa = list->value->dpa * 1024 * 1024;
+ len = list->value->len * 1024 * 1024;
+ rid = list->value->region_id;
+
+ switch (type) {
+ case DC_EVENT_ADD_CAPACITY:
+ set_region_block_backed(dcd, dpa, len);
+ break;
+ case DC_EVENT_RELEASE_CAPACITY:
+ clear_region_block_backed(dcd, dpa, len);
+ break;
+ default:
+ error_setg(errp, "DC event type not handled yet");
+ break;
+ }
+ list = list->next;
+ }
}
void qmp_cxl_add_dynamic_capacity_event(const char *path,
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index 5bf1dd4024..40ae96d824 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -411,6 +411,7 @@ typedef struct CXLDCD_Region {
uint64_t block_size;
uint32_t dsmadhandle;
uint8_t flags;
+ unsigned long *blk_bitmap;
} CXLDCD_Region;
struct CXLType3Dev {
--
2.39.2
^ permalink raw reply related [flat|nested] 10+ messages in thread