* [PATCH 1/2] hw/nvme: generalize the mbar size helper
2024-03-10 11:07 [PATCH 0/2] hw/nvme: fix hibernation-based migration Klaus Jensen
@ 2024-03-10 11:07 ` Klaus Jensen
2024-03-10 11:07 ` [PATCH 2/2] hw/nvme: add machine compatibility parameter to enable msix exclusive bar Klaus Jensen
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Klaus Jensen @ 2024-03-10 11:07 UTC (permalink / raw)
To: Keith Busch, Klaus Jensen, Eduardo Habkost, Marcel Apfelbaum,
Philippe Mathieu-Daudé, Yanan Wang
Cc: Julien Grall, qemu-block, qemu-devel, Klaus Jensen
From: Klaus Jensen <k.jensen@samsung.com>
Generalize the mbar size helper such that it can handle cases where the
MSI-X table and PBA are expected to be in an exclusive bar.
Signed-off-by: Klaus Jensen <k.jensen@samsung.com>
---
hw/nvme/ctrl.c | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
index 76fe0397045b..8cca8a762124 100644
--- a/hw/nvme/ctrl.c
+++ b/hw/nvme/ctrl.c
@@ -8003,13 +8003,18 @@ static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
memory_region_set_enabled(&n->pmr.dev->mr, false);
}
-static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
- unsigned *msix_table_offset,
- unsigned *msix_pba_offset)
+static uint64_t nvme_mbar_size(unsigned total_queues, unsigned total_irqs,
+ unsigned *msix_table_offset,
+ unsigned *msix_pba_offset)
{
- uint64_t bar_size, msix_table_size, msix_pba_size;
+ uint64_t bar_size, msix_table_size;
bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
+
+ if (total_irqs == 0) {
+ goto out;
+ }
+
bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
if (msix_table_offset) {
@@ -8024,11 +8029,10 @@ static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
*msix_pba_offset = bar_size;
}
- msix_pba_size = QEMU_ALIGN_UP(total_irqs, 64) / 8;
- bar_size += msix_pba_size;
+ bar_size += QEMU_ALIGN_UP(total_irqs, 64) / 8;
- bar_size = pow2ceil(bar_size);
- return bar_size;
+out:
+ return pow2ceil(bar_size);
}
static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
@@ -8036,7 +8040,7 @@ static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
uint16_t vf_dev_id = n->params.use_intel_id ?
PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
- uint64_t bar_size = nvme_bar_size(le16_to_cpu(cap->vqfrsm),
+ uint64_t bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm),
le16_to_cpu(cap->vifrsm),
NULL, NULL);
@@ -8098,8 +8102,8 @@ static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
}
/* add one to max_ioqpairs to account for the admin queue pair */
- bar_size = nvme_bar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
- &msix_table_offset, &msix_pba_offset);
+ bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
+ &msix_table_offset, &msix_pba_offset);
memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/2] hw/nvme: add machine compatibility parameter to enable msix exclusive bar
2024-03-10 11:07 [PATCH 0/2] hw/nvme: fix hibernation-based migration Klaus Jensen
2024-03-10 11:07 ` [PATCH 1/2] hw/nvme: generalize the mbar size helper Klaus Jensen
@ 2024-03-10 11:07 ` Klaus Jensen
2024-03-10 11:12 ` [PATCH 0/2] hw/nvme: fix hibernation-based migration Klaus Jensen
2024-03-11 11:15 ` Jesper Wendel Devantier
3 siblings, 0 replies; 5+ messages in thread
From: Klaus Jensen @ 2024-03-10 11:07 UTC (permalink / raw)
To: Keith Busch, Klaus Jensen, Eduardo Habkost, Marcel Apfelbaum,
Philippe Mathieu-Daudé, Yanan Wang
Cc: Julien Grall, qemu-block, qemu-devel, Klaus Jensen
From: Klaus Jensen <k.jensen@samsung.com>
Commit 1901b4967c3f ("hw/block/nvme: move msix table and pba to BAR 0")
moved the MSI-X table and PBA to BAR 0 to make room for enabling CMR and
PMR at the same time. As reported by Julien Grall in #2184, this breaks
migration through system hibernation.
Add a machine compatibility parameter and set it on machines pre 6.0 to
enable the old behavior automatically, restoring the hibernation
migration support.
Reported-by: Julien Grall <julien@xen.org>
Tested-by: Julien Grall <julien@xen.org>
Signed-off-by: Klaus Jensen <k.jensen@samsung.com>
---
hw/core/machine.c | 1 +
hw/nvme/ctrl.c | 51 +++++++++++++++++++++++++++++++++++----------------
hw/nvme/nvme.h | 1 +
3 files changed, 37 insertions(+), 16 deletions(-)
diff --git a/hw/core/machine.c b/hw/core/machine.c
index 9ac5d5389a6c..f3012bca1370 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -100,6 +100,7 @@ GlobalProperty hw_compat_5_2[] = {
{ "PIIX4_PM", "smm-compat", "on"},
{ "virtio-blk-device", "report-discard-granularity", "off" },
{ "virtio-net-pci-base", "vectors", "3"},
+ { "nvme", "msix-exclusive-bar", "on"},
};
const size_t hw_compat_5_2_len = G_N_ELEMENTS(hw_compat_5_2);
diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
index 8cca8a762124..649467723e7e 100644
--- a/hw/nvme/ctrl.c
+++ b/hw/nvme/ctrl.c
@@ -7798,6 +7798,11 @@ static bool nvme_check_params(NvmeCtrl *n, Error **errp)
}
if (n->pmr.dev) {
+ if (params->msix_exclusive_bar) {
+ error_setg(errp, "not enough BARs available to enable PMR");
+ return false;
+ }
+
if (host_memory_backend_is_mapped(n->pmr.dev)) {
error_setg(errp, "can't use already busy memdev: %s",
object_get_canonical_path_component(OBJECT(n->pmr.dev)));
@@ -8101,24 +8106,36 @@ static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
pcie_ari_init(pci_dev, 0x100);
}
- /* add one to max_ioqpairs to account for the admin queue pair */
- bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
- &msix_table_offset, &msix_pba_offset);
-
- memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
- memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
- msix_table_offset);
- memory_region_add_subregion(&n->bar0, 0, &n->iomem);
-
- if (pci_is_vf(pci_dev)) {
- pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
- } else {
+ if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) {
+ bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, 0, NULL, NULL);
+ memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
+ bar_size);
pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
- PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
+ PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
+ ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp);
+ } else {
+ /* add one to max_ioqpairs to account for the admin queue pair */
+ bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1,
+ n->params.msix_qsize, &msix_table_offset,
+ &msix_pba_offset);
+
+ memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
+ memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
+ msix_table_offset);
+ memory_region_add_subregion(&n->bar0, 0, &n->iomem);
+
+ if (pci_is_vf(pci_dev)) {
+ pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
+ } else {
+ pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
+ PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
+ }
+
+ ret = msix_init(pci_dev, n->params.msix_qsize,
+ &n->bar0, 0, msix_table_offset,
+ &n->bar0, 0, msix_pba_offset, 0, errp);
}
- ret = msix_init(pci_dev, n->params.msix_qsize,
- &n->bar0, 0, msix_table_offset,
- &n->bar0, 0, msix_pba_offset, 0, errp);
+
if (ret == -ENOTSUP) {
/* report that msix is not supported, but do not error out */
warn_report_err(*errp);
@@ -8416,6 +8433,8 @@ static Property nvme_props[] = {
params.sriov_max_vi_per_vf, 0),
DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
params.sriov_max_vq_per_vf, 0),
+ DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar,
+ false),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/hw/nvme/nvme.h b/hw/nvme/nvme.h
index 5f2ae7b28b9c..eccf14acc906 100644
--- a/hw/nvme/nvme.h
+++ b/hw/nvme/nvme.h
@@ -522,6 +522,7 @@ typedef struct NvmeParams {
uint16_t sriov_vi_flexible;
uint8_t sriov_max_vq_per_vf;
uint8_t sriov_max_vi_per_vf;
+ bool msix_exclusive_bar;
} NvmeParams;
typedef struct NvmeCtrl {
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread