[PATCH 0/3] update live migration configuration region

[PATCH 0/3] update live migration configuration region

[Longfang Liu]

On the new hardware platform, the configuration register space
of the live migration function is set on the PF, while on the
old platform, this part is placed on the VF.

Longfang Liu (3):
  migration: update BAR space size
  migration: qm updates BAR configuration
  migration: adapt to new migration configuration

 drivers/crypto/hisilicon/qm.c                 |  28 +++
 .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 206 ++++++++++++------
 .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h    |   7 +
 3 files changed, 179 insertions(+), 62 deletions(-)

-- 
2.24.0

[PATCH 1/3] migration: update BAR space size

[Longfang Liu]

On the new hardware platform, the live migration configuration region
is moved from VF to PF. The VF's own configuration space is
restored to the complete 64KB, and there is no need to divide the
size of the BAR configuration space equally.

Signed-off-by: Longfang Liu <liulongfang@huawei.com>
---
 .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 41 +++++++++++++++----
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index 451c639299eb..599905dbb707 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -1195,6 +1195,33 @@ static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
 	return !IS_ERR(pf_qm) ? pf_qm : NULL;
 }
 
+static size_t hisi_acc_get_resource_len(struct vfio_pci_core_device *vdev,
+					unsigned int index)
+{
+	struct hisi_acc_vf_core_device *hisi_acc_vdev =
+			hisi_acc_drvdata(vdev->pdev);
+
+	/*
+	 * ACC VF dev 64KB BAR2 region consists of both functional
+	 * register space and migration control register space, each
+	 * uses 32KB BAR2 region, on the system with more than 64KB
+	 * page size, even if the migration control register space
+	 * is written by VM, it will only affects the VF.
+	 *
+	 * In order to support the live migration function in the
+	 * system with a page size above 64KB, the driver needs
+	 * to ensure that the VF region size is aligned with the
+	 * system page size.
+	 *
+	 * On the new hardware platform, the live migration control register
+	 * has been moved from VF to PF.
+	 */
+	if (hisi_acc_vdev->pf_qm->ver == QM_HW_V3)
+		return (pci_resource_len(vdev->pdev, index) >> 1);
+
+	return pci_resource_len(vdev->pdev, index);
+}
+
 static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
 					size_t count, loff_t *ppos,
 					size_t *new_count)
@@ -1205,8 +1232,9 @@ static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
 
 	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
 		loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
-		resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
+		resource_size_t end;
 
+		end = hisi_acc_get_resource_len(vdev, index);
 		/* Check if access is for migration control region */
 		if (pos >= end)
 			return -EINVAL;
@@ -1227,8 +1255,9 @@ static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
 	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
 	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
 		u64 req_len, pgoff, req_start;
-		resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
+		resource_size_t end;
 
+		end = PAGE_ALIGN(hisi_acc_get_resource_len(vdev, index));
 		req_len = vma->vm_end - vma->vm_start;
 		pgoff = vma->vm_pgoff &
 			((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
@@ -1275,7 +1304,6 @@ static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int
 	if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
 		struct vfio_pci_core_device *vdev =
 			container_of(core_vdev, struct vfio_pci_core_device, vdev);
-		struct pci_dev *pdev = vdev->pdev;
 		struct vfio_region_info info;
 		unsigned long minsz;
 
@@ -1290,12 +1318,7 @@ static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int
 		if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 
-			/*
-			 * ACC VF dev BAR2 region consists of both functional
-			 * register space and migration control register space.
-			 * Report only the functional region to Guest.
-			 */
-			info.size = pci_resource_len(pdev, info.index) / 2;
+			info.size = hisi_acc_get_resource_len(vdev, info.index);
 
 			info.flags = VFIO_REGION_INFO_FLAG_READ |
 					VFIO_REGION_INFO_FLAG_WRITE |
-- 
2.24.0

[PATCH 2/3] migration: qm updates BAR configuration

[Longfang Liu]

On the new hardware platform, the configuration region for the
live migration function of the accelerator device is no longer
placed in the VF, but is instead placed in the PF.

Therefore, the configuration region of the live migration function
needs to be opened when the QM driver is loaded. When the QM driver
is uninstalled, the driver needs to clear this configuration.

Signed-off-by: Longfang Liu <liulongfang@huawei.com>
---
 drivers/crypto/hisilicon/qm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 19c1b5d3c954..75e2e77a8f5b 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -235,6 +235,8 @@
 #define QM_AUTOSUSPEND_DELAY		3000
 
 #define QM_DEV_ALG_MAX_LEN		256
+#define QM_MIG_REGION_SEL		0x100198
+#define QM_MIG_REGION_EN		0x1
 
  /* abnormal status value for stopping queue */
 #define QM_STOP_QUEUE_FAIL		1
@@ -2895,11 +2897,36 @@ static void qm_put_pci_res(struct hisi_qm *qm)
 	pci_release_mem_regions(pdev);
 }
 
+static void hisi_mig_region_clear(struct hisi_qm *qm)
+{
+	u32 val;
+
+	/* Clear migration region set of PF */
+	if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) {
+		val = readl(qm->io_base + QM_MIG_REGION_SEL);
+		val &= ~BIT(0);
+		writel(val, qm->io_base + QM_MIG_REGION_SEL);
+	}
+}
+
+static void hisi_mig_region_enable(struct hisi_qm *qm)
+{
+	u32 val;
+
+	/* Select migration region of PF */
+	if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) {
+		val = readl(qm->io_base + QM_MIG_REGION_SEL);
+		val |= QM_MIG_REGION_EN;
+		writel(val, qm->io_base + QM_MIG_REGION_SEL);
+	}
+}
+
 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
 {
 	struct pci_dev *pdev = qm->pdev;
 
 	pci_free_irq_vectors(pdev);
+	hisi_mig_region_clear(qm);
 	qm_put_pci_res(qm);
 	pci_disable_device(pdev);
 }
@@ -5497,6 +5524,7 @@ int hisi_qm_init(struct hisi_qm *qm)
 		goto err_free_qm_memory;
 
 	qm_cmd_init(qm);
+	hisi_mig_region_enable(qm);
 
 	return 0;
 
-- 
2.24.0

[PATCH 3/3] migration: adapt to new migration configuration

[Longfang Liu]

On the new hardware platform, the migration region is moved from
VF to PF. the driver also needs to be modified accordingly to adapt
to the new hardware device.

Use PF's io base directly on the new hardware platform. and no mmap
operation is required.
If it is on an old platform, the driver needs to be compatible with
the old solution.

Signed-off-by: Longfang Liu <liulongfang@huawei.com>
---
 .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 165 ++++++++++++------
 .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h    |   7 +
 2 files changed, 119 insertions(+), 53 deletions(-)

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index 599905dbb707..cf5a807c2199 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -125,6 +125,72 @@ static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
 	return 0;
 }
 
+static int qm_get_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
+			   struct acc_vf_data *vf_data)
+{
+	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
+	struct device *dev = &qm->pdev->dev;
+	u32 eqc_addr, aeqc_addr;
+	int ret;
+
+	if (qm->ver == QM_HW_V3) {
+		eqc_addr = QM_EQC_DW0;
+		aeqc_addr = QM_AEQC_DW0;
+	} else {
+		eqc_addr = QM_EQC_PF_DW0;
+		aeqc_addr = QM_AEQC_PF_DW0;
+	}
+
+	/* QM_EQC_DW has 7 regs */
+	ret = qm_read_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
+	if (ret) {
+		dev_err(dev, "failed to read QM_EQC_DW\n");
+		return ret;
+	}
+
+	/* QM_AEQC_DW has 7 regs */
+	ret = qm_read_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
+	if (ret) {
+		dev_err(dev, "failed to read QM_AEQC_DW\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int qm_set_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
+			   struct acc_vf_data *vf_data)
+{
+	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
+	struct device *dev = &qm->pdev->dev;
+	u32 eqc_addr, aeqc_addr;
+	int ret;
+
+	if (qm->ver == QM_HW_V3) {
+		eqc_addr = QM_EQC_DW0;
+		aeqc_addr = QM_AEQC_DW0;
+	} else {
+		eqc_addr = QM_EQC_PF_DW0;
+		aeqc_addr = QM_AEQC_PF_DW0;
+	}
+
+	/* QM_EQC_DW has 7 regs */
+	ret = qm_write_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
+	if (ret) {
+		dev_err(dev, "failed to write QM_EQC_DW\n");
+		return ret;
+	}
+
+	/* QM_AEQC_DW has 7 regs */
+	ret = qm_write_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
+	if (ret) {
+		dev_err(dev, "failed to write QM_AEQC_DW\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 {
 	struct device *dev = &qm->pdev->dev;
@@ -167,20 +233,6 @@ static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 		return ret;
 	}
 
-	/* QM_EQC_DW has 7 regs */
-	ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
-	if (ret) {
-		dev_err(dev, "failed to read QM_EQC_DW\n");
-		return ret;
-	}
-
-	/* QM_AEQC_DW has 7 regs */
-	ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
-	if (ret) {
-		dev_err(dev, "failed to read QM_AEQC_DW\n");
-		return ret;
-	}
-
 	return 0;
 }
 
@@ -238,20 +290,6 @@ static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 		return ret;
 	}
 
-	/* QM_EQC_DW has 7 regs */
-	ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
-	if (ret) {
-		dev_err(dev, "failed to write QM_EQC_DW\n");
-		return ret;
-	}
-
-	/* QM_AEQC_DW has 7 regs */
-	ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
-	if (ret) {
-		dev_err(dev, "failed to write QM_AEQC_DW\n");
-		return ret;
-	}
-
 	return 0;
 }
 
@@ -470,6 +508,10 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 		return ret;
 	}
 
+	ret = qm_set_xqc_regs(hisi_acc_vdev, vf_data);
+	if (ret)
+		return ret;
+
 	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
 	if (ret) {
 		dev_err(dev, "set sqc failed\n");
@@ -544,6 +586,10 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 		return ret;
 	}
 
+	ret = qm_get_xqc_regs(hisi_acc_vdev, vf_data);
+	if (ret)
+		return -EINVAL;
+
 	ret = vf_qm_read_data(vf_qm, vf_data);
 	if (ret)
 		return -EINVAL;
@@ -1131,34 +1177,46 @@ static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 {
 	struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
 	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
+	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
 	struct pci_dev *vf_dev = vdev->pdev;
 
-	/*
-	 * ACC VF dev BAR2 region consists of both functional register space
-	 * and migration control register space. For migration to work, we
-	 * need access to both. Hence, we map the entire BAR2 region here.
-	 * But unnecessarily exposing the migration BAR region to the Guest
-	 * has the potential to prevent/corrupt the Guest migration. Hence,
-	 * we restrict access to the migration control space from
-	 * Guest(Please see mmap/ioctl/read/write override functions).
-	 *
-	 * Please note that it is OK to expose the entire VF BAR if migration
-	 * is not supported or required as this cannot affect the ACC PF
-	 * configurations.
-	 *
-	 * Also the HiSilicon ACC VF devices supported by this driver on
-	 * HiSilicon hardware platforms are integrated end point devices
-	 * and the platform lacks the capability to perform any PCIe P2P
-	 * between these devices.
-	 */
+	if (pf_qm->ver == QM_HW_V3) {
+		/*
+		 * ACC VF dev BAR2 region consists of both functional register space
+		 * and migration control register space. For migration to work, we
+		 * need access to both. Hence, we map the entire BAR2 region here.
+		 * But unnecessarily exposing the migration BAR region to the Guest
+		 * has the potential to prevent/corrupt the Guest migration. Hence,
+		 * we restrict access to the migration control space from
+		 * Guest(Please see mmap/ioctl/read/write override functions).
+		 *
+		 * Please note that it is OK to expose the entire VF BAR if migration
+		 * is not supported or required as this cannot affect the ACC PF
+		 * configurations.
+		 *
+		 * Also the HiSilicon ACC VF devices supported by this driver on
+		 * HiSilicon hardware platforms are integrated end point devices
+		 * and the platform lacks the capability to perform any PCIe P2P
+		 * between these devices.
+		 */
 
-	vf_qm->io_base =
-		ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
-			pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
-	if (!vf_qm->io_base)
-		return -EIO;
+		vf_qm->io_base =
+			ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
+				pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
+		if (!vf_qm->io_base)
+			return -EIO;
 
-	vf_qm->fun_type = QM_HW_VF;
+		vf_qm->fun_type = QM_HW_VF;
+		vf_qm->ver = pf_qm->ver;
+	} else {
+		/*
+		 * In the new HW platform, the migration function register space is in BAR2 of PF,
+		 * and each VF occupies 8KB address space.
+		 */
+		vf_qm->io_base = pf_qm->io_base + QM_MIG_REGION_OFFSET +
+				hisi_acc_vdev->vf_id * QM_MIG_REGION_SIZE;
+		vf_qm->fun_type = QM_HW_PF;
+	}
 	vf_qm->pdev = vf_dev;
 	mutex_init(&vf_qm->mailbox_lock);
 
@@ -1488,7 +1546,8 @@ static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
 
 	mutex_lock(&hisi_acc_vdev->open_mutex);
 	hisi_acc_vdev->dev_opened = false;
-	iounmap(vf_qm->io_base);
+	if (vf_qm->ver == QM_HW_V3)
+		iounmap(vf_qm->io_base);
 	mutex_unlock(&hisi_acc_vdev->open_mutex);
 	vfio_pci_core_close_device(core_vdev);
 }
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
index 245d7537b2bc..b01eb54525d3 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
@@ -50,6 +50,13 @@
 #define QM_EQC_DW0		0X8000
 #define QM_AEQC_DW0		0X8020
 
+#define QM_MIG_REGION_OFFSET	0x180000
+#define QM_MIG_REGION_SIZE	0x2000
+
+#define QM_SUB_VERSION_ID		0x100210
+#define QM_EQC_PF_DW0		0x1c00
+#define QM_AEQC_PF_DW0		0x1c20
+
 struct acc_vf_data {
 #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)
 	/* QM match information */
-- 
2.24.0

RE: [PATCH 1/3] migration: update BAR space size

[Shameerali Kolothum Thodi]

> -----Original Message-----
> From: liulongfang <liulongfang@huawei.com>
> Sent: Tuesday, February 18, 2025 2:15 AM
> To: alex.williamson@redhat.com; jgg@nvidia.com; Shameerali Kolothum
> Thodi <shameerali.kolothum.thodi@huawei.com>; Jonathan Cameron
> <jonathan.cameron@huawei.com>
> Cc: kvm@vger.kernel.org; linux-kernel@vger.kernel.org;
> linuxarm@openeuler.org; liulongfang <liulongfang@huawei.com>
> Subject: [PATCH 1/3] migration: update BAR space size
> 
> On the new hardware platform, the live migration configuration region
> is moved from VF to PF. The VF's own configuration space is
> restored to the complete 64KB, and there is no need to divide the
> size of the BAR configuration space equally.
> 
> Signed-off-by: Longfang Liu <liulongfang@huawei.com>
> ---
>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 41 +++++++++++++++----
>  1 file changed, 32 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> index 451c639299eb..599905dbb707 100644
> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> @@ -1195,6 +1195,33 @@ static struct hisi_qm *hisi_acc_get_pf_qm(struct
> pci_dev *pdev)
>  	return !IS_ERR(pf_qm) ? pf_qm : NULL;
>  }
> 
> +static size_t hisi_acc_get_resource_len(struct vfio_pci_core_device *vdev,
> +					unsigned int index)
> +{
> +	struct hisi_acc_vf_core_device *hisi_acc_vdev =
> +			hisi_acc_drvdata(vdev->pdev);
> +
> +	/*
> +	 * ACC VF dev 64KB BAR2 region consists of both functional
> +	 * register space and migration control register space, each
> +	 * uses 32KB BAR2 region, on the system with more than 64KB
> +	 * page size, even if the migration control register space
> +	 * is written by VM, it will only affects the VF.
> +	 *
> +	 * In order to support the live migration function in the
> +	 * system with a page size above 64KB, the driver needs
> +	 * to ensure that the VF region size is aligned with the
> +	 * system page size.

I didn't get this. Are you referring to kernel with 64K page size? And this is 
for new hardware or QM_HW_V3 one?

> +	 *
> +	 * On the new hardware platform, the live migration control register
> +	 * has been moved from VF to PF.
> +	 */
> +	if (hisi_acc_vdev->pf_qm->ver == QM_HW_V3)
> +		return (pci_resource_len(vdev->pdev, index) >> 1);
> +
> +	return pci_resource_len(vdev->pdev, index);
> +}
> +
>  static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
>  					size_t count, loff_t *ppos,
>  					size_t *new_count)
> @@ -1205,8 +1232,9 @@ static int hisi_acc_pci_rw_access_check(struct
> vfio_device *core_vdev,
> 
>  	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
>  		loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
> -		resource_size_t end = pci_resource_len(vdev->pdev, index) /
> 2;
> +		resource_size_t end;
> 
> +		end = hisi_acc_get_resource_len(vdev, index);
>  		/* Check if access is for migration control region */
>  		if (pos >= end)
>  			return -EINVAL;
> @@ -1227,8 +1255,9 @@ static int hisi_acc_vfio_pci_mmap(struct
> vfio_device *core_vdev,
>  	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
>  	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
>  		u64 req_len, pgoff, req_start;
> -		resource_size_t end = pci_resource_len(vdev->pdev, index) /
> 2;
> +		resource_size_t end;
> 
> +		end = PAGE_ALIGN(hisi_acc_get_resource_len(vdev, index));

So here, the whole BAR2 will be mapped to Guest in case of QM_HW_V3 &&
64K kernel as well, right?

Thanks,
Shameer

>  		req_len = vma->vm_end - vma->vm_start;
>  		pgoff = vma->vm_pgoff &
>  			((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
> @@ -1275,7 +1304,6 @@ static long hisi_acc_vfio_pci_ioctl(struct
> vfio_device *core_vdev, unsigned int
>  	if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
>  		struct vfio_pci_core_device *vdev =
>  			container_of(core_vdev, struct vfio_pci_core_device,
> vdev);
> -		struct pci_dev *pdev = vdev->pdev;
>  		struct vfio_region_info info;
>  		unsigned long minsz;
> 
> @@ -1290,12 +1318,7 @@ static long hisi_acc_vfio_pci_ioctl(struct
> vfio_device *core_vdev, unsigned int
>  		if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
>  			info.offset =
> VFIO_PCI_INDEX_TO_OFFSET(info.index);
> 
> -			/*
> -			 * ACC VF dev BAR2 region consists of both
> functional
> -			 * register space and migration control register
> space.
> -			 * Report only the functional region to Guest.
> -			 */
> -			info.size = pci_resource_len(pdev, info.index) / 2;
> +			info.size = hisi_acc_get_resource_len(vdev,
> info.index);
> 
>  			info.flags = VFIO_REGION_INFO_FLAG_READ |
>  					VFIO_REGION_INFO_FLAG_WRITE |
> --
> 2.24.0

RE: [PATCH 3/3] migration: adapt to new migration configuration

[Shameerali Kolothum Thodi]

> -----Original Message-----
> From: liulongfang <liulongfang@huawei.com>
> Sent: Tuesday, February 18, 2025 2:15 AM
> To: alex.williamson@redhat.com; jgg@nvidia.com; Shameerali Kolothum
> Thodi <shameerali.kolothum.thodi@huawei.com>; Jonathan Cameron
> <jonathan.cameron@huawei.com>
> Cc: kvm@vger.kernel.org; linux-kernel@vger.kernel.org;
> linuxarm@openeuler.org; liulongfang <liulongfang@huawei.com>
> Subject: [PATCH 3/3] migration: adapt to new migration configuration
> 
> On the new hardware platform, the migration region is moved from
> VF to PF. the driver also needs to be modified accordingly to adapt
> to the new hardware device.
> 
> Use PF's io base directly on the new hardware platform. and no mmap
> operation is required.
> If it is on an old platform, the driver needs to be compatible with
> the old solution.


I see that you have used "new hardware platform" everywhere and I think
it is better we make it more specific, like QM_HW_V4?

Also it is not clear to me, what happens if you try migration between
QM_HW_V3 <--> QM_HW_V4? Is that going to be success or we are
failing it somewhere? I see only the dev_id check in vf_qm_check_match().

Thanks,
Shameer

> Signed-off-by: Longfang Liu <liulongfang@huawei.com>
> ---
>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 165 ++++++++++++------
>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h    |   7 +
>  2 files changed, 119 insertions(+), 53 deletions(-)
> 
> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> index 599905dbb707..cf5a807c2199 100644
> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> @@ -125,6 +125,72 @@ static int qm_get_cqc(struct hisi_qm *qm, u64
> *addr)
>  	return 0;
>  }
> 
> +static int qm_get_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
> +			   struct acc_vf_data *vf_data)
> +{
> +	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
> +	struct device *dev = &qm->pdev->dev;
> +	u32 eqc_addr, aeqc_addr;
> +	int ret;
> +
> +	if (qm->ver == QM_HW_V3) {
> +		eqc_addr = QM_EQC_DW0;
> +		aeqc_addr = QM_AEQC_DW0;
> +	} else {
> +		eqc_addr = QM_EQC_PF_DW0;
> +		aeqc_addr = QM_AEQC_PF_DW0;
> +	}
> +
> +	/* QM_EQC_DW has 7 regs */
> +	ret = qm_read_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
> +	if (ret) {
> +		dev_err(dev, "failed to read QM_EQC_DW\n");
> +		return ret;
> +	}
> +
> +	/* QM_AEQC_DW has 7 regs */
> +	ret = qm_read_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
> +	if (ret) {
> +		dev_err(dev, "failed to read QM_AEQC_DW\n");
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +static int qm_set_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
> +			   struct acc_vf_data *vf_data)
> +{
> +	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
> +	struct device *dev = &qm->pdev->dev;
> +	u32 eqc_addr, aeqc_addr;
> +	int ret;
> +
> +	if (qm->ver == QM_HW_V3) {
> +		eqc_addr = QM_EQC_DW0;
> +		aeqc_addr = QM_AEQC_DW0;
> +	} else {
> +		eqc_addr = QM_EQC_PF_DW0;
> +		aeqc_addr = QM_AEQC_PF_DW0;
> +	}
> +
> +	/* QM_EQC_DW has 7 regs */
> +	ret = qm_write_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
> +	if (ret) {
> +		dev_err(dev, "failed to write QM_EQC_DW\n");
> +		return ret;
> +	}
> +
> +	/* QM_AEQC_DW has 7 regs */
> +	ret = qm_write_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
> +	if (ret) {
> +		dev_err(dev, "failed to write QM_AEQC_DW\n");
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
>  static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
>  {
>  	struct device *dev = &qm->pdev->dev;
> @@ -167,20 +233,6 @@ static int qm_get_regs(struct hisi_qm *qm, struct
> acc_vf_data *vf_data)
>  		return ret;
>  	}
> 
> -	/* QM_EQC_DW has 7 regs */
> -	ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
> -	if (ret) {
> -		dev_err(dev, "failed to read QM_EQC_DW\n");
> -		return ret;
> -	}
> -
> -	/* QM_AEQC_DW has 7 regs */
> -	ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw,
> 7);
> -	if (ret) {
> -		dev_err(dev, "failed to read QM_AEQC_DW\n");
> -		return ret;
> -	}
> -
>  	return 0;
>  }
> 
> @@ -238,20 +290,6 @@ static int qm_set_regs(struct hisi_qm *qm, struct
> acc_vf_data *vf_data)
>  		return ret;
>  	}
> 
> -	/* QM_EQC_DW has 7 regs */
> -	ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
> -	if (ret) {
> -		dev_err(dev, "failed to write QM_EQC_DW\n");
> -		return ret;
> -	}
> -
> -	/* QM_AEQC_DW has 7 regs */
> -	ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw,
> 7);
> -	if (ret) {
> -		dev_err(dev, "failed to write QM_AEQC_DW\n");
> -		return ret;
> -	}
> -
>  	return 0;
>  }
> 
> @@ -470,6 +508,10 @@ static int vf_qm_load_data(struct
> hisi_acc_vf_core_device *hisi_acc_vdev,
>  		return ret;
>  	}
> 
> +	ret = qm_set_xqc_regs(hisi_acc_vdev, vf_data);
> +	if (ret)
> +		return ret;
> +
>  	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
>  	if (ret) {
>  		dev_err(dev, "set sqc failed\n");
> @@ -544,6 +586,10 @@ static int vf_qm_state_save(struct
> hisi_acc_vf_core_device *hisi_acc_vdev,
>  		return ret;
>  	}
> 
> +	ret = qm_get_xqc_regs(hisi_acc_vdev, vf_data);
> +	if (ret)
> +		return -EINVAL;
> +
>  	ret = vf_qm_read_data(vf_qm, vf_data);
>  	if (ret)
>  		return -EINVAL;
> @@ -1131,34 +1177,46 @@ static int hisi_acc_vf_qm_init(struct
> hisi_acc_vf_core_device *hisi_acc_vdev)
>  {
>  	struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
>  	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
> +	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
>  	struct pci_dev *vf_dev = vdev->pdev;
> 
> -	/*
> -	 * ACC VF dev BAR2 region consists of both functional register space
> -	 * and migration control register space. For migration to work, we
> -	 * need access to both. Hence, we map the entire BAR2 region here.
> -	 * But unnecessarily exposing the migration BAR region to the Guest
> -	 * has the potential to prevent/corrupt the Guest migration. Hence,
> -	 * we restrict access to the migration control space from
> -	 * Guest(Please see mmap/ioctl/read/write override functions).
> -	 *
> -	 * Please note that it is OK to expose the entire VF BAR if migration
> -	 * is not supported or required as this cannot affect the ACC PF
> -	 * configurations.
> -	 *
> -	 * Also the HiSilicon ACC VF devices supported by this driver on
> -	 * HiSilicon hardware platforms are integrated end point devices
> -	 * and the platform lacks the capability to perform any PCIe P2P
> -	 * between these devices.
> -	 */
> +	if (pf_qm->ver == QM_HW_V3) {
> +		/*
> +		 * ACC VF dev BAR2 region consists of both functional
> register space
> +		 * and migration control register space. For migration to
> work, we
> +		 * need access to both. Hence, we map the entire BAR2
> region here.
> +		 * But unnecessarily exposing the migration BAR region to
> the Guest
> +		 * has the potential to prevent/corrupt the Guest migration.
> Hence,
> +		 * we restrict access to the migration control space from
> +		 * Guest(Please see mmap/ioctl/read/write override
> functions).
> +		 *
> +		 * Please note that it is OK to expose the entire VF BAR if
> migration
> +		 * is not supported or required as this cannot affect the ACC
> PF
> +		 * configurations.
> +		 *
> +		 * Also the HiSilicon ACC VF devices supported by this driver
> on
> +		 * HiSilicon hardware platforms are integrated end point
> devices
> +		 * and the platform lacks the capability to perform any PCIe
> P2P
> +		 * between these devices.
> +		 */
> 
> -	vf_qm->io_base =
> -		ioremap(pci_resource_start(vf_dev,
> VFIO_PCI_BAR2_REGION_INDEX),
> -			pci_resource_len(vf_dev,
> VFIO_PCI_BAR2_REGION_INDEX));
> -	if (!vf_qm->io_base)
> -		return -EIO;
> +		vf_qm->io_base =
> +			ioremap(pci_resource_start(vf_dev,
> VFIO_PCI_BAR2_REGION_INDEX),
> +				pci_resource_len(vf_dev,
> VFIO_PCI_BAR2_REGION_INDEX));
> +		if (!vf_qm->io_base)
> +			return -EIO;
> 
> -	vf_qm->fun_type = QM_HW_VF;
> +		vf_qm->fun_type = QM_HW_VF;
> +		vf_qm->ver = pf_qm->ver;
> +	} else {
> +		/*
> +		 * In the new HW platform, the migration function register
> space is in BAR2 of PF,
> +		 * and each VF occupies 8KB address space.
> +		 */
> +		vf_qm->io_base = pf_qm->io_base +
> QM_MIG_REGION_OFFSET +
> +				hisi_acc_vdev->vf_id *
> QM_MIG_REGION_SIZE;
> +		vf_qm->fun_type = QM_HW_PF;
> +	}
>  	vf_qm->pdev = vf_dev;
>  	mutex_init(&vf_qm->mailbox_lock);
> 
> @@ -1488,7 +1546,8 @@ static void hisi_acc_vfio_pci_close_device(struct
> vfio_device *core_vdev)
> 
>  	mutex_lock(&hisi_acc_vdev->open_mutex);
>  	hisi_acc_vdev->dev_opened = false;
> -	iounmap(vf_qm->io_base);
> +	if (vf_qm->ver == QM_HW_V3)
> +		iounmap(vf_qm->io_base);
>  	mutex_unlock(&hisi_acc_vdev->open_mutex);
>  	vfio_pci_core_close_device(core_vdev);
>  }
> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
> index 245d7537b2bc..b01eb54525d3 100644
> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
> @@ -50,6 +50,13 @@
>  #define QM_EQC_DW0		0X8000
>  #define QM_AEQC_DW0		0X8020
> 
> +#define QM_MIG_REGION_OFFSET	0x180000
> +#define QM_MIG_REGION_SIZE	0x2000
> +
> +#define QM_SUB_VERSION_ID		0x100210
> +#define QM_EQC_PF_DW0		0x1c00
> +#define QM_AEQC_PF_DW0		0x1c20
> +
>  struct acc_vf_data {
>  #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)
>  	/* QM match information */
> --
> 2.24.0

Re: [PATCH 1/3] migration: update BAR space size

[liulongfang]

On 2025/2/18 18:20, Shameerali Kolothum Thodi wrote:
> 
> 
>> -----Original Message-----
>> From: liulongfang <liulongfang@huawei.com>
>> Sent: Tuesday, February 18, 2025 2:15 AM
>> To: alex.williamson@redhat.com; jgg@nvidia.com; Shameerali Kolothum
>> Thodi <shameerali.kolothum.thodi@huawei.com>; Jonathan Cameron
>> <jonathan.cameron@huawei.com>
>> Cc: kvm@vger.kernel.org; linux-kernel@vger.kernel.org;
>> linuxarm@openeuler.org; liulongfang <liulongfang@huawei.com>
>> Subject: [PATCH 1/3] migration: update BAR space size
>>
>> On the new hardware platform, the live migration configuration region
>> is moved from VF to PF. The VF's own configuration space is
>> restored to the complete 64KB, and there is no need to divide the
>> size of the BAR configuration space equally.
>>
>> Signed-off-by: Longfang Liu <liulongfang@huawei.com>
>> ---
>>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 41 +++++++++++++++----
>>  1 file changed, 32 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> index 451c639299eb..599905dbb707 100644
>> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> @@ -1195,6 +1195,33 @@ static struct hisi_qm *hisi_acc_get_pf_qm(struct
>> pci_dev *pdev)
>>  	return !IS_ERR(pf_qm) ? pf_qm : NULL;
>>  }
>>
>> +static size_t hisi_acc_get_resource_len(struct vfio_pci_core_device *vdev,
>> +					unsigned int index)
>> +{
>> +	struct hisi_acc_vf_core_device *hisi_acc_vdev =
>> +			hisi_acc_drvdata(vdev->pdev);
>> +
>> +	/*
>> +	 * ACC VF dev 64KB BAR2 region consists of both functional
>> +	 * register space and migration control register space, each
>> +	 * uses 32KB BAR2 region, on the system with more than 64KB
>> +	 * page size, even if the migration control register space
>> +	 * is written by VM, it will only affects the VF.
>> +	 *
>> +	 * In order to support the live migration function in the
>> +	 * system with a page size above 64KB, the driver needs
>> +	 * to ensure that the VF region size is aligned with the
>> +	 * system page size.
> 
> I didn't get this. Are you referring to kernel with 64K page size? And this is 
> for new hardware or QM_HW_V3 one?
>

This 64KB page refers to the 64KB page size of BAR2.
If the configuration space of the live migration device is not
aligned to the 64KB page (for example, only 32KB on the QM_HW_V3 platform),
when the host kernel page is 64KB, multiple VF devices will fail during VFIO
enumeration.
Therefore, on the new hardware platform, the live migration configuration
space is moved to the PF, which can align the 64KB memory space and solve
this problem.

>> +	 *
>> +	 * On the new hardware platform, the live migration control register
>> +	 * has been moved from VF to PF.
>> +	 */
>> +	if (hisi_acc_vdev->pf_qm->ver == QM_HW_V3)
>> +		return (pci_resource_len(vdev->pdev, index) >> 1);
>> +
>> +	return pci_resource_len(vdev->pdev, index);
>> +}
>> +
>>  static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
>>  					size_t count, loff_t *ppos,
>>  					size_t *new_count)
>> @@ -1205,8 +1232,9 @@ static int hisi_acc_pci_rw_access_check(struct
>> vfio_device *core_vdev,
>>
>>  	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
>>  		loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
>> -		resource_size_t end = pci_resource_len(vdev->pdev, index) /
>> 2;
>> +		resource_size_t end;
>>
>> +		end = hisi_acc_get_resource_len(vdev, index);
>>  		/* Check if access is for migration control region */
>>  		if (pos >= end)
>>  			return -EINVAL;
>> @@ -1227,8 +1255,9 @@ static int hisi_acc_vfio_pci_mmap(struct
>> vfio_device *core_vdev,
>>  	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
>>  	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
>>  		u64 req_len, pgoff, req_start;
>> -		resource_size_t end = pci_resource_len(vdev->pdev, index) /
>> 2;
>> +		resource_size_t end;
>>
>> +		end = PAGE_ALIGN(hisi_acc_get_resource_len(vdev, index));
> 
> So here, the whole BAR2 will be mapped to Guest in case of QM_HW_V3 &&
> 64K kernel as well, right?
>


Here, the length of BAR2 is mapped to the Guest OS according to the actual
situation of the platform. If it is QM_HW_V3, only 32KB is mapped.
Others are mapped 64KB.

Thanks.

> Thanks,
> Shameer
> 
>>  		req_len = vma->vm_end - vma->vm_start;
>>  		pgoff = vma->vm_pgoff &
>>  			((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
>> @@ -1275,7 +1304,6 @@ static long hisi_acc_vfio_pci_ioctl(struct
>> vfio_device *core_vdev, unsigned int
>>  	if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
>>  		struct vfio_pci_core_device *vdev =
>>  			container_of(core_vdev, struct vfio_pci_core_device,
>> vdev);
>> -		struct pci_dev *pdev = vdev->pdev;
>>  		struct vfio_region_info info;
>>  		unsigned long minsz;
>>
>> @@ -1290,12 +1318,7 @@ static long hisi_acc_vfio_pci_ioctl(struct
>> vfio_device *core_vdev, unsigned int
>>  		if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
>>  			info.offset =
>> VFIO_PCI_INDEX_TO_OFFSET(info.index);
>>
>> -			/*
>> -			 * ACC VF dev BAR2 region consists of both
>> functional
>> -			 * register space and migration control register
>> space.
>> -			 * Report only the functional region to Guest.
>> -			 */
>> -			info.size = pci_resource_len(pdev, info.index) / 2;
>> +			info.size = hisi_acc_get_resource_len(vdev,
>> info.index);
>>
>>  			info.flags = VFIO_REGION_INFO_FLAG_READ |
>>  					VFIO_REGION_INFO_FLAG_WRITE |
>> --
>> 2.24.0
> 
> .
> 

Re: [PATCH 3/3] migration: adapt to new migration configuration

[liulongfang]

On 2025/2/18 18:58, Shameerali Kolothum Thodi wrote:
> 
> 
>> -----Original Message-----
>> From: liulongfang <liulongfang@huawei.com>
>> Sent: Tuesday, February 18, 2025 2:15 AM
>> To: alex.williamson@redhat.com; jgg@nvidia.com; Shameerali Kolothum
>> Thodi <shameerali.kolothum.thodi@huawei.com>; Jonathan Cameron
>> <jonathan.cameron@huawei.com>
>> Cc: kvm@vger.kernel.org; linux-kernel@vger.kernel.org;
>> linuxarm@openeuler.org; liulongfang <liulongfang@huawei.com>
>> Subject: [PATCH 3/3] migration: adapt to new migration configuration
>>
>> On the new hardware platform, the migration region is moved from
>> VF to PF. the driver also needs to be modified accordingly to adapt
>> to the new hardware device.
>>
>> Use PF's io base directly on the new hardware platform. and no mmap
>> operation is required.
>> If it is on an old platform, the driver needs to be compatible with
>> the old solution.
> 
> 
> I see that you have used "new hardware platform" everywhere and I think
> it is better we make it more specific, like QM_HW_V4?
> 
> Also it is not clear to me, what happens if you try migration between
> QM_HW_V3 <--> QM_HW_V4? Is that going to be success or we are
> failing it somewhere? I see only the dev_id check in vf_qm_check_match().
> 

The hardware platforms that support live migration are begin from QM_HW_V3.
Before that, there is no support for live migration. After that, all migration
configuration registers  will be placed in the PF configuration space.
Therefore.so there is no need to make special configurations for QM_HW_V4.

Thanks.
Longfang.

> Thanks,
> Shameer
> 
>> Signed-off-by: Longfang Liu <liulongfang@huawei.com>
>> ---
>>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c    | 165 ++++++++++++------
>>  .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h    |   7 +
>>  2 files changed, 119 insertions(+), 53 deletions(-)
>>
>> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> index 599905dbb707..cf5a807c2199 100644
>> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
>> @@ -125,6 +125,72 @@ static int qm_get_cqc(struct hisi_qm *qm, u64
>> *addr)
>>  	return 0;
>>  }
>>
>> +static int qm_get_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
>> +			   struct acc_vf_data *vf_data)
>> +{
>> +	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
>> +	struct device *dev = &qm->pdev->dev;
>> +	u32 eqc_addr, aeqc_addr;
>> +	int ret;
>> +
>> +	if (qm->ver == QM_HW_V3) {
>> +		eqc_addr = QM_EQC_DW0;
>> +		aeqc_addr = QM_AEQC_DW0;
>> +	} else {
>> +		eqc_addr = QM_EQC_PF_DW0;
>> +		aeqc_addr = QM_AEQC_PF_DW0;
>> +	}
>> +
>> +	/* QM_EQC_DW has 7 regs */
>> +	ret = qm_read_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
>> +	if (ret) {
>> +		dev_err(dev, "failed to read QM_EQC_DW\n");
>> +		return ret;
>> +	}
>> +
>> +	/* QM_AEQC_DW has 7 regs */
>> +	ret = qm_read_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
>> +	if (ret) {
>> +		dev_err(dev, "failed to read QM_AEQC_DW\n");
>> +		return ret;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static int qm_set_xqc_regs(struct hisi_acc_vf_core_device *hisi_acc_vdev,
>> +			   struct acc_vf_data *vf_data)
>> +{
>> +	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
>> +	struct device *dev = &qm->pdev->dev;
>> +	u32 eqc_addr, aeqc_addr;
>> +	int ret;
>> +
>> +	if (qm->ver == QM_HW_V3) {
>> +		eqc_addr = QM_EQC_DW0;
>> +		aeqc_addr = QM_AEQC_DW0;
>> +	} else {
>> +		eqc_addr = QM_EQC_PF_DW0;
>> +		aeqc_addr = QM_AEQC_PF_DW0;
>> +	}
>> +
>> +	/* QM_EQC_DW has 7 regs */
>> +	ret = qm_write_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
>> +	if (ret) {
>> +		dev_err(dev, "failed to write QM_EQC_DW\n");
>> +		return ret;
>> +	}
>> +
>> +	/* QM_AEQC_DW has 7 regs */
>> +	ret = qm_write_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
>> +	if (ret) {
>> +		dev_err(dev, "failed to write QM_AEQC_DW\n");
>> +		return ret;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>>  static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
>>  {
>>  	struct device *dev = &qm->pdev->dev;
>> @@ -167,20 +233,6 @@ static int qm_get_regs(struct hisi_qm *qm, struct
>> acc_vf_data *vf_data)
>>  		return ret;
>>  	}
>>
>> -	/* QM_EQC_DW has 7 regs */
>> -	ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
>> -	if (ret) {
>> -		dev_err(dev, "failed to read QM_EQC_DW\n");
>> -		return ret;
>> -	}
>> -
>> -	/* QM_AEQC_DW has 7 regs */
>> -	ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw,
>> 7);
>> -	if (ret) {
>> -		dev_err(dev, "failed to read QM_AEQC_DW\n");
>> -		return ret;
>> -	}
>> -
>>  	return 0;
>>  }
>>
>> @@ -238,20 +290,6 @@ static int qm_set_regs(struct hisi_qm *qm, struct
>> acc_vf_data *vf_data)
>>  		return ret;
>>  	}
>>
>> -	/* QM_EQC_DW has 7 regs */
>> -	ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
>> -	if (ret) {
>> -		dev_err(dev, "failed to write QM_EQC_DW\n");
>> -		return ret;
>> -	}
>> -
>> -	/* QM_AEQC_DW has 7 regs */
>> -	ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw,
>> 7);
>> -	if (ret) {
>> -		dev_err(dev, "failed to write QM_AEQC_DW\n");
>> -		return ret;
>> -	}
>> -
>>  	return 0;
>>  }
>>
>> @@ -470,6 +508,10 @@ static int vf_qm_load_data(struct
>> hisi_acc_vf_core_device *hisi_acc_vdev,
>>  		return ret;
>>  	}
>>
>> +	ret = qm_set_xqc_regs(hisi_acc_vdev, vf_data);
>> +	if (ret)
>> +		return ret;
>> +
>>  	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
>>  	if (ret) {
>>  		dev_err(dev, "set sqc failed\n");
>> @@ -544,6 +586,10 @@ static int vf_qm_state_save(struct
>> hisi_acc_vf_core_device *hisi_acc_vdev,
>>  		return ret;
>>  	}
>>
>> +	ret = qm_get_xqc_regs(hisi_acc_vdev, vf_data);
>> +	if (ret)
>> +		return -EINVAL;
>> +
>>  	ret = vf_qm_read_data(vf_qm, vf_data);
>>  	if (ret)
>>  		return -EINVAL;
>> @@ -1131,34 +1177,46 @@ static int hisi_acc_vf_qm_init(struct
>> hisi_acc_vf_core_device *hisi_acc_vdev)
>>  {
>>  	struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
>>  	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
>> +	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
>>  	struct pci_dev *vf_dev = vdev->pdev;
>>
>> -	/*
>> -	 * ACC VF dev BAR2 region consists of both functional register space
>> -	 * and migration control register space. For migration to work, we
>> -	 * need access to both. Hence, we map the entire BAR2 region here.
>> -	 * But unnecessarily exposing the migration BAR region to the Guest
>> -	 * has the potential to prevent/corrupt the Guest migration. Hence,
>> -	 * we restrict access to the migration control space from
>> -	 * Guest(Please see mmap/ioctl/read/write override functions).
>> -	 *
>> -	 * Please note that it is OK to expose the entire VF BAR if migration
>> -	 * is not supported or required as this cannot affect the ACC PF
>> -	 * configurations.
>> -	 *
>> -	 * Also the HiSilicon ACC VF devices supported by this driver on
>> -	 * HiSilicon hardware platforms are integrated end point devices
>> -	 * and the platform lacks the capability to perform any PCIe P2P
>> -	 * between these devices.
>> -	 */
>> +	if (pf_qm->ver == QM_HW_V3) {
>> +		/*
>> +		 * ACC VF dev BAR2 region consists of both functional
>> register space
>> +		 * and migration control register space. For migration to
>> work, we
>> +		 * need access to both. Hence, we map the entire BAR2
>> region here.
>> +		 * But unnecessarily exposing the migration BAR region to
>> the Guest
>> +		 * has the potential to prevent/corrupt the Guest migration.
>> Hence,
>> +		 * we restrict access to the migration control space from
>> +		 * Guest(Please see mmap/ioctl/read/write override
>> functions).
>> +		 *
>> +		 * Please note that it is OK to expose the entire VF BAR if
>> migration
>> +		 * is not supported or required as this cannot affect the ACC
>> PF
>> +		 * configurations.
>> +		 *
>> +		 * Also the HiSilicon ACC VF devices supported by this driver
>> on
>> +		 * HiSilicon hardware platforms are integrated end point
>> devices
>> +		 * and the platform lacks the capability to perform any PCIe
>> P2P
>> +		 * between these devices.
>> +		 */
>>
>> -	vf_qm->io_base =
>> -		ioremap(pci_resource_start(vf_dev,
>> VFIO_PCI_BAR2_REGION_INDEX),
>> -			pci_resource_len(vf_dev,
>> VFIO_PCI_BAR2_REGION_INDEX));
>> -	if (!vf_qm->io_base)
>> -		return -EIO;
>> +		vf_qm->io_base =
>> +			ioremap(pci_resource_start(vf_dev,
>> VFIO_PCI_BAR2_REGION_INDEX),
>> +				pci_resource_len(vf_dev,
>> VFIO_PCI_BAR2_REGION_INDEX));
>> +		if (!vf_qm->io_base)
>> +			return -EIO;
>>
>> -	vf_qm->fun_type = QM_HW_VF;
>> +		vf_qm->fun_type = QM_HW_VF;
>> +		vf_qm->ver = pf_qm->ver;
>> +	} else {
>> +		/*
>> +		 * In the new HW platform, the migration function register
>> space is in BAR2 of PF,
>> +		 * and each VF occupies 8KB address space.
>> +		 */
>> +		vf_qm->io_base = pf_qm->io_base +
>> QM_MIG_REGION_OFFSET +
>> +				hisi_acc_vdev->vf_id *
>> QM_MIG_REGION_SIZE;
>> +		vf_qm->fun_type = QM_HW_PF;
>> +	}
>>  	vf_qm->pdev = vf_dev;
>>  	mutex_init(&vf_qm->mailbox_lock);
>>
>> @@ -1488,7 +1546,8 @@ static void hisi_acc_vfio_pci_close_device(struct
>> vfio_device *core_vdev)
>>
>>  	mutex_lock(&hisi_acc_vdev->open_mutex);
>>  	hisi_acc_vdev->dev_opened = false;
>> -	iounmap(vf_qm->io_base);
>> +	if (vf_qm->ver == QM_HW_V3)
>> +		iounmap(vf_qm->io_base);
>>  	mutex_unlock(&hisi_acc_vdev->open_mutex);
>>  	vfio_pci_core_close_device(core_vdev);
>>  }
>> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
>> b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
>> index 245d7537b2bc..b01eb54525d3 100644
>> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
>> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
>> @@ -50,6 +50,13 @@
>>  #define QM_EQC_DW0		0X8000
>>  #define QM_AEQC_DW0		0X8020
>>
>> +#define QM_MIG_REGION_OFFSET	0x180000
>> +#define QM_MIG_REGION_SIZE	0x2000
>> +
>> +#define QM_SUB_VERSION_ID		0x100210
>> +#define QM_EQC_PF_DW0		0x1c00
>> +#define QM_AEQC_PF_DW0		0x1c20
>> +
>>  struct acc_vf_data {
>>  #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)
>>  	/* QM match information */
>> --
>> 2.24.0
> 
> .
>