From: Lei Rao <lei.rao@intel.com>
To: alex.williamson@redhat.com, kevin.tian@intel.com,
eddie.dong@intel.com, jason.zeng@intel.com, quintela@redhat.com,
dgilbert@redhat.com, yadong.li@intel.com, yi.l.liu@intel.com
Cc: qemu-devel@nongnu.org, Lei Rao <lei.rao@intel.com>
Subject: [RFC PATCH 06/13] vfio/migration: introduce VFIOMigrationOps layer in VFIO live migration framework
Date: Tue, 24 May 2022 14:18:41 +0800 [thread overview]
Message-ID: <20220524061848.1615706-7-lei.rao@intel.com> (raw)
In-Reply-To: <20220524061848.1615706-1-lei.rao@intel.com>
Add an abstraction layer, VFIOMigrationOps, to the VFIO live migration
framework. Also adapt the In-Band approach to this abstraction layer by defining
its own VFIOMigrationOps callbacks.
Signed-off-by: Lei Rao <lei.rao@intel.com>
Reviewed-by: Eddie Dong <eddie.dong@intel.com>
---
hw/vfio/migration.c | 203 +++++++++++++++++++++-------------
include/hw/vfio/vfio-common.h | 14 +++
2 files changed, 142 insertions(+), 75 deletions(-)
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 04360e1f17..4736af90e7 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -407,7 +407,11 @@ static void vfio_migration_cleanup_local(VFIODevice *vbasedev)
static void vfio_migration_cleanup(VFIODevice *vbasedev)
{
- vfio_migration_cleanup_local(vbasedev);
+ VFIOMigration *migration = vbasedev->migration;
+
+ if (migration->ops->cleanup) {
+ migration->ops->cleanup(vbasedev);
+ }
}
/* ---------------------------------------------------------------------- */
@@ -438,24 +442,29 @@ static int vfio_migration_save_setup_local(VFIODevice *vbasedev)
static int vfio_save_setup(QEMUFile *f, void *opaque)
{
VFIODevice *vbasedev = opaque;
+ VFIOMigration *migration = vbasedev->migration;
int ret;
trace_vfio_save_setup(vbasedev->name);
qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
- ret = vfio_migration_save_setup_local(vbasedev);
- if (ret) {
- error_report("%s: Failed to vfio lm save setup:%s",
- vbasedev->name, strerror(-ret));
- return ret;
+ if (migration->ops->save_setup) {
+ ret = migration->ops->save_setup(vbasedev);
+ if (ret) {
+ error_report("%s: Failed to vfio lm save setup:%s",
+ vbasedev->name, strerror(-ret));
+ return ret;
+ }
}
- ret = vfio_migration_set_state_local(vbasedev, VFIO_DEVICE_STATE_MASK,
- VFIO_DEVICE_STATE_V1_SAVING);
- if (ret) {
- error_report("%s: Failed to set state SAVING", vbasedev->name);
- return ret;
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev, VFIO_DEVICE_STATE_MASK,
+ VFIO_DEVICE_STATE_V1_SAVING);
+ if (ret) {
+ error_report("%s: Failed to set state SAVING", vbasedev->name);
+ return ret;
+ }
}
qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
@@ -486,9 +495,11 @@ static void vfio_save_pending(QEMUFile *f, void *opaque,
VFIOMigration *migration = vbasedev->migration;
int ret;
- ret = vfio_migration_update_pending_local(vbasedev);
- if (ret) {
- return;
+ if (migration->ops->update_pending) {
+ ret = migration->ops->update_pending(vbasedev);
+ if (ret) {
+ return;
+ }
}
*res_precopy_only += migration->pending_bytes;
@@ -507,9 +518,11 @@ static int vfio_save_iterate(QEMUFile *f, void *opaque)
qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
if (migration->pending_bytes == 0) {
- ret = vfio_migration_update_pending_local(vbasedev);
- if (ret) {
- return ret;
+ if (migration->ops->update_pending) {
+ ret = migration->ops->update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
}
if (migration->pending_bytes == 0) {
@@ -520,11 +533,13 @@ static int vfio_save_iterate(QEMUFile *f, void *opaque)
}
}
- ret = vfio_migration_save_buffer_local(f, vbasedev, &data_size);
- if (ret) {
- error_report("%s: vfio_miragion_save_buffer_local failed %s",
- vbasedev->name, strerror(errno));
- return ret;
+ if (migration->ops->save_buffer) {
+ ret = migration->ops->save_buffer(f, vbasedev, &data_size);
+ if (ret) {
+ error_report("%s: vfio_miragion_save_buffer_local failed %s",
+ vbasedev->name, strerror(errno));
+ return ret;
+ }
}
qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
@@ -551,35 +566,43 @@ static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
uint64_t data_size;
int ret;
- ret = vfio_migration_set_state_local(vbasedev,
- ~VFIO_DEVICE_STATE_V1_RUNNING,
- VFIO_DEVICE_STATE_V1_SAVING);
- if (ret) {
- error_report("%s: Failed to set state STOP and SAVING",
- vbasedev->name);
- return ret;
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev,
+ ~VFIO_DEVICE_STATE_V1_RUNNING,
+ VFIO_DEVICE_STATE_V1_SAVING);
+ if (ret) {
+ error_report("%s: Failed to set state STOP and SAVING",
+ vbasedev->name);
+ return ret;
+ }
}
- ret = vfio_migration_update_pending_local(vbasedev);
- if (ret) {
- return ret;
+ if (migration->ops->update_pending) {
+ ret = migration->ops->update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
}
while (migration->pending_bytes > 0) {
qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
- ret = vfio_migration_save_buffer_local(f, vbasedev, &data_size);
- if (ret < 0) {
- error_report("%s: Failed to save buffer", vbasedev->name);
- return ret;
+ if (migration->ops->save_buffer) {
+ ret = migration->ops->save_buffer(f, vbasedev, &data_size);
+ if (ret < 0) {
+ error_report("%s: Failed to save buffer", vbasedev->name);
+ return ret;
+ }
}
if (data_size == 0) {
break;
}
- ret = vfio_migration_update_pending_local(vbasedev);
- if (ret) {
- return ret;
+ if (migration->ops->update_pending) {
+ ret = migration->ops->update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
}
}
@@ -590,11 +613,13 @@ static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
return ret;
}
- ret = vfio_migration_set_state_local(vbasedev, ~VFIO_DEVICE_STATE_V1_SAVING,
- 0);
- if (ret) {
- error_report("%s: Failed to set state STOPPED", vbasedev->name);
- return ret;
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev, ~VFIO_DEVICE_STATE_V1_SAVING,
+ 0);
+ if (ret) {
+ error_report("%s: Failed to set state STOPPED", vbasedev->name);
+ return ret;
+ }
}
trace_vfio_save_complete_precopy(vbasedev->name);
@@ -634,20 +659,25 @@ static int vfio_migration_load_setup_local(VFIODevice *vbasedev)
static int vfio_load_setup(QEMUFile *f, void *opaque)
{
VFIODevice *vbasedev = opaque;
+ VFIOMigration *migration = vbasedev->migration;
int ret = 0;
- ret = vfio_migration_load_setup_local(vbasedev);
- if (ret < 0) {
- error_report("%s: Failed to migration load setup", vbasedev->name);
- return ret;
+ if (migration->ops->load_setup) {
+ ret = migration->ops->load_setup(vbasedev);
+ if (ret < 0) {
+ error_report("%s: Failed to migration load setup", vbasedev->name);
+ return ret;
+ }
}
- ret = vfio_migration_set_state_local(vbasedev, ~VFIO_DEVICE_STATE_MASK,
- VFIO_DEVICE_STATE_V1_RESUMING);
- if (ret) {
- error_report("%s: Failed to set state RESUMING", vbasedev->name);
- vfio_migration_cleanup(vbasedev);
- return ret;
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev, ~VFIO_DEVICE_STATE_MASK,
+ VFIO_DEVICE_STATE_V1_RESUMING);
+ if (ret) {
+ error_report("%s: Failed to set state RESUMING", vbasedev->name);
+ vfio_migration_cleanup(vbasedev);
+ return ret;
+ }
}
return ret;
}
@@ -692,11 +722,14 @@ static int vfio_load_state(QEMUFile *f, void *opaque, int version_id)
case VFIO_MIG_FLAG_DEV_DATA_STATE:
{
uint64_t data_size = qemu_get_be64(f);
+ VFIOMigration *migration = vbasedev->migration;
if (data_size) {
- ret = vfio_migration_load_buffer_local(f, vbasedev, data_size);
- if (ret < 0) {
- return ret;
+ if (migration->ops->load_buffer) {
+ ret = migration->ops->load_buffer(f, vbasedev, data_size);
+ if (ret < 0) {
+ return ret;
+ }
}
}
break;
@@ -736,7 +769,7 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state)
uint32_t value, mask;
int ret;
- if (vbasedev->migration->vm_running == running) {
+ if (migration->vm_running == running) {
return;
}
@@ -769,17 +802,19 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state)
}
}
- ret = vfio_migration_set_state_local(vbasedev, mask, value);
- if (ret) {
- /*
- * Migration should be aborted in this case, but vm_state_notify()
- * currently does not support reporting failures.
- */
- error_report("%s: Failed to set device state 0x%x", vbasedev->name,
- (migration->device_state & mask) | value);
- qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev, mask, value);
+ if (ret) {
+ /*
+ * Migration should be aborted in this case, but vm_state_notify()
+ * currently does not support reporting failures.
+ */
+ error_report("%s: Failed to set device state 0x%x", vbasedev->name,
+ (migration->device_state & mask) | value);
+ qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
+ }
}
- vbasedev->migration->vm_running = running;
+ migration->vm_running = running;
trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state),
(migration->device_state & mask) | value);
}
@@ -800,12 +835,14 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data)
case MIGRATION_STATUS_CANCELLED:
case MIGRATION_STATUS_FAILED:
bytes_transferred = 0;
- ret = vfio_migration_set_state_local(vbasedev,
- ~(VFIO_DEVICE_STATE_V1_SAVING |
- VFIO_DEVICE_STATE_V1_RESUMING),
- VFIO_DEVICE_STATE_V1_RUNNING);
- if (ret) {
- error_report("%s: Failed to set state RUNNING", vbasedev->name);
+ if (migration->ops->set_state) {
+ ret = migration->ops->set_state(vbasedev,
+ ~(VFIO_DEVICE_STATE_V1_SAVING |
+ VFIO_DEVICE_STATE_V1_RESUMING),
+ VFIO_DEVICE_STATE_V1_RUNNING);
+ if (ret) {
+ error_report("%s: Failed to set state RUNNING", vbasedev->name);
+ }
}
}
}
@@ -820,7 +857,11 @@ static void vfio_migration_exit_local(VFIODevice *vbasedev)
static void vfio_migration_exit(VFIODevice *vbasedev)
{
- vfio_migration_exit_local(vbasedev);
+ VFIOMigration *migration = vbasedev->migration;
+
+ if (migration->ops->exit) {
+ migration->ops->exit(vbasedev);
+ }
g_free(vbasedev->migration);
vbasedev->migration = NULL;
}
@@ -840,6 +881,17 @@ static int vfio_migration_check(VFIODevice *vbasedev)
return 0;
}
+static VFIOMigrationOps vfio_local_method = {
+ .save_setup = vfio_migration_save_setup_local,
+ .load_setup = vfio_migration_load_setup_local,
+ .update_pending = vfio_migration_update_pending_local,
+ .save_buffer = vfio_migration_save_buffer_local,
+ .load_buffer = vfio_migration_load_buffer_local,
+ .set_state = vfio_migration_set_state_local,
+ .cleanup = vfio_migration_cleanup_local,
+ .exit = vfio_migration_exit_local,
+};
+
static int vfio_migration_probe_local(VFIODevice *vbasedev)
{
int ret;
@@ -895,6 +947,7 @@ static int vfio_migration_probe_local(VFIODevice *vbasedev)
add_migration_state_change_notifier(&migration->migration_state);
trace_vfio_migration_probe_local(vbasedev->name, info->index);
+ migration->ops = &vfio_local_method;
g_free(info);
return 0;
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index e573f5a9f1..8ef85a871c 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -58,10 +58,13 @@ typedef struct VFIORegion {
uint8_t nr; /* cache the region number for debug */
} VFIORegion;
+typedef struct VFIOMigrationOps VFIOMigrationOps;
+
typedef struct VFIOMigration {
struct VFIODevice *vbasedev;
VMChangeStateEntry *vm_state;
VFIORegion region;
+ VFIOMigrationOps *ops;
uint32_t device_state;
int vm_running;
Notifier migration_state;
@@ -154,6 +157,17 @@ struct VFIODeviceOps {
int (*vfio_load_config)(VFIODevice *vdev, QEMUFile *f);
};
+typedef struct VFIOMigrationOps {
+ int (*save_setup)(VFIODevice *vbasedev);
+ int (*load_setup)(VFIODevice *vbasedev);
+ int (*update_pending)(VFIODevice *vbasedev);
+ int (*save_buffer)(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size);
+ int (*load_buffer)(QEMUFile *f, VFIODevice *vbasedev, uint64_t data_size);
+ int (*set_state)(VFIODevice *vbasedev, uint32_t mask, uint32_t value);
+ void (*cleanup)(VFIODevice *vbasedev);
+ void (*exit)(VFIODevice *vbasedev);
+} VFIOMigrationOps;
+
typedef struct VFIOGroup {
int fd;
int groupid;
--
2.32.0
next prev parent reply other threads:[~2022-05-24 6:34 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-05-24 6:18 [RFC PATCH 00/13] Add a plugin to support out-of-band live migration for VFIO pass-through device Lei Rao
2022-05-24 6:18 ` [RFC PATCH 01/13] vfio/migration: put together checks of migration initialization conditions Lei Rao
2022-05-24 6:18 ` [RFC PATCH 02/13] vfio/migration: move migration struct allocation out of vfio_migration_init Lei Rao
2022-05-24 6:18 ` [RFC PATCH 03/13] vfio/migration: move vfio_get_dev_region_info out of vfio_migration_probe Lei Rao
2022-05-24 6:18 ` [RFC PATCH 04/13] vfio/migration: Separated functions that relate to the In-Band approach Lei Rao
2022-05-24 6:18 ` [RFC PATCH 05/13] vfio/migration: rename " Lei Rao
2022-05-24 6:18 ` Lei Rao [this message]
2022-05-24 6:18 ` [RFC PATCH 07/13] vfio/migration: move the statistics of bytes_transferred to generic VFIO migration layer Lei Rao
2022-05-24 6:18 ` [RFC PATCH 08/13] vfio/migration: split migration handler registering from vfio_migration_init Lei Rao
2022-05-24 6:18 ` [RFC PATCH 09/13] vfio/migration: move the functions of In-Band approach to a new file Lei Rao
2022-05-24 6:18 ` [RFC PATCH 10/13] vfio/pci: introduce command-line parameters to specify migration method Lei Rao
2022-05-24 6:18 ` [RFC PATCH 11/13] vfio/migration: add a plugin layer to support out-of-band live migration Lei Rao
2022-05-24 6:18 ` [RFC PATCH 12/13] vfio/migration: add some trace-events for vfio migration plugin Lei Rao
2022-05-24 6:18 ` [RFC PATCH 13/13] vfio/migration: make the region and plugin member of struct VFIOMigration to be a union Lei Rao
2022-05-26 18:44 ` [RFC PATCH 00/13] Add a plugin to support out-of-band live migration for VFIO pass-through device Alex Williamson
2022-06-01 17:09 ` Dong, Eddie
2022-06-01 18:01 ` Alex Williamson
2022-06-02 4:24 ` Tian, Kevin
2022-06-14 21:10 ` Dong, Eddie
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220524061848.1615706-7-lei.rao@intel.com \
--to=lei.rao@intel.com \
--cc=alex.williamson@redhat.com \
--cc=dgilbert@redhat.com \
--cc=eddie.dong@intel.com \
--cc=jason.zeng@intel.com \
--cc=kevin.tian@intel.com \
--cc=qemu-devel@nongnu.org \
--cc=quintela@redhat.com \
--cc=yadong.li@intel.com \
--cc=yi.l.liu@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).