From: Yajun Wu <yajunw@nvidia.com>
To: <qemu-devel@nongnu.org>, <jasowang@redhat.com>, <mst@redhat.com>,
<yajunw@nvidia.com>
Cc: Avihai Horon <avihaih@nvidia.com>, Jiri Pirko <jiri@nvidia.com>
Subject: [RFC PATCH 4/5] virtio: Add VMState for early load
Date: Mon, 18 Sep 2023 12:49:31 +0800 [thread overview]
Message-ID: <20230918044932.1433744-5-yajunw@nvidia.com> (raw)
In-Reply-To: <20230918044932.1433744-1-yajunw@nvidia.com>
Define new virtio device vmstate for early save/load (happens in
LM setup stage). Same as original vmstate, except:
In LM setup phase of the destination VM, the guest memory has not
been synchronized yet. To address this, a flag has been added to
virtio_load function to skip the index check.
Signed-off-by: Yajun Wu <yajunw@nvidia.com>
Reviewed-by: Avihai Horon <avihaih@nvidia.com>
Reviewed-by: Jiri Pirko <jiri@nvidia.com>
---
hw/virtio/virtio.c | 152 +++++++++++++++++++++++--------------
include/hw/virtio/virtio.h | 10 ++-
2 files changed, 103 insertions(+), 59 deletions(-)
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 969c25f4cf..8c73c245dd 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -2832,7 +2832,17 @@ virtio_device_get(QEMUFile *f, void *opaque, size_t size,
VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
- return virtio_load(vdev, f, dc->vmsd->version_id);
+ return virtio_load(vdev, f, dc->vmsd->version_id, false);
+}
+
+/* A wrapper for use as a VMState .get function */
+static int virtio_early_device_get(QEMUFile *f, void *opaque, size_t size,
+ const VMStateField *field)
+{
+ VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
+ DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
+
+ return virtio_load(vdev, f, dc->vmsd->version_id, true);
}
const VMStateInfo virtio_vmstate_info = {
@@ -2841,6 +2851,12 @@ const VMStateInfo virtio_vmstate_info = {
.put = virtio_device_put,
};
+const VMStateInfo virtio_early_vmstate_info = {
+ .name = "virtio-early",
+ .get = virtio_early_device_get,
+ .put = virtio_device_put,
+};
+
static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
{
VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
@@ -2940,8 +2956,75 @@ size_t virtio_get_config_size(const VirtIOConfigSizeParams *params,
return config_size;
}
+static int virtio_load_check_index(VirtIODevice *vdev, int num)
+{
+ int i;
+
+ RCU_READ_LOCK_GUARD();
+
+ for (i = 0; i < num; i++) {
+ if (vdev->vq[i].vring.desc) {
+ uint16_t nheads;
+
+ /*
+ * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
+ * only the region cache needs to be set up. Legacy devices need
+ * to calculate used and avail ring addresses based on the desc
+ * address.
+ */
+ if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+ virtio_init_region_cache(vdev, i);
+ } else {
+ virtio_queue_update_rings(vdev, i);
+ }
+
+ if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
+ vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
+ vdev->vq[i].shadow_avail_wrap_counter =
+ vdev->vq[i].last_avail_wrap_counter;
+ continue;
+ }
+
+ nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
+ /* Check it isn't doing strange things with descriptor numbers. */
+ if (nheads > vdev->vq[i].vring.num) {
+ virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x "
+ "inconsistent with Host index 0x%x: delta 0x%x",
+ i, vdev->vq[i].vring.num,
+ vring_avail_idx(&vdev->vq[i]),
+ vdev->vq[i].last_avail_idx, nheads);
+ vdev->vq[i].used_idx = 0;
+ vdev->vq[i].shadow_avail_idx = 0;
+ vdev->vq[i].inuse = 0;
+ continue;
+ }
+ vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
+ vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
+
+ /*
+ * Some devices migrate VirtQueueElements that have been popped
+ * from the avail ring but not yet returned to the used ring.
+ * Since max ring size < UINT16_MAX it's safe to use modulo
+ * UINT16_MAX + 1 subtraction.
+ */
+ vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
+ vdev->vq[i].used_idx);
+ if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
+ error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
+ "used_idx 0x%x",
+ i, vdev->vq[i].vring.num,
+ vdev->vq[i].last_avail_idx,
+ vdev->vq[i].used_idx);
+ return -1;
+ }
+ }
+ }
+
+ return 0;
+}
+
int coroutine_mixed_fn
-virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
+virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id, bool early)
{
int i, ret;
int32_t config_len;
@@ -3078,62 +3161,15 @@ virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
vdev->start_on_kick = true;
}
- RCU_READ_LOCK_GUARD();
- for (i = 0; i < num; i++) {
- if (vdev->vq[i].vring.desc) {
- uint16_t nheads;
-
- /*
- * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
- * only the region cache needs to be set up. Legacy devices need
- * to calculate used and avail ring addresses based on the desc
- * address.
- */
- if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
- virtio_init_region_cache(vdev, i);
- } else {
- virtio_queue_update_rings(vdev, i);
- }
-
- if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
- vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
- vdev->vq[i].shadow_avail_wrap_counter =
- vdev->vq[i].last_avail_wrap_counter;
- continue;
- }
-
- nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
- /* Check it isn't doing strange things with descriptor numbers. */
- if (nheads > vdev->vq[i].vring.num) {
- virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x "
- "inconsistent with Host index 0x%x: delta 0x%x",
- i, vdev->vq[i].vring.num,
- vring_avail_idx(&vdev->vq[i]),
- vdev->vq[i].last_avail_idx, nheads);
- vdev->vq[i].used_idx = 0;
- vdev->vq[i].shadow_avail_idx = 0;
- vdev->vq[i].inuse = 0;
- continue;
- }
- vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
- vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
-
- /*
- * Some devices migrate VirtQueueElements that have been popped
- * from the avail ring but not yet returned to the used ring.
- * Since max ring size < UINT16_MAX it's safe to use modulo
- * UINT16_MAX + 1 subtraction.
- */
- vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
- vdev->vq[i].used_idx);
- if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
- error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
- "used_idx 0x%x",
- i, vdev->vq[i].vring.num,
- vdev->vq[i].last_avail_idx,
- vdev->vq[i].used_idx);
- return -1;
- }
+ /*
+ * Early setup happens in LM setup stage when the guest memory hasn't
+ * synced to target VM yet. So skip all guest memory access and index check
+ * in early load.
+ */
+ if (!early) {
+ ret = virtio_load_check_index(vdev, num);
+ if (ret) {
+ return ret;
}
}
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index c8f72850bc..c9e6faf72c 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -280,6 +280,7 @@ void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
int virtio_save(VirtIODevice *vdev, QEMUFile *f);
extern const VMStateInfo virtio_vmstate_info;
+extern const VMStateInfo virtio_early_vmstate_info;
#define VMSTATE_VIRTIO_DEVICE \
{ \
@@ -288,7 +289,14 @@ extern const VMStateInfo virtio_vmstate_info;
.flags = VMS_SINGLE, \
}
-int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id);
+#define VMSTATE_EARLY_VIRTIO_DEVICE \
+ { \
+ .name = "virtio-early", \
+ .info = &virtio_early_vmstate_info,\
+ .flags = VMS_SINGLE, \
+ }
+
+int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id, bool early);
/**
* virtio_notify_config() - signal a change to device config
--
2.27.0
next prev parent reply other threads:[~2023-09-18 4:51 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-09-18 4:49 [RFC PATCH 0/5] virtio-net: Introduce LM early load Yajun Wu
2023-09-18 4:49 ` [RFC PATCH 1/5] vhost-user: Add presetup protocol feature and op Yajun Wu
2023-09-18 4:49 ` [RFC PATCH 2/5] vhost: Add support for presetup Yajun Wu
2023-12-22 18:46 ` Eugenio Perez Martin
2023-09-18 4:49 ` [RFC PATCH 3/5] vhost-net: " Yajun Wu
2023-09-18 4:49 ` Yajun Wu [this message]
2023-12-22 17:36 ` [RFC PATCH 4/5] virtio: Add VMState for early load Eugenio Perez Martin
2023-09-18 4:49 ` [RFC PATCH 5/5] virtio-net: Introduce LM " Yajun Wu
2023-12-22 18:58 ` Eugenio Perez Martin
2023-10-17 7:32 ` [RFC PATCH 0/5] " Yajun Wu
2023-10-17 16:47 ` Eugenio Perez Martin
2023-10-18 6:40 ` Yajun Wu
2023-10-19 15:00 ` Eugenio Perez Martin
2023-12-22 19:07 ` Eugenio Perez Martin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230918044932.1433744-5-yajunw@nvidia.com \
--to=yajunw@nvidia.com \
--cc=avihaih@nvidia.com \
--cc=jasowang@redhat.com \
--cc=jiri@nvidia.com \
--cc=mst@redhat.com \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).