* [Qemu-devel] [PATCH 1/6] vhost: allow backends to filter memory sections
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 2/6] vhost-user: allow slave to send fds via slave channel Tiwei Bie
` (4 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
This patch introduces a vhost op for vhost backends to allow
them to filter the memory sections that they can handle.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
hw/virtio/vhost-user.c | 11 +++++++++++
hw/virtio/vhost.c | 9 +++++++--
include/hw/virtio/vhost-backend.h | 4 ++++
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index ca554d4ff1..da0756effe 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -1620,6 +1620,16 @@ vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
return 0;
}
+static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
+ MemoryRegionSection *section)
+{
+ bool result;
+
+ result = memory_region_get_fd(section->mr) >= 0;
+
+ return result;
+}
+
const VhostOps user_ops = {
.backend_type = VHOST_BACKEND_TYPE_USER,
.vhost_backend_init = vhost_user_init,
@@ -1650,4 +1660,5 @@ const VhostOps user_ops = {
.vhost_set_config = vhost_user_set_config,
.vhost_crypto_create_session = vhost_user_crypto_create_session,
.vhost_crypto_close_session = vhost_user_crypto_close_session,
+ .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
};
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index b08290036d..624ade9682 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -386,7 +386,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,
return r;
}
-static bool vhost_section(MemoryRegionSection *section)
+static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
{
bool result;
bool log_dirty = memory_region_get_dirty_log_mask(section->mr) &
@@ -399,6 +399,11 @@ static bool vhost_section(MemoryRegionSection *section)
*/
result &= !log_dirty;
+ if (result && dev->vhost_ops->vhost_backend_mem_section_filter) {
+ result &=
+ dev->vhost_ops->vhost_backend_mem_section_filter(dev, section);
+ }
+
trace_vhost_section(section->mr->name, result);
return result;
}
@@ -632,7 +637,7 @@ static void vhost_region_addnop(MemoryListener *listener,
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
memory_listener);
- if (!vhost_section(section)) {
+ if (!vhost_section(dev, section)) {
return;
}
vhost_region_add_section(dev, section);
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index 5dac61f9ea..81283ec50f 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -101,6 +101,9 @@ typedef int (*vhost_crypto_create_session_op)(struct vhost_dev *dev,
typedef int (*vhost_crypto_close_session_op)(struct vhost_dev *dev,
uint64_t session_id);
+typedef bool (*vhost_backend_mem_section_filter_op)(struct vhost_dev *dev,
+ MemoryRegionSection *section);
+
typedef struct VhostOps {
VhostBackendType backend_type;
vhost_backend_init vhost_backend_init;
@@ -138,6 +141,7 @@ typedef struct VhostOps {
vhost_set_config_op vhost_set_config;
vhost_crypto_create_session_op vhost_crypto_create_session;
vhost_crypto_close_session_op vhost_crypto_close_session;
+ vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
} VhostOps;
extern const VhostOps user_ops;
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* [Qemu-devel] [PATCH 2/6] vhost-user: allow slave to send fds via slave channel
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 1/6] vhost: allow backends to filter memory sections Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 3/6] vhost-user: introduce shared vhost-user state Tiwei Bie
` (3 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
Introduce VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD protocol
feature to allow slave to send at most 8 descriptors
in each message to master via ancillary data using the
slave channel.
Suggested-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
docs/interop/vhost-user.txt | 5 +++++
hw/virtio/vhost-user.c | 27 +++++++++++++++++----------
2 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt
index 534caab18a..682a683eb4 100644
--- a/docs/interop/vhost-user.txt
+++ b/docs/interop/vhost-user.txt
@@ -367,6 +367,10 @@ The fd is provided via VHOST_USER_SET_SLAVE_REQ_FD ancillary data.
A slave may then send VHOST_USER_SLAVE_* messages to the master
using this fd communication channel.
+If VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD protocol feature is negotiated,
+slave can send file descriptors (at most 8 descriptors in each message)
+to master via ancillary data using this fd communication channel.
+
Protocol features
-----------------
@@ -380,6 +384,7 @@ Protocol features
#define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7
#define VHOST_USER_PROTOCOL_F_PAGEFAULT 8
#define VHOST_USER_PROTOCOL_F_CONFIG 9
+#define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10
Master message types
--------------------
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index da0756effe..75a3faef2a 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -30,6 +30,7 @@
#define VHOST_MEMORY_MAX_NREGIONS 8
#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#define VHOST_USER_SLAVE_MAX_FDS 8
/*
* Maximum size of virtio device config space
@@ -47,6 +48,7 @@ enum VhostUserProtocolFeature {
VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
VHOST_USER_PROTOCOL_F_CONFIG = 9,
+ VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
VHOST_USER_PROTOCOL_F_MAX
};
@@ -854,10 +856,10 @@ static void slave_read(void *opaque)
int size, ret = 0;
struct iovec iov;
struct msghdr msgh;
- int fd = -1;
+ int fd[VHOST_USER_SLAVE_MAX_FDS];
char control[CMSG_SPACE(sizeof(fd))];
struct cmsghdr *cmsg;
- size_t fdsize;
+ int i, fdsize = 0;
memset(&msgh, 0, sizeof(msgh));
msgh.msg_iov = &iov;
@@ -865,6 +867,8 @@ static void slave_read(void *opaque)
msgh.msg_control = control;
msgh.msg_controllen = sizeof(control);
+ memset(fd, -1, sizeof(fd));
+
/* Read header */
iov.iov_base = &hdr;
iov.iov_len = VHOST_USER_HDR_SIZE;
@@ -885,7 +889,7 @@ static void slave_read(void *opaque)
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SCM_RIGHTS) {
fdsize = cmsg->cmsg_len - CMSG_LEN(0);
- memcpy(&fd, CMSG_DATA(cmsg), fdsize);
+ memcpy(fd, CMSG_DATA(cmsg), fdsize);
break;
}
}
@@ -913,14 +917,15 @@ static void slave_read(void *opaque)
break;
default:
error_report("Received unexpected msg type.");
- if (fd != -1) {
- close(fd);
- }
ret = -EINVAL;
}
- /* Message handlers need to make sure that fd will be consumed. */
- fd = -1;
+ /* Close the remaining file descriptors. */
+ for (i = 0; i < fdsize; i++) {
+ if (fd[i] != -1) {
+ close(fd[i]);
+ }
+ }
/*
* REPLY_ACK feature handling. Other reply types has to be managed
@@ -954,8 +959,10 @@ err:
qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
close(u->slave_fd);
u->slave_fd = -1;
- if (fd != -1) {
- close(fd);
+ for (i = 0; i < fdsize; i++) {
+ if (fd[i] != -1) {
+ close(fd[i]);
+ }
}
return;
}
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* [Qemu-devel] [PATCH 3/6] vhost-user: introduce shared vhost-user state
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 1/6] vhost: allow backends to filter memory sections Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 2/6] vhost-user: allow slave to send fds via slave channel Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers Tiwei Bie
` (2 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
When multi queue is enabled e.g. for a virtio-net device,
each queue pair will have a vhost_dev, and the only thing
shared between vhost devs currently is the chardev. This
patch introduces a vhost-user state structure which will
be shared by all vhost devs of the same virtio device.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
backends/cryptodev-vhost-user.c | 20 ++++++++++++-
hw/block/vhost-user-blk.c | 22 ++++++++++++++-
hw/scsi/vhost-user-scsi.c | 20 ++++++++++++-
hw/virtio/Makefile.objs | 2 +-
hw/virtio/vhost-stub.c | 10 +++++++
hw/virtio/vhost-user.c | 31 ++++++++++++++------
include/hw/virtio/vhost-user-blk.h | 2 ++
include/hw/virtio/vhost-user-scsi.h | 2 ++
include/hw/virtio/vhost-user.h | 20 +++++++++++++
net/vhost-user.c | 44 +++++++++++++++++++++++------
10 files changed, 152 insertions(+), 21 deletions(-)
create mode 100644 include/hw/virtio/vhost-user.h
diff --git a/backends/cryptodev-vhost-user.c b/backends/cryptodev-vhost-user.c
index 862d4f2580..d52daccfcd 100644
--- a/backends/cryptodev-vhost-user.c
+++ b/backends/cryptodev-vhost-user.c
@@ -26,6 +26,7 @@
#include "qapi/error.h"
#include "qapi/qmp/qerror.h"
#include "qemu/error-report.h"
+#include "hw/virtio/vhost-user.h"
#include "standard-headers/linux/virtio_crypto.h"
#include "sysemu/cryptodev-vhost.h"
#include "chardev/char-fe.h"
@@ -46,6 +47,7 @@
typedef struct CryptoDevBackendVhostUser {
CryptoDevBackend parent_obj;
+ VhostUserState *vhost_user;
CharBackend chr;
char *chr_name;
bool opened;
@@ -102,7 +104,7 @@ cryptodev_vhost_user_start(int queues,
continue;
}
- options.opaque = &s->chr;
+ options.opaque = s->vhost_user;
options.backend_type = VHOST_BACKEND_TYPE_USER;
options.cc = b->conf.peers.ccs[i];
s->vhost_crypto[i] = cryptodev_vhost_init(&options);
@@ -185,6 +187,7 @@ static void cryptodev_vhost_user_init(
size_t i;
Error *local_err = NULL;
Chardev *chr;
+ VhostUserState *user;
CryptoDevBackendClient *cc;
CryptoDevBackendVhostUser *s =
CRYPTODEV_BACKEND_VHOST_USER(backend);
@@ -215,6 +218,15 @@ static void cryptodev_vhost_user_init(
}
}
+ user = vhost_user_init();
+ if (!user) {
+ error_setg(errp, "Failed to init vhost_user");
+ return;
+ }
+
+ user->chr = &s->chr;
+ s->vhost_user = user;
+
qemu_chr_fe_set_handlers(&s->chr, NULL, NULL,
cryptodev_vhost_user_event, NULL, s, NULL, true);
@@ -299,6 +311,12 @@ static void cryptodev_vhost_user_cleanup(
backend->conf.peers.ccs[i] = NULL;
}
}
+
+ if (s->vhost_user) {
+ vhost_user_cleanup(s->vhost_user);
+ g_free(s->vhost_user);
+ s->vhost_user = NULL;
+ }
}
static void cryptodev_vhost_user_set_chardev(Object *obj,
diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 975eae6211..7c3fa8bb1c 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -226,6 +226,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VHostUserBlk *s = VHOST_USER_BLK(vdev);
+ VhostUserState *user;
int i, ret;
if (!s->chardev.chr) {
@@ -243,6 +244,15 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
return;
}
+ user = vhost_user_init();
+ if (!user) {
+ error_setg(errp, "vhost-user-blk: failed to init vhost_user");
+ return;
+ }
+
+ user->chr = &s->chardev;
+ s->vhost_user = user;
+
virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
sizeof(struct virtio_blk_config));
@@ -258,7 +268,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
vhost_dev_set_config_notifier(&s->dev, &blk_ops);
- ret = vhost_dev_init(&s->dev, &s->chardev, VHOST_BACKEND_TYPE_USER, 0);
+ ret = vhost_dev_init(&s->dev, s->vhost_user, VHOST_BACKEND_TYPE_USER, 0);
if (ret < 0) {
error_setg(errp, "vhost-user-blk: vhost initialization failed: %s",
strerror(-ret));
@@ -283,6 +293,10 @@ vhost_err:
virtio_err:
g_free(s->dev.vqs);
virtio_cleanup(vdev);
+
+ vhost_user_cleanup(user);
+ g_free(user);
+ s->vhost_user = NULL;
}
static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp)
@@ -294,6 +308,12 @@ static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp)
vhost_dev_cleanup(&s->dev);
g_free(s->dev.vqs);
virtio_cleanup(vdev);
+
+ if (s->vhost_user) {
+ vhost_user_cleanup(s->vhost_user);
+ g_free(s->vhost_user);
+ s->vhost_user = NULL;
+ }
}
static void vhost_user_blk_instance_init(Object *obj)
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index 9389ed48e0..9355cfdf07 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -69,6 +69,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(dev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+ VhostUserState *user;
Error *err = NULL;
int ret;
@@ -85,19 +86,30 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
return;
}
+ user = vhost_user_init();
+ if (!user) {
+ error_setg(errp, "vhost-user-scsi: failed to init vhost_user");
+ return;
+ }
+ user->chr = &vs->conf.chardev;
+
vsc->dev.nvqs = 2 + vs->conf.num_queues;
vsc->dev.vqs = g_new(struct vhost_virtqueue, vsc->dev.nvqs);
vsc->dev.vq_index = 0;
vsc->dev.backend_features = 0;
- ret = vhost_dev_init(&vsc->dev, (void *)&vs->conf.chardev,
+ ret = vhost_dev_init(&vsc->dev, user,
VHOST_BACKEND_TYPE_USER, 0);
if (ret < 0) {
error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s",
strerror(-ret));
+ vhost_user_cleanup(user);
+ g_free(user);
return;
}
+ s->vhost_user = user;
+
/* Channel and lun both are 0 for bootable vhost-user-scsi disk */
vsc->channel = 0;
vsc->lun = 0;
@@ -117,6 +129,12 @@ static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp)
g_free(vsc->dev.vqs);
virtio_scsi_common_unrealize(dev, errp);
+
+ if (s->vhost_user) {
+ vhost_user_cleanup(s->vhost_user);
+ g_free(s->vhost_user);
+ s->vhost_user = NULL;
+ }
}
static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev,
diff --git a/hw/virtio/Makefile.objs b/hw/virtio/Makefile.objs
index 765d363c1f..030969e28c 100644
--- a/hw/virtio/Makefile.objs
+++ b/hw/virtio/Makefile.objs
@@ -11,5 +11,5 @@ obj-y += virtio-crypto.o
obj-$(CONFIG_VIRTIO_PCI) += virtio-crypto-pci.o
endif
-common-obj-$(call lnot,$(CONFIG_LINUX)) += vhost-stub.o
+common-obj-$(call lnot,$(call land,$(CONFIG_VIRTIO),$(CONFIG_LINUX))) += vhost-stub.o
common-obj-$(CONFIG_ALL) += vhost-stub.o
diff --git a/hw/virtio/vhost-stub.c b/hw/virtio/vhost-stub.c
index 2d76cdebdc..049089b5e2 100644
--- a/hw/virtio/vhost-stub.c
+++ b/hw/virtio/vhost-stub.c
@@ -1,7 +1,17 @@
#include "qemu/osdep.h"
#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
bool vhost_has_free_slot(void)
{
return true;
}
+
+VhostUserState *vhost_user_init(void)
+{
+ return NULL;
+}
+
+void vhost_user_cleanup(VhostUserState *user)
+{
+}
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 75a3faef2a..70b9610c87 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -11,6 +11,7 @@
#include "qemu/osdep.h"
#include "qapi/error.h"
#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
#include "hw/virtio/vhost-backend.h"
#include "hw/virtio/virtio-net.h"
#include "chardev/char-fe.h"
@@ -175,7 +176,8 @@ static VhostUserMsg m __attribute__ ((unused));
struct vhost_user {
struct vhost_dev *dev;
- CharBackend *chr;
+ /* Shared between vhost devs of the same virtio device */
+ VhostUserState *user;
int slave_fd;
NotifierWithReturn postcopy_notifier;
struct PostCopyFD postcopy_fd;
@@ -201,7 +203,7 @@ static bool ioeventfd_enabled(void)
static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
{
struct vhost_user *u = dev->opaque;
- CharBackend *chr = u->chr;
+ CharBackend *chr = u->user->chr;
uint8_t *p = (uint8_t *) msg;
int r, size = VHOST_USER_HDR_SIZE;
@@ -287,7 +289,7 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
int *fds, int fd_num)
{
struct vhost_user *u = dev->opaque;
- CharBackend *chr = u->chr;
+ CharBackend *chr = u->user->chr;
int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
/*
@@ -1090,7 +1092,7 @@ static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
{
struct vhost_user *u = dev->opaque;
- CharBackend *chr = u->chr;
+ CharBackend *chr = u->user->chr;
int ufd;
VhostUserMsg msg = {
.hdr.request = VHOST_USER_POSTCOPY_ADVISE,
@@ -1228,7 +1230,7 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
return 0;
}
-static int vhost_user_init(struct vhost_dev *dev, void *opaque)
+static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
{
uint64_t features, protocol_features;
struct vhost_user *u;
@@ -1237,7 +1239,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
u = g_new0(struct vhost_user, 1);
- u->chr = opaque;
+ u->user = opaque;
u->slave_fd = -1;
u->dev = dev;
dev->opaque = u;
@@ -1313,7 +1315,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
return 0;
}
-static int vhost_user_cleanup(struct vhost_dev *dev)
+static int vhost_user_backend_cleanup(struct vhost_dev *dev)
{
struct vhost_user *u;
@@ -1637,10 +1639,21 @@ static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
return result;
}
+VhostUserState *vhost_user_init(void)
+{
+ VhostUserState *user = g_new0(struct VhostUserState, 1);
+
+ return user;
+}
+
+void vhost_user_cleanup(VhostUserState *user)
+{
+}
+
const VhostOps user_ops = {
.backend_type = VHOST_BACKEND_TYPE_USER,
- .vhost_backend_init = vhost_user_init,
- .vhost_backend_cleanup = vhost_user_cleanup,
+ .vhost_backend_init = vhost_user_backend_init,
+ .vhost_backend_cleanup = vhost_user_backend_cleanup,
.vhost_backend_memslots_limit = vhost_user_memslots_limit,
.vhost_set_log_base = vhost_user_set_log_base,
.vhost_set_mem_table = vhost_user_set_mem_table,
diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h
index 5804cc904a..f1258ae545 100644
--- a/include/hw/virtio/vhost-user-blk.h
+++ b/include/hw/virtio/vhost-user-blk.h
@@ -21,6 +21,7 @@
#include "hw/block/block.h"
#include "chardev/char-fe.h"
#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
#define TYPE_VHOST_USER_BLK "vhost-user-blk"
#define VHOST_USER_BLK(obj) \
@@ -36,6 +37,7 @@ typedef struct VHostUserBlk {
uint32_t config_wce;
uint32_t config_ro;
struct vhost_dev dev;
+ VhostUserState *vhost_user;
} VHostUserBlk;
#endif
diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h
index 01861f78d0..3ec34ae867 100644
--- a/include/hw/virtio/vhost-user-scsi.h
+++ b/include/hw/virtio/vhost-user-scsi.h
@@ -21,6 +21,7 @@
#include "hw/qdev.h"
#include "hw/virtio/virtio-scsi.h"
#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
#include "hw/virtio/vhost-scsi-common.h"
#define TYPE_VHOST_USER_SCSI "vhost-user-scsi"
@@ -30,6 +31,7 @@
typedef struct VHostUserSCSI {
VHostSCSICommon parent_obj;
uint64_t host_features;
+ VhostUserState *vhost_user;
} VHostUserSCSI;
#endif /* VHOST_USER_SCSI_H */
diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
new file mode 100644
index 0000000000..eb8bc0d90d
--- /dev/null
+++ b/include/hw/virtio/vhost-user.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2017-2018 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef HW_VIRTIO_VHOST_USER_H
+#define HW_VIRTIO_VHOST_USER_H
+
+#include "chardev/char-fe.h"
+
+typedef struct VhostUserState {
+ CharBackend *chr;
+} VhostUserState;
+
+VhostUserState *vhost_user_init(void);
+void vhost_user_cleanup(VhostUserState *user);
+
+#endif
diff --git a/net/vhost-user.c b/net/vhost-user.c
index fa28aad12d..608b837175 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -12,6 +12,7 @@
#include "clients.h"
#include "net/vhost_net.h"
#include "net/vhost-user.h"
+#include "hw/virtio/vhost-user.h"
#include "chardev/char-fe.h"
#include "qapi/error.h"
#include "qapi/qapi-commands-net.h"
@@ -23,6 +24,7 @@
typedef struct NetVhostUserState {
NetClientState nc;
CharBackend chr; /* only queue index 0 */
+ VhostUserState *vhost_user;
VHostNetState *vhost_net;
guint watch;
uint64_t acked_features;
@@ -64,7 +66,8 @@ static void vhost_user_stop(int queues, NetClientState *ncs[])
}
}
-static int vhost_user_start(int queues, NetClientState *ncs[], CharBackend *be)
+static int vhost_user_start(int queues, NetClientState *ncs[],
+ VhostUserState *be)
{
VhostNetOptions options;
struct vhost_net *net = NULL;
@@ -144,7 +147,7 @@ static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
return size;
}
-static void vhost_user_cleanup(NetClientState *nc)
+static void net_vhost_user_cleanup(NetClientState *nc)
{
NetVhostUserState *s = DO_UPCAST(NetVhostUserState, nc, nc);
@@ -159,6 +162,11 @@ static void vhost_user_cleanup(NetClientState *nc)
s->watch = 0;
}
qemu_chr_fe_deinit(&s->chr, true);
+ if (s->vhost_user) {
+ vhost_user_cleanup(s->vhost_user);
+ g_free(s->vhost_user);
+ s->vhost_user = NULL;
+ }
}
qemu_purge_queued_packets(nc);
@@ -182,7 +190,7 @@ static NetClientInfo net_vhost_user_info = {
.type = NET_CLIENT_DRIVER_VHOST_USER,
.size = sizeof(NetVhostUserState),
.receive = vhost_user_receive,
- .cleanup = vhost_user_cleanup,
+ .cleanup = net_vhost_user_cleanup,
.has_vnet_hdr = vhost_user_has_vnet_hdr,
.has_ufo = vhost_user_has_ufo,
};
@@ -244,7 +252,7 @@ static void net_vhost_user_event(void *opaque, int event)
trace_vhost_user_event(chr->label, event);
switch (event) {
case CHR_EVENT_OPENED:
- if (vhost_user_start(queues, ncs, &s->chr) < 0) {
+ if (vhost_user_start(queues, ncs, s->vhost_user) < 0) {
qemu_chr_fe_disconnect(&s->chr);
return;
}
@@ -283,12 +291,19 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
{
Error *err = NULL;
NetClientState *nc, *nc0 = NULL;
- NetVhostUserState *s;
+ VhostUserState *user = NULL;
+ NetVhostUserState *s = NULL;
int i;
assert(name);
assert(queues > 0);
+ user = vhost_user_init();
+ if (!user) {
+ error_report("failed to init vhost_user");
+ goto err;
+ }
+
for (i = 0; i < queues; i++) {
nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user%d to %s",
@@ -299,17 +314,19 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
s = DO_UPCAST(NetVhostUserState, nc, nc);
if (!qemu_chr_fe_init(&s->chr, chr, &err)) {
error_report_err(err);
- return -1;
+ goto err;
}
+ user->chr = &s->chr;
}
-
+ s = DO_UPCAST(NetVhostUserState, nc, nc);
+ s->vhost_user = user;
}
s = DO_UPCAST(NetVhostUserState, nc, nc0);
do {
if (qemu_chr_fe_wait_connected(&s->chr, &err) < 0) {
error_report_err(err);
- return -1;
+ goto err;
}
qemu_chr_fe_set_handlers(&s->chr, NULL, NULL,
net_vhost_user_event, NULL, nc0->name, NULL,
@@ -319,6 +336,17 @@ static int net_vhost_user_init(NetClientState *peer, const char *device,
assert(s->vhost_net);
return 0;
+
+err:
+ if (user) {
+ vhost_user_cleanup(user);
+ g_free(user);
+ if (s) {
+ s->vhost_user = NULL;
+ }
+ }
+
+ return -1;
}
static Chardev *net_vhost_claim_chardev(
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
` (2 preceding siblings ...)
2018-05-24 10:33 ` [Qemu-devel] [PATCH 3/6] vhost-user: introduce shared vhost-user state Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
2018-05-24 15:49 ` Michael S. Tsirkin
2018-05-24 10:33 ` [Qemu-devel] [PATCH 5/6] libvhost-user: support host notifier Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 6/6] vhost-user-bridge: " Tiwei Bie
5 siblings, 1 reply; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER.
With this feature negotiated, vhost-user backend can register
memory region based host notifiers. And it will allow the guest
driver in the VM to notify the hardware accelerator at the
vhost-user backend directly.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
docs/interop/vhost-user.txt | 33 ++++++++++
hw/virtio/vhost-user.c | 113 +++++++++++++++++++++++++++++++++
include/hw/virtio/vhost-user.h | 8 +++
3 files changed, 154 insertions(+)
diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt
index 682a683eb4..d51fd58242 100644
--- a/docs/interop/vhost-user.txt
+++ b/docs/interop/vhost-user.txt
@@ -132,6 +132,16 @@ Depending on the request type, payload can be:
Payload: Size bytes array holding the contents of the virtio
device's configuration space
+ * Vring area description
+ -----------------------
+ | u64 | size | offset |
+ -----------------------
+
+ u64: a 64-bit integer contains vring index and flags
+ Size: a 64-bit size of this area
+ Offset: a 64-bit offset of this area from the start of the
+ supplied file descriptor
+
In QEMU the vhost-user message is implemented with the following struct:
typedef struct VhostUserMsg {
@@ -146,6 +156,7 @@ typedef struct VhostUserMsg {
VhostUserLog log;
struct vhost_iotlb_msg iotlb;
VhostUserConfig config;
+ VhostUserVringArea area;
};
} QEMU_PACKED VhostUserMsg;
@@ -385,6 +396,7 @@ Protocol features
#define VHOST_USER_PROTOCOL_F_PAGEFAULT 8
#define VHOST_USER_PROTOCOL_F_CONFIG 9
#define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10
+#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11
Master message types
--------------------
@@ -782,6 +794,27 @@ Slave message types
the VHOST_USER_NEED_REPLY flag, master must respond with zero when
operation is successfully completed, or non-zero otherwise.
+ * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
+
+ Id: 3
+ Equivalent ioctl: N/A
+ Slave payload: vring area description
+ Master payload: N/A
+
+ Sets host notifier for a specified queue. The queue index is contained
+ in the u64 field of the vring area description. The host notifier is
+ described by the file descriptor (typically it's a VFIO device fd) which
+ is passed as ancillary data and the size (which is mmap size and should
+ be the same as host page size) and offset (which is mmap offset) carried
+ in the vring area description. QEMU can mmap the file descriptor based
+ on the size and offset to get a memory range. Registering a host notifier
+ means mapping this memory range to the VM as the specified queue's notify
+ MMIO region. Slave sends this request to tell QEMU to de-register the
+ existing notifier if any and register the new notifier if the request is
+ sent with a file descriptor.
+ This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
+ protocol feature has been successfully negotiated.
+
VHOST_USER_PROTOCOL_F_REPLY_ACK:
-------------------------------
The original vhost-user specification only demands replies for certain
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 70b9610c87..b041343632 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -13,6 +13,7 @@
#include "hw/virtio/vhost.h"
#include "hw/virtio/vhost-user.h"
#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio.h"
#include "hw/virtio/virtio-net.h"
#include "chardev/char-fe.h"
#include "sysemu/kvm.h"
@@ -50,6 +51,7 @@ enum VhostUserProtocolFeature {
VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
VHOST_USER_PROTOCOL_F_CONFIG = 9,
VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
+ VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
VHOST_USER_PROTOCOL_F_MAX
};
@@ -94,6 +96,7 @@ typedef enum VhostUserSlaveRequest {
VHOST_USER_SLAVE_NONE = 0,
VHOST_USER_SLAVE_IOTLB_MSG = 1,
VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
+ VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
VHOST_USER_SLAVE_MAX
} VhostUserSlaveRequest;
@@ -138,6 +141,12 @@ static VhostUserConfig c __attribute__ ((unused));
+ sizeof(c.size) \
+ sizeof(c.flags))
+typedef struct VhostUserVringArea {
+ uint64_t u64;
+ uint64_t size;
+ uint64_t offset;
+} VhostUserVringArea;
+
typedef struct {
VhostUserRequest request;
@@ -159,6 +168,7 @@ typedef union {
struct vhost_iotlb_msg iotlb;
VhostUserConfig config;
VhostUserCryptoSession session;
+ VhostUserVringArea area;
} VhostUserPayload;
typedef struct VhostUserMsg {
@@ -640,9 +650,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,
return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
}
+static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
+ int queue_idx)
+{
+ struct vhost_user *u = dev->opaque;
+ VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
+ VirtIODevice *vdev = dev->vdev;
+
+ if (n->addr && !n->set) {
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
+ n->set = true;
+ }
+}
+
+static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
+ int queue_idx)
+{
+ struct vhost_user *u = dev->opaque;
+ VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
+ VirtIODevice *vdev = dev->vdev;
+
+ if (n->addr && n->set) {
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
+ n->set = false;
+ }
+}
+
static int vhost_user_set_vring_base(struct vhost_dev *dev,
struct vhost_vring_state *ring)
{
+ vhost_user_host_notifier_restore(dev, ring->index);
+
return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
}
@@ -676,6 +714,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
.hdr.size = sizeof(msg.payload.state),
};
+ vhost_user_host_notifier_remove(dev, ring->index);
+
if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
return -1;
}
@@ -849,6 +889,66 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
return ret;
}
+static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
+ VhostUserVringArea *area,
+ int fd)
+{
+ int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
+ size_t page_size = qemu_real_host_page_size;
+ struct vhost_user *u = dev->opaque;
+ VhostUserState *user = u->user;
+ VirtIODevice *vdev = dev->vdev;
+ VhostUserHostNotifier *n;
+ void *addr;
+ char *name;
+
+ if (!virtio_has_feature(dev->protocol_features,
+ VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
+ vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
+ return -1;
+ }
+
+ n = &user->notifier[queue_idx];
+
+ if (n->addr) {
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
+ object_unparent(OBJECT(&n->mr));
+ munmap(n->addr, page_size);
+ n->addr = NULL;
+ }
+
+ if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
+ return 0;
+ }
+
+ /* Sanity check. */
+ if (area->size != page_size) {
+ return -1;
+ }
+
+ addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+ fd, area->offset);
+ if (addr == MAP_FAILED) {
+ return -1;
+ }
+
+ name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
+ user, queue_idx);
+ memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+ page_size, addr);
+ g_free(name);
+
+ if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
+ munmap(addr, page_size);
+ return -1;
+ }
+
+ n->addr = addr;
+ n->set = true;
+
+ return 0;
+}
+
static void slave_read(void *opaque)
{
struct vhost_dev *dev = opaque;
@@ -917,6 +1017,10 @@ static void slave_read(void *opaque)
case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
ret = vhost_user_slave_handle_config_change(dev);
break;
+ case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
+ ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
+ fd[0]);
+ break;
default:
error_report("Received unexpected msg type.");
ret = -EINVAL;
@@ -1648,6 +1752,15 @@ VhostUserState *vhost_user_init(void)
void vhost_user_cleanup(VhostUserState *user)
{
+ int i;
+
+ for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+ if (user->notifier[i].addr) {
+ object_unparent(OBJECT(&user->notifier[i].mr));
+ munmap(user->notifier[i].addr, qemu_real_host_page_size);
+ user->notifier[i].addr = NULL;
+ }
+ }
}
const VhostOps user_ops = {
diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
index eb8bc0d90d..fd660393a0 100644
--- a/include/hw/virtio/vhost-user.h
+++ b/include/hw/virtio/vhost-user.h
@@ -9,9 +9,17 @@
#define HW_VIRTIO_VHOST_USER_H
#include "chardev/char-fe.h"
+#include "hw/virtio/virtio.h"
+
+typedef struct VhostUserHostNotifier {
+ MemoryRegion mr;
+ void *addr;
+ bool set;
+} VhostUserHostNotifier;
typedef struct VhostUserState {
CharBackend *chr;
+ VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
} VhostUserState;
VhostUserState *vhost_user_init(void);
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers
2018-05-24 10:33 ` [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers Tiwei Bie
@ 2018-05-24 15:49 ` Michael S. Tsirkin
2018-05-24 16:05 ` Tiwei Bie
0 siblings, 1 reply; 11+ messages in thread
From: Michael S. Tsirkin @ 2018-05-24 15:49 UTC (permalink / raw)
To: Tiwei Bie; +Cc: jasowang, qemu-devel, virtio-dev
On Thu, May 24, 2018 at 06:33:34PM +0800, Tiwei Bie wrote:
> This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER.
> With this feature negotiated, vhost-user backend can register
> memory region based host notifiers. And it will allow the guest
> driver in the VM to notify the hardware accelerator at the
> vhost-user backend directly.
>
> Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
So maybe we don't need a new VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD?
Let's check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER instead?
> ---
> docs/interop/vhost-user.txt | 33 ++++++++++
> hw/virtio/vhost-user.c | 113 +++++++++++++++++++++++++++++++++
> include/hw/virtio/vhost-user.h | 8 +++
> 3 files changed, 154 insertions(+)
>
> diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt
> index 682a683eb4..d51fd58242 100644
> --- a/docs/interop/vhost-user.txt
> +++ b/docs/interop/vhost-user.txt
> @@ -132,6 +132,16 @@ Depending on the request type, payload can be:
> Payload: Size bytes array holding the contents of the virtio
> device's configuration space
>
> + * Vring area description
> + -----------------------
> + | u64 | size | offset |
> + -----------------------
> +
> + u64: a 64-bit integer contains vring index and flags
> + Size: a 64-bit size of this area
> + Offset: a 64-bit offset of this area from the start of the
> + supplied file descriptor
> +
> In QEMU the vhost-user message is implemented with the following struct:
>
> typedef struct VhostUserMsg {
> @@ -146,6 +156,7 @@ typedef struct VhostUserMsg {
> VhostUserLog log;
> struct vhost_iotlb_msg iotlb;
> VhostUserConfig config;
> + VhostUserVringArea area;
> };
> } QEMU_PACKED VhostUserMsg;
>
> @@ -385,6 +396,7 @@ Protocol features
> #define VHOST_USER_PROTOCOL_F_PAGEFAULT 8
> #define VHOST_USER_PROTOCOL_F_CONFIG 9
> #define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10
> +#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11
>
> Master message types
> --------------------
> @@ -782,6 +794,27 @@ Slave message types
> the VHOST_USER_NEED_REPLY flag, master must respond with zero when
> operation is successfully completed, or non-zero otherwise.
>
> + * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
> +
> + Id: 3
> + Equivalent ioctl: N/A
> + Slave payload: vring area description
> + Master payload: N/A
> +
> + Sets host notifier for a specified queue. The queue index is contained
> + in the u64 field of the vring area description. The host notifier is
> + described by the file descriptor (typically it's a VFIO device fd) which
> + is passed as ancillary data and the size (which is mmap size and should
> + be the same as host page size) and offset (which is mmap offset) carried
> + in the vring area description. QEMU can mmap the file descriptor based
> + on the size and offset to get a memory range. Registering a host notifier
> + means mapping this memory range to the VM as the specified queue's notify
> + MMIO region. Slave sends this request to tell QEMU to de-register the
> + existing notifier if any and register the new notifier if the request is
> + sent with a file descriptor.
> + This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
> + protocol feature has been successfully negotiated.
> +
> VHOST_USER_PROTOCOL_F_REPLY_ACK:
> -------------------------------
> The original vhost-user specification only demands replies for certain
> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
> index 70b9610c87..b041343632 100644
> --- a/hw/virtio/vhost-user.c
> +++ b/hw/virtio/vhost-user.c
> @@ -13,6 +13,7 @@
> #include "hw/virtio/vhost.h"
> #include "hw/virtio/vhost-user.h"
> #include "hw/virtio/vhost-backend.h"
> +#include "hw/virtio/virtio.h"
> #include "hw/virtio/virtio-net.h"
> #include "chardev/char-fe.h"
> #include "sysemu/kvm.h"
> @@ -50,6 +51,7 @@ enum VhostUserProtocolFeature {
> VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
> VHOST_USER_PROTOCOL_F_CONFIG = 9,
> VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
> + VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
> VHOST_USER_PROTOCOL_F_MAX
> };
>
> @@ -94,6 +96,7 @@ typedef enum VhostUserSlaveRequest {
> VHOST_USER_SLAVE_NONE = 0,
> VHOST_USER_SLAVE_IOTLB_MSG = 1,
> VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
> + VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
> VHOST_USER_SLAVE_MAX
> } VhostUserSlaveRequest;
>
> @@ -138,6 +141,12 @@ static VhostUserConfig c __attribute__ ((unused));
> + sizeof(c.size) \
> + sizeof(c.flags))
>
> +typedef struct VhostUserVringArea {
> + uint64_t u64;
> + uint64_t size;
> + uint64_t offset;
> +} VhostUserVringArea;
> +
> typedef struct {
> VhostUserRequest request;
>
> @@ -159,6 +168,7 @@ typedef union {
> struct vhost_iotlb_msg iotlb;
> VhostUserConfig config;
> VhostUserCryptoSession session;
> + VhostUserVringArea area;
> } VhostUserPayload;
>
> typedef struct VhostUserMsg {
> @@ -640,9 +650,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,
> return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
> }
>
> +static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
> + int queue_idx)
> +{
> + struct vhost_user *u = dev->opaque;
> + VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
> + VirtIODevice *vdev = dev->vdev;
> +
> + if (n->addr && !n->set) {
> + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
> + n->set = true;
> + }
> +}
> +
> +static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
> + int queue_idx)
> +{
> + struct vhost_user *u = dev->opaque;
> + VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
> + VirtIODevice *vdev = dev->vdev;
> +
> + if (n->addr && n->set) {
> + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
> + n->set = false;
> + }
> +}
> +
> static int vhost_user_set_vring_base(struct vhost_dev *dev,
> struct vhost_vring_state *ring)
> {
> + vhost_user_host_notifier_restore(dev, ring->index);
> +
> return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
> }
>
> @@ -676,6 +714,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
> .hdr.size = sizeof(msg.payload.state),
> };
>
> + vhost_user_host_notifier_remove(dev, ring->index);
> +
> if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
> return -1;
> }
> @@ -849,6 +889,66 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
> return ret;
> }
>
> +static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
> + VhostUserVringArea *area,
> + int fd)
> +{
> + int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
> + size_t page_size = qemu_real_host_page_size;
> + struct vhost_user *u = dev->opaque;
> + VhostUserState *user = u->user;
> + VirtIODevice *vdev = dev->vdev;
> + VhostUserHostNotifier *n;
> + void *addr;
> + char *name;
> +
> + if (!virtio_has_feature(dev->protocol_features,
> + VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
> + vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
> + return -1;
> + }
> +
> + n = &user->notifier[queue_idx];
> +
> + if (n->addr) {
> + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
> + object_unparent(OBJECT(&n->mr));
> + munmap(n->addr, page_size);
> + n->addr = NULL;
> + }
> +
> + if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
> + return 0;
> + }
> +
> + /* Sanity check. */
> + if (area->size != page_size) {
> + return -1;
> + }
> +
> + addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
> + fd, area->offset);
> + if (addr == MAP_FAILED) {
> + return -1;
> + }
> +
> + name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
> + user, queue_idx);
> + memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
> + page_size, addr);
> + g_free(name);
> +
> + if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
> + munmap(addr, page_size);
> + return -1;
> + }
> +
> + n->addr = addr;
> + n->set = true;
> +
> + return 0;
> +}
> +
> static void slave_read(void *opaque)
> {
> struct vhost_dev *dev = opaque;
> @@ -917,6 +1017,10 @@ static void slave_read(void *opaque)
> case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
> ret = vhost_user_slave_handle_config_change(dev);
> break;
> + case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
> + ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
> + fd[0]);
> + break;
> default:
> error_report("Received unexpected msg type.");
> ret = -EINVAL;
> @@ -1648,6 +1752,15 @@ VhostUserState *vhost_user_init(void)
>
> void vhost_user_cleanup(VhostUserState *user)
> {
> + int i;
> +
> + for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
> + if (user->notifier[i].addr) {
> + object_unparent(OBJECT(&user->notifier[i].mr));
> + munmap(user->notifier[i].addr, qemu_real_host_page_size);
> + user->notifier[i].addr = NULL;
> + }
> + }
> }
>
> const VhostOps user_ops = {
> diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h
> index eb8bc0d90d..fd660393a0 100644
> --- a/include/hw/virtio/vhost-user.h
> +++ b/include/hw/virtio/vhost-user.h
> @@ -9,9 +9,17 @@
> #define HW_VIRTIO_VHOST_USER_H
>
> #include "chardev/char-fe.h"
> +#include "hw/virtio/virtio.h"
> +
> +typedef struct VhostUserHostNotifier {
> + MemoryRegion mr;
> + void *addr;
> + bool set;
> +} VhostUserHostNotifier;
>
> typedef struct VhostUserState {
> CharBackend *chr;
> + VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
> } VhostUserState;
So this notifier is per-queue. Can't we maintain it in
NetVhostUserState then?
>
> VhostUserState *vhost_user_init(void);
> --
> 2.17.0
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers
2018-05-24 15:49 ` Michael S. Tsirkin
@ 2018-05-24 16:05 ` Tiwei Bie
2018-05-24 16:15 ` Michael S. Tsirkin
0 siblings, 1 reply; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 16:05 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: jasowang, qemu-devel, virtio-dev
On Thu, May 24, 2018 at 06:49:47PM +0300, Michael S. Tsirkin wrote:
> On Thu, May 24, 2018 at 06:33:34PM +0800, Tiwei Bie wrote:
> > This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER.
> > With this feature negotiated, vhost-user backend can register
> > memory region based host notifiers. And it will allow the guest
> > driver in the VM to notify the hardware accelerator at the
> > vhost-user backend directly.
> >
> > Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
>
> So maybe we don't need a new VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD?
> Let's check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER instead?
Yeah. I think it would be the best choice!
If this feature (HOST NOTIFIER) is negotiated, it means
the QEMU is able to safely receive (the expected number
of) file descriptors.
>
> > ---
[...]
> > +typedef struct VhostUserHostNotifier {
> > + MemoryRegion mr;
> > + void *addr;
> > + bool set;
> > +} VhostUserHostNotifier;
> >
> > typedef struct VhostUserState {
> > CharBackend *chr;
> > + VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
> > } VhostUserState;
>
> So this notifier is per-queue. Can't we maintain it in
> NetVhostUserState then?
This notifier is per virtio-queue. But NetVhostUserState
is per net-queue-pair.
And ideally, I think this structure shouldn't be visible
to net/vhost-user, vhost-user-scsi, vhost-user-crypto, etc.
Best regards,
Tiwei Bie
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers
2018-05-24 16:05 ` Tiwei Bie
@ 2018-05-24 16:15 ` Michael S. Tsirkin
2018-05-24 16:25 ` Tiwei Bie
0 siblings, 1 reply; 11+ messages in thread
From: Michael S. Tsirkin @ 2018-05-24 16:15 UTC (permalink / raw)
To: Tiwei Bie; +Cc: jasowang, qemu-devel, virtio-dev
On Fri, May 25, 2018 at 12:05:50AM +0800, Tiwei Bie wrote:
> On Thu, May 24, 2018 at 06:49:47PM +0300, Michael S. Tsirkin wrote:
> > On Thu, May 24, 2018 at 06:33:34PM +0800, Tiwei Bie wrote:
> > > This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER.
> > > With this feature negotiated, vhost-user backend can register
> > > memory region based host notifiers. And it will allow the guest
> > > driver in the VM to notify the hardware accelerator at the
> > > vhost-user backend directly.
> > >
> > > Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
> >
> > So maybe we don't need a new VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD?
> > Let's check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER instead?
>
> Yeah. I think it would be the best choice!
>
> If this feature (HOST NOTIFIER) is negotiated, it means
> the QEMU is able to safely receive (the expected number
> of) file descriptors.
>
> >
> > > ---
> [...]
> > > +typedef struct VhostUserHostNotifier {
> > > + MemoryRegion mr;
> > > + void *addr;
> > > + bool set;
> > > +} VhostUserHostNotifier;
> > >
> > > typedef struct VhostUserState {
> > > CharBackend *chr;
> > > + VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
> > > } VhostUserState;
> >
> > So this notifier is per-queue. Can't we maintain it in
> > NetVhostUserState then?
>
> This notifier is per virtio-queue. But NetVhostUserState
> is per net-queue-pair.
>
> And ideally, I think this structure shouldn't be visible
> to net/vhost-user, vhost-user-scsi, vhost-user-crypto, etc.
>
> Best regards,
> Tiwei Bie
I wonder whether we can create a new per vq structure.
--
MST
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers
2018-05-24 16:15 ` Michael S. Tsirkin
@ 2018-05-24 16:25 ` Tiwei Bie
0 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 16:25 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: jasowang, qemu-devel, virtio-dev
On Thu, May 24, 2018 at 07:15:24PM +0300, Michael S. Tsirkin wrote:
> On Fri, May 25, 2018 at 12:05:50AM +0800, Tiwei Bie wrote:
> > On Thu, May 24, 2018 at 06:49:47PM +0300, Michael S. Tsirkin wrote:
> > > On Thu, May 24, 2018 at 06:33:34PM +0800, Tiwei Bie wrote:
> > > > This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER.
> > > > With this feature negotiated, vhost-user backend can register
> > > > memory region based host notifiers. And it will allow the guest
> > > > driver in the VM to notify the hardware accelerator at the
> > > > vhost-user backend directly.
> > > >
> > > > Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
> > >
> > > So maybe we don't need a new VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD?
> > > Let's check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER instead?
> >
> > Yeah. I think it would be the best choice!
> >
> > If this feature (HOST NOTIFIER) is negotiated, it means
> > the QEMU is able to safely receive (the expected number
> > of) file descriptors.
> >
> > >
> > > > ---
> > [...]
> > > > +typedef struct VhostUserHostNotifier {
> > > > + MemoryRegion mr;
> > > > + void *addr;
> > > > + bool set;
> > > > +} VhostUserHostNotifier;
> > > >
> > > > typedef struct VhostUserState {
> > > > CharBackend *chr;
> > > > + VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
> > > > } VhostUserState;
> > >
> > > So this notifier is per-queue. Can't we maintain it in
> > > NetVhostUserState then?
> >
> > This notifier is per virtio-queue. But NetVhostUserState
> > is per net-queue-pair.
> >
> > And ideally, I think this structure shouldn't be visible
> > to net/vhost-user, vhost-user-scsi, vhost-user-crypto, etc.
> >
> > Best regards,
> > Tiwei Bie
>
> I wonder whether we can create a new per vq structure.
Maybe something like:
typedef struct VhostUserStatePerVQ {
VhostUserHostNotifier notifier;
} VhostUserStatePerVQ;
typedef struct VhostUserState {
CharBackend *chr;
VhostUserStatePerVQ queue[VIRTIO_QUEUE_MAX];
} VhostUserState;
Best regards,
Tiwei Bie
^ permalink raw reply [flat|nested] 11+ messages in thread
* [Qemu-devel] [PATCH 5/6] libvhost-user: support host notifier
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
` (3 preceding siblings ...)
2018-05-24 10:33 ` [Qemu-devel] [PATCH 4/6] vhost-user: support registering external host notifiers Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
2018-05-24 10:33 ` [Qemu-devel] [PATCH 6/6] vhost-user-bridge: " Tiwei Bie
5 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
This patch introduces the host notifier support in
libvhost-user. A new API is added to support setting
host notifier for each queue.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
contrib/libvhost-user/libvhost-user.c | 81 ++++++++++++++++++++++++---
contrib/libvhost-user/libvhost-user.h | 32 +++++++++++
2 files changed, 105 insertions(+), 8 deletions(-)
diff --git a/contrib/libvhost-user/libvhost-user.c b/contrib/libvhost-user/libvhost-user.c
index 54e643d871..a6b46cdc03 100644
--- a/contrib/libvhost-user/libvhost-user.c
+++ b/contrib/libvhost-user/libvhost-user.c
@@ -314,11 +314,6 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
msg.msg_controllen = 0;
}
- /* Set the version in the flags when sending the reply */
- vmsg->flags &= ~VHOST_USER_VERSION_MASK;
- vmsg->flags |= VHOST_USER_VERSION;
- vmsg->flags |= VHOST_USER_REPLY_MASK;
-
do {
rc = sendmsg(conn_fd, &msg, 0);
} while (rc < 0 && (errno == EINTR || errno == EAGAIN));
@@ -341,6 +336,39 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
return true;
}
+static bool
+vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
+{
+ /* Set the version in the flags when sending the reply */
+ vmsg->flags &= ~VHOST_USER_VERSION_MASK;
+ vmsg->flags |= VHOST_USER_VERSION;
+ vmsg->flags |= VHOST_USER_REPLY_MASK;
+
+ return vu_message_write(dev, conn_fd, vmsg);
+}
+
+static bool
+vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
+{
+ VhostUserMsg msg_reply;
+
+ if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
+ return true;
+ }
+
+ if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {
+ return false;
+ }
+
+ if (msg_reply.request != vmsg->request) {
+ DPRINT("Received unexpected msg type. Expected %d received %d",
+ vmsg->request, msg_reply.request);
+ return false;
+ }
+
+ return msg_reply.payload.u64 == 0;
+}
+
/* Kick the log_call_fd if required. */
static void
vu_log_kick(VuDev *dev)
@@ -536,7 +564,7 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)
/* Send the message back to qemu with the addresses filled in */
vmsg->fd_num = 0;
- if (!vu_message_write(dev, dev->sock, vmsg)) {
+ if (!vu_send_reply(dev, dev->sock, vmsg)) {
vu_panic(dev, "failed to respond to set-mem-table for postcopy");
return false;
}
@@ -916,6 +944,41 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
}
}
+bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
+ int size, int offset)
+{
+ int qidx = vq - dev->vq;
+ int fd_num = 0;
+ VhostUserMsg vmsg = {
+ .request = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
+ .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
+ .size = sizeof(vmsg.payload.area),
+ .payload.area = {
+ .u64 = qidx & VHOST_USER_VRING_IDX_MASK,
+ .size = size,
+ .offset = offset,
+ },
+ };
+
+ if (fd == -1) {
+ vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
+ } else {
+ vmsg.fds[fd_num++] = fd;
+ }
+
+ vmsg.fd_num = fd_num;
+
+ if ((dev->protocol_features & VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) == 0) {
+ return false;
+ }
+
+ if (!vu_message_write(dev, dev->slave_fd, &vmsg)) {
+ return false;
+ }
+
+ return vu_process_message_reply(dev, &vmsg);
+}
+
static bool
vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
{
@@ -968,7 +1031,9 @@ static bool
vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
{
uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
- 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ;
+ 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ |
+ 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER |
+ 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD;
if (have_userfault()) {
features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
@@ -1252,7 +1317,7 @@ vu_dispatch(VuDev *dev)
goto end;
}
- if (!vu_message_write(dev, dev->sock, &vmsg)) {
+ if (!vu_send_reply(dev, dev->sock, &vmsg)) {
goto end;
}
diff --git a/contrib/libvhost-user/libvhost-user.h b/contrib/libvhost-user/libvhost-user.h
index b27075ea3b..4aa55b4d2d 100644
--- a/contrib/libvhost-user/libvhost-user.h
+++ b/contrib/libvhost-user/libvhost-user.h
@@ -51,6 +51,8 @@ enum VhostUserProtocolFeature {
VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
VHOST_USER_PROTOCOL_F_CONFIG = 9,
+ VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
+ VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
VHOST_USER_PROTOCOL_F_MAX
};
@@ -92,6 +94,14 @@ typedef enum VhostUserRequest {
VHOST_USER_MAX
} VhostUserRequest;
+typedef enum VhostUserSlaveRequest {
+ VHOST_USER_SLAVE_NONE = 0,
+ VHOST_USER_SLAVE_IOTLB_MSG = 1,
+ VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
+ VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
+ VHOST_USER_SLAVE_MAX
+} VhostUserSlaveRequest;
+
typedef struct VhostUserMemoryRegion {
uint64_t guest_phys_addr;
uint64_t memory_size;
@@ -122,6 +132,12 @@ static VhostUserConfig c __attribute__ ((unused));
+ sizeof(c.size) \
+ sizeof(c.flags))
+typedef struct VhostUserVringArea {
+ uint64_t u64;
+ uint64_t size;
+ uint64_t offset;
+} VhostUserVringArea;
+
#if defined(_WIN32)
# define VU_PACKED __attribute__((gcc_struct, packed))
#else
@@ -133,6 +149,7 @@ typedef struct VhostUserMsg {
#define VHOST_USER_VERSION_MASK (0x3)
#define VHOST_USER_REPLY_MASK (0x1 << 2)
+#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
uint32_t flags;
uint32_t size; /* the following payload size */
@@ -145,6 +162,7 @@ typedef struct VhostUserMsg {
VhostUserMemory memory;
VhostUserLog log;
VhostUserConfig config;
+ VhostUserVringArea area;
} payload;
int fds[VHOST_MEMORY_MAX_NREGIONS];
@@ -368,6 +386,20 @@ VuVirtq *vu_get_queue(VuDev *dev, int qidx);
void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
vu_queue_handler_cb handler);
+/**
+ * vu_set_queue_host_notifier:
+ * @dev: a VuDev context
+ * @vq: a VuVirtq queue
+ * @fd: a file descriptor
+ * @size: host page size
+ * @offset: notifier offset in @fd file
+ *
+ * Set queue's host notifier. This function may be called several
+ * times for the same queue. If called with -1 @fd, the notifier
+ * is removed.
+ */
+bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
+ int size, int offset);
/**
* vu_queue_set_notification:
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* [Qemu-devel] [PATCH 6/6] vhost-user-bridge: support host notifier
2018-05-24 10:33 [Qemu-devel] [PATCH 0/6] Support host notifiers in vhost-user Tiwei Bie
` (4 preceding siblings ...)
2018-05-24 10:33 ` [Qemu-devel] [PATCH 5/6] libvhost-user: support host notifier Tiwei Bie
@ 2018-05-24 10:33 ` Tiwei Bie
5 siblings, 0 replies; 11+ messages in thread
From: Tiwei Bie @ 2018-05-24 10:33 UTC (permalink / raw)
To: mst, jasowang, qemu-devel, virtio-dev; +Cc: tiwei.bie
This patch introduces the host notifier support in
vhost-user-bridge. A new option (-H) is added to use
the host notifier. This is mainly used to test the
host notifier implementation in vhost user.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
tests/vhost-user-bridge.c | 98 +++++++++++++++++++++++++++++++++++++--
1 file changed, 95 insertions(+), 3 deletions(-)
diff --git a/tests/vhost-user-bridge.c b/tests/vhost-user-bridge.c
index e0605a529e..0884294141 100644
--- a/tests/vhost-user-bridge.c
+++ b/tests/vhost-user-bridge.c
@@ -29,6 +29,7 @@
#define _FILE_OFFSET_BITS 64
+#include "qemu/atomic.h"
#include "qemu/osdep.h"
#include "qemu/iov.h"
#include "standard-headers/linux/virtio_net.h"
@@ -65,6 +66,11 @@ typedef struct VubrDev {
int sock;
int ready;
int quit;
+ struct {
+ int fd;
+ void *addr;
+ pthread_t thread;
+ } notifier;
} VubrDev;
static void
@@ -445,14 +451,22 @@ static uint64_t
vubr_get_features(VuDev *dev)
{
return 1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE |
- 1ULL << VIRTIO_NET_F_MRG_RXBUF;
+ 1ULL << VIRTIO_NET_F_MRG_RXBUF |
+ 1ULL << VIRTIO_F_VERSION_1;
}
static void
vubr_queue_set_started(VuDev *dev, int qidx, bool started)
{
+ VubrDev *vubr = container_of(dev, VubrDev, vudev);
VuVirtq *vq = vu_get_queue(dev, qidx);
+ if (started && vubr->notifier.fd >= 0) {
+ vu_set_queue_host_notifier(dev, vq, vubr->notifier.fd,
+ getpagesize(),
+ qidx * getpagesize());
+ }
+
if (qidx % 2 == 1) {
vu_set_queue_handler(dev, vq, started ? vubr_handle_tx : NULL);
}
@@ -522,6 +536,8 @@ vubr_new(const char *path, bool client)
vubr_die("socket");
}
+ dev->notifier.fd = -1;
+
un.sun_family = AF_UNIX;
strcpy(un.sun_path, path);
len = sizeof(un.sun_family) + strlen(path);
@@ -559,6 +575,73 @@ vubr_new(const char *path, bool client)
return dev;
}
+static void *notifier_thread(void *arg)
+{
+ VuDev *dev = (VuDev *)arg;
+ VubrDev *vubr = container_of(dev, VubrDev, vudev);
+ int pagesize = getpagesize();
+ int qidx;
+
+ while (true) {
+ for (qidx = 0; qidx < VHOST_MAX_NR_VIRTQUEUE; qidx++) {
+ uint16_t *n = vubr->notifier.addr + pagesize * qidx;
+
+ if (*n == qidx) {
+ *n = 0xffff;
+ /* We won't miss notifications if we reset
+ * the memory first. */
+ smp_mb();
+
+ DPRINT("Got a notification for queue%d via host notifier.\n",
+ qidx);
+
+ if (qidx % 2 == 1) {
+ vubr_handle_tx(dev, qidx);
+ }
+ }
+ usleep(1000);
+ }
+ }
+
+ return NULL;
+}
+
+static void
+vubr_host_notifier_setup(VubrDev *dev)
+{
+ char template[] = "/tmp/vubr-XXXXXX";
+ pthread_t thread;
+ size_t length;
+ void *addr;
+ int fd;
+
+ length = getpagesize() * VHOST_MAX_NR_VIRTQUEUE;
+
+ fd = mkstemp(template);
+ if (fd < 0) {
+ vubr_die("mkstemp()");
+ }
+
+ if (posix_fallocate(fd, 0, length) != 0) {
+ vubr_die("posix_fallocate()");
+ }
+
+ addr = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (addr == MAP_FAILED) {
+ vubr_die("mmap()");
+ }
+
+ memset(addr, 0xff, length);
+
+ if (pthread_create(&thread, NULL, notifier_thread, &dev->vudev) != 0) {
+ vubr_die("pthread_create()");
+ }
+
+ dev->notifier.fd = fd;
+ dev->notifier.addr = addr;
+ dev->notifier.thread = thread;
+}
+
static void
vubr_set_host(struct sockaddr_in *saddr, const char *host)
{
@@ -673,8 +756,9 @@ main(int argc, char *argv[])
VubrDev *dev;
int opt;
bool client = false;
+ bool host_notifier = false;
- while ((opt = getopt(argc, argv, "l:r:u:c")) != -1) {
+ while ((opt = getopt(argc, argv, "l:r:u:cH")) != -1) {
switch (opt) {
case 'l':
@@ -693,6 +777,9 @@ main(int argc, char *argv[])
case 'c':
client = true;
break;
+ case 'H':
+ host_notifier = true;
+ break;
default:
goto out;
}
@@ -708,6 +795,10 @@ main(int argc, char *argv[])
return 1;
}
+ if (host_notifier) {
+ vubr_host_notifier_setup(dev);
+ }
+
vubr_backend_udp_setup(dev, lhost, lport, rhost, rport);
vubr_run(dev);
@@ -717,7 +808,7 @@ main(int argc, char *argv[])
out:
fprintf(stderr, "Usage: %s ", argv[0]);
- fprintf(stderr, "[-c] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
+ fprintf(stderr, "[-c] [-H] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
fprintf(stderr, "\t-u path to unix doman socket. default: %s\n",
DEFAULT_UD_SOCKET);
fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
@@ -725,6 +816,7 @@ out:
fprintf(stderr, "\t-r remote host and port. default: %s:%s\n",
DEFAULT_RHOST, DEFAULT_RPORT);
fprintf(stderr, "\t-c client mode\n");
+ fprintf(stderr, "\t-H use host notifier\n");
return 1;
}
--
2.17.0
^ permalink raw reply related [flat|nested] 11+ messages in thread