* [Qemu-devel] [PULL 2/3] virtio-ccw: Wire up ioeventfd.
2013-06-25 16:25 [Qemu-devel] [PULL 0/3] virtio-ccw: fix + notifiers Cornelia Huck
2013-06-25 16:25 ` [Qemu-devel] [PULL 1/3] s390/virtio-ccw: Fix virtio reset Cornelia Huck
@ 2013-06-25 16:25 ` Cornelia Huck
2013-06-27 18:28 ` Stefan Weil
2013-06-25 16:25 ` [Qemu-devel] [PULL 3/3] virtio-ccw: Wire up guest and host notifies Cornelia Huck
2 siblings, 1 reply; 8+ messages in thread
From: Cornelia Huck @ 2013-06-25 16:25 UTC (permalink / raw)
To: qemu-devel
Cc: Blue Swirl, Cornelia Huck, Anthony Liguori, Aurélien Jarno
On hosts that support ioeventfd, make use of it for host-to-guest
notifications via diagnose 500.
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
hw/s390x/css.c | 2 +-
hw/s390x/css.h | 1 +
hw/s390x/virtio-ccw.c | 117 +++++++++++++++++++++++++++++++++++++++++++++++++
hw/s390x/virtio-ccw.h | 7 +++
target-s390x/cpu.h | 16 +++++++
target-s390x/kvm.c | 19 ++++++++
6 files changed, 161 insertions(+), 1 deletion(-)
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index f82abfe..93b0b97 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -68,7 +68,7 @@ int css_create_css_image(uint8_t cssid, bool default_image)
return 0;
}
-static uint16_t css_build_subchannel_id(SubchDev *sch)
+uint16_t css_build_subchannel_id(SubchDev *sch)
{
if (channel_subsys->max_cssid > 0) {
return (sch->cssid << 8) | (1 << 3) | (sch->ssid << 1) | 1;
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
index 85ed05d..b536ab5 100644
--- a/hw/s390x/css.h
+++ b/hw/s390x/css.h
@@ -90,6 +90,7 @@ bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno);
void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
uint16_t devno, SubchDev *sch);
void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type);
+uint16_t css_build_subchannel_id(SubchDev *sch);
void css_reset(void);
void css_reset_sch(SubchDev *sch);
void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid);
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index de51589..4ad9d9a 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -63,6 +63,90 @@ VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch)
return vdev;
}
+static int virtio_ccw_set_guest2host_notifier(VirtioCcwDevice *dev, int n,
+ bool assign, bool set_handler)
+{
+ VirtQueue *vq = virtio_get_queue(dev->vdev, n);
+ EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
+ int r = 0;
+ SubchDev *sch = dev->sch;
+ uint32_t sch_id = (css_build_subchannel_id(sch) << 16) | sch->schid;
+
+ if (assign) {
+ r = event_notifier_init(notifier, 1);
+ if (r < 0) {
+ error_report("%s: unable to init event notifier: %d", __func__, r);
+ return r;
+ }
+ virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
+ r = s390_assign_subch_ioeventfd(event_notifier_get_fd(notifier), sch_id,
+ n, assign);
+ if (r < 0) {
+ error_report("%s: unable to assign ioeventfd: %d", __func__, r);
+ virtio_queue_set_host_notifier_fd_handler(vq, false, false);
+ event_notifier_cleanup(notifier);
+ return r;
+ }
+ } else {
+ virtio_queue_set_host_notifier_fd_handler(vq, false, false);
+ s390_assign_subch_ioeventfd(event_notifier_get_fd(notifier), sch_id,
+ n, assign);
+ event_notifier_cleanup(notifier);
+ }
+ return r;
+}
+
+static void virtio_ccw_start_ioeventfd(VirtioCcwDevice *dev)
+{
+ int n, r;
+
+ if (!(dev->flags & VIRTIO_CCW_FLAG_USE_IOEVENTFD) ||
+ dev->ioeventfd_started) {
+ return;
+ }
+ for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+ if (!virtio_queue_get_num(dev->vdev, n)) {
+ continue;
+ }
+ r = virtio_ccw_set_guest2host_notifier(dev, n, true, true);
+ if (r < 0) {
+ goto assign_error;
+ }
+ }
+ dev->ioeventfd_started = true;
+ return;
+
+ assign_error:
+ while (--n >= 0) {
+ if (!virtio_queue_get_num(dev->vdev, n)) {
+ continue;
+ }
+ r = virtio_ccw_set_guest2host_notifier(dev, n, false, false);
+ assert(r >= 0);
+ }
+ dev->ioeventfd_started = false;
+ /* Disable ioeventfd for this device. */
+ dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
+ error_report("%s: failed. Fallback to userspace (slower).", __func__);
+}
+
+static void virtio_ccw_stop_ioeventfd(VirtioCcwDevice *dev)
+{
+ int n, r;
+
+ if (!dev->ioeventfd_started) {
+ return;
+ }
+ for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+ if (!virtio_queue_get_num(dev->vdev, n)) {
+ continue;
+ }
+ r = virtio_ccw_set_guest2host_notifier(dev, n, false, false);
+ assert(r >= 0);
+ }
+ dev->ioeventfd_started = false;
+}
+
VirtualCssBus *virtual_css_bus_init(void)
{
VirtualCssBus *cbus;
@@ -187,6 +271,7 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
}
break;
case CCW_CMD_VDEV_RESET:
+ virtio_ccw_stop_ioeventfd(dev);
virtio_reset(dev->vdev);
ret = 0;
break;
@@ -309,10 +394,16 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
ret = -EFAULT;
} else {
status = ldub_phys(ccw.cda);
+ if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+ virtio_ccw_stop_ioeventfd(dev);
+ }
virtio_set_status(dev->vdev, status);
if (dev->vdev->status == 0) {
virtio_reset(dev->vdev);
}
+ if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+ virtio_ccw_start_ioeventfd(dev);
+ }
sch->curr_status.scsw.count = ccw.count - sizeof(status);
ret = 0;
}
@@ -540,6 +631,7 @@ static int virtio_ccw_exit(VirtioCcwDevice *dev)
{
SubchDev *sch = dev->sch;
+ virtio_ccw_stop_ioeventfd(dev);
if (sch) {
css_subch_assign(sch->cssid, sch->ssid, sch->schid, sch->devno, NULL);
g_free(sch);
@@ -801,12 +893,24 @@ static void virtio_ccw_reset(DeviceState *d)
{
VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+ virtio_ccw_stop_ioeventfd(dev);
virtio_reset(dev->vdev);
css_reset_sch(dev->sch);
dev->indicators = 0;
dev->indicators2 = 0;
}
+static void virtio_ccw_vmstate_change(DeviceState *d, bool running)
+{
+ VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+ if (running) {
+ virtio_ccw_start_ioeventfd(dev);
+ } else {
+ virtio_ccw_stop_ioeventfd(dev);
+ }
+}
+
/**************** Virtio-ccw Bus Device Descriptions *******************/
static Property virtio_ccw_net_properties[] = {
@@ -814,6 +918,8 @@ static Property virtio_ccw_net_properties[] = {
DEFINE_VIRTIO_NET_FEATURES(VirtioCcwDevice, host_features[0]),
DEFINE_VIRTIO_NET_PROPERTIES(VirtIONetCcw, vdev.net_conf),
DEFINE_NIC_PROPERTIES(VirtIONetCcw, vdev.nic_conf),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -840,6 +946,8 @@ static Property virtio_ccw_blk_properties[] = {
DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
DEFINE_VIRTIO_BLK_FEATURES(VirtioCcwDevice, host_features[0]),
DEFINE_VIRTIO_BLK_PROPERTIES(VirtIOBlkCcw, blk),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -866,6 +974,8 @@ static Property virtio_ccw_serial_properties[] = {
DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
DEFINE_VIRTIO_SERIAL_PROPERTIES(VirtioSerialCcw, vdev.serial),
DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -891,6 +1001,8 @@ static const TypeInfo virtio_ccw_serial = {
static Property virtio_ccw_balloon_properties[] = {
DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -917,6 +1029,8 @@ static Property virtio_ccw_scsi_properties[] = {
DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOSCSICcw, vdev.parent_obj.conf),
DEFINE_VIRTIO_SCSI_FEATURES(VirtioCcwDevice, host_features[0]),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -980,6 +1094,8 @@ static Property virtio_ccw_rng_properties[] = {
DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
DEFINE_VIRTIO_RNG_PROPERTIES(VirtIORNGCcw, vdev.conf),
+ DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+ VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
DEFINE_PROP_END_OF_LIST(),
};
@@ -1107,6 +1223,7 @@ static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
bus_class->max_dev = 1;
k->notify = virtio_ccw_notify;
k->get_features = virtio_ccw_get_features;
+ k->vmstate_change = virtio_ccw_vmstate_change;
}
static const TypeInfo virtio_ccw_bus_info = {
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
index 121a5f8..433f27a 100644
--- a/hw/s390x/virtio-ccw.h
+++ b/hw/s390x/virtio-ccw.h
@@ -69,6 +69,11 @@ typedef struct VirtIOCCWDeviceClass {
/* Change here if we want to support more feature bits. */
#define VIRTIO_CCW_FEATURE_SIZE 1
+/* Performance improves when virtqueue kick processing is decoupled from the
+ * vcpu thread using ioeventfd for some devices. */
+#define VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT 1
+#define VIRTIO_CCW_FLAG_USE_IOEVENTFD (1 << VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT)
+
struct VirtioCcwDevice {
DeviceState parent_obj;
SubchDev *sch;
@@ -76,6 +81,8 @@ struct VirtioCcwDevice {
char *bus_id;
uint32_t host_features[VIRTIO_CCW_FEATURE_SIZE];
VirtioBusState bus;
+ bool ioeventfd_started;
+ uint32_t flags;
/* Guest provided values: */
hwaddr indicators;
hwaddr indicators2;
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 0ce82cf..918c819 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -1081,6 +1081,7 @@ void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
void kvm_s390_crw_mchk(S390CPU *cpu);
void kvm_s390_enable_css_support(S390CPU *cpu);
int kvm_s390_get_registers_partial(CPUState *cpu);
+int kvm_s390_assign_subch_ioeventfd(int fd, uint32_t sch, int vq, bool assign);
#else
static inline void kvm_s390_io_interrupt(S390CPU *cpu,
uint16_t subchannel_id,
@@ -1099,6 +1100,11 @@ static inline int kvm_s390_get_registers_partial(CPUState *cpu)
{
return -ENOSYS;
}
+static inline int kvm_s390_assign_subch_ioeventfd(int fd, uint32_t sch, int vq,
+ bool assign)
+{
+ return -ENOSYS;
+}
#endif
static inline void s390_io_interrupt(S390CPU *cpu,
@@ -1125,4 +1131,14 @@ static inline void s390_crw_mchk(S390CPU *cpu)
}
}
+static inline int s390_assign_subch_ioeventfd(int fd, uint32_t sch_id, int vq,
+ bool assign)
+{
+ if (kvm_enabled()) {
+ return kvm_s390_assign_subch_ioeventfd(fd, sch_id, vq, assign);
+ } else {
+ return -ENOSYS;
+ }
+}
+
#endif
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 4d9ac4a..650d3a5 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -932,3 +932,22 @@ void kvm_s390_enable_css_support(S390CPU *cpu)
void kvm_arch_init_irq_routing(KVMState *s)
{
}
+
+int kvm_s390_assign_subch_ioeventfd(int fd, uint32_t sch, int vq, bool assign)
+{
+ struct kvm_ioeventfd kick = {
+ .flags = KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY |
+ KVM_IOEVENTFD_FLAG_DATAMATCH,
+ .fd = fd,
+ .datamatch = vq,
+ .addr = sch,
+ .len = 8,
+ };
+ if (!kvm_check_extension(kvm_state, KVM_CAP_IOEVENTFD)) {
+ return -ENOSYS;
+ }
+ if (!assign) {
+ kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
+ }
+ return kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
+}
--
1.7.9.5
^ permalink raw reply related [flat|nested] 8+ messages in thread* [Qemu-devel] [PULL 3/3] virtio-ccw: Wire up guest and host notifies.
2013-06-25 16:25 [Qemu-devel] [PULL 0/3] virtio-ccw: fix + notifiers Cornelia Huck
2013-06-25 16:25 ` [Qemu-devel] [PULL 1/3] s390/virtio-ccw: Fix virtio reset Cornelia Huck
2013-06-25 16:25 ` [Qemu-devel] [PULL 2/3] virtio-ccw: Wire up ioeventfd Cornelia Huck
@ 2013-06-25 16:25 ` Cornelia Huck
2 siblings, 0 replies; 8+ messages in thread
From: Cornelia Huck @ 2013-06-25 16:25 UTC (permalink / raw)
To: qemu-devel
Cc: Blue Swirl, Cornelia Huck, Anthony Liguori, Aurélien Jarno
Guest and host notifiers are needed by vhost. We use ioeventfds for
the guest notifiers, but need to fall back on qemu injecting interrupts
for the host notifiers.
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
hw/s390x/virtio-ccw.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++
hw/s390x/virtio-ccw.h | 1 +
2 files changed, 89 insertions(+)
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index 4ad9d9a..faef5dd 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -101,6 +101,7 @@ static void virtio_ccw_start_ioeventfd(VirtioCcwDevice *dev)
int n, r;
if (!(dev->flags & VIRTIO_CCW_FLAG_USE_IOEVENTFD) ||
+ dev->ioeventfd_disabled ||
dev->ioeventfd_started) {
return;
}
@@ -911,6 +912,90 @@ static void virtio_ccw_vmstate_change(DeviceState *d, bool running)
}
}
+static bool virtio_ccw_query_guest_notifiers(DeviceState *d)
+{
+ VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+ return !!(dev->sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ENA);
+}
+
+static int virtio_ccw_set_host_notifier(DeviceState *d, int n, bool assign)
+{
+ VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+ /* Stop using the generic ioeventfd, we are doing eventfd handling
+ * ourselves below */
+ dev->ioeventfd_disabled = assign;
+ if (assign) {
+ virtio_ccw_stop_ioeventfd(dev);
+ }
+ return virtio_ccw_set_guest2host_notifier(dev, n, assign, false);
+}
+
+static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n,
+ bool assign, bool with_irqfd)
+{
+ VirtQueue *vq = virtio_get_queue(dev->vdev, n);
+ EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
+ VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(dev->vdev);
+
+ if (assign) {
+ int r = event_notifier_init(notifier, 0);
+
+ if (r < 0) {
+ return r;
+ }
+ virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
+ /* We do not support irqfd for classic I/O interrupts, because the
+ * classic interrupts are intermixed with the subchannel status, that
+ * is queried with test subchannel. We want to use vhost, though.
+ * Lets make sure to have vhost running and wire up the irq fd to
+ * land in qemu (and only the irq fd) in this code.
+ */
+ if (k->guest_notifier_mask) {
+ k->guest_notifier_mask(dev->vdev, n, false);
+ }
+ /* get lost events and re-inject */
+ if (k->guest_notifier_pending &&
+ k->guest_notifier_pending(dev->vdev, n)) {
+ event_notifier_set(notifier);
+ }
+ } else {
+ if (k->guest_notifier_mask) {
+ k->guest_notifier_mask(dev->vdev, n, true);
+ }
+ virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
+ event_notifier_cleanup(notifier);
+ }
+ return 0;
+}
+
+static int virtio_ccw_set_guest_notifiers(DeviceState *d, int nvqs,
+ bool assigned)
+{
+ VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+ VirtIODevice *vdev = dev->vdev;
+ int r, n;
+
+ for (n = 0; n < nvqs; n++) {
+ if (!virtio_queue_get_num(vdev, n)) {
+ break;
+ }
+ /* false -> true, as soon as irqfd works */
+ r = virtio_ccw_set_guest_notifier(dev, n, assigned, false);
+ if (r < 0) {
+ goto assign_error;
+ }
+ }
+ return 0;
+
+assign_error:
+ while (--n >= 0) {
+ virtio_ccw_set_guest_notifier(dev, n, !assigned, false);
+ }
+ return r;
+}
+
/**************** Virtio-ccw Bus Device Descriptions *******************/
static Property virtio_ccw_net_properties[] = {
@@ -1224,6 +1309,9 @@ static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
k->notify = virtio_ccw_notify;
k->get_features = virtio_ccw_get_features;
k->vmstate_change = virtio_ccw_vmstate_change;
+ k->query_guest_notifiers = virtio_ccw_query_guest_notifiers;
+ k->set_host_notifier = virtio_ccw_set_host_notifier;
+ k->set_guest_notifiers = virtio_ccw_set_guest_notifiers;
}
static const TypeInfo virtio_ccw_bus_info = {
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
index 433f27a..96d6f5d 100644
--- a/hw/s390x/virtio-ccw.h
+++ b/hw/s390x/virtio-ccw.h
@@ -82,6 +82,7 @@ struct VirtioCcwDevice {
uint32_t host_features[VIRTIO_CCW_FEATURE_SIZE];
VirtioBusState bus;
bool ioeventfd_started;
+ bool ioeventfd_disabled;
uint32_t flags;
/* Guest provided values: */
hwaddr indicators;
--
1.7.9.5
^ permalink raw reply related [flat|nested] 8+ messages in thread