[PATCH 1/3] virt-core: binding together drivers and hypervisors

[PATCH 1/3] virt-core: binding together drivers and hypervisors

[Michael S. Tsirkin]

This adds support for virt-core: generic glue making it possible to implement
kernel-level accelerators (drivers) which are independent of a hypervisor, that
is can in theory work on top of either kvm or lguest.

Each driver and hypervisor registers with core, and then when user adds a
virtual device, drivers can find and interfact with hypervisors through the
virt_dev object.

Users add devices by making a hypervisor-specific system call (e.g. ioctl)
the result of which is a file descriptor controlling the device.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 arch/x86/kvm/Kconfig     |    1 +
 drivers/Makefile         |    1 +
 drivers/virt/Kconfig     |    5 ++
 drivers/virt/Makefile    |    1 +
 drivers/virt/virt_core.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virt.h     |   94 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 213 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virt/Kconfig
 create mode 100644 drivers/virt/Makefile
 create mode 100644 drivers/virt/virt_core.c
 create mode 100644 include/linux/virt.h

diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index a58504e..693b807 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -71,6 +71,7 @@ config KVM_TRACE
 
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under
 # the virtualization menu.
+source drivers/virt/Kconfig
 source drivers/lguest/Kconfig
 source drivers/virtio/Kconfig
 
diff --git a/drivers/Makefile b/drivers/Makefile
index 1266ead..9f14b08 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -104,6 +104,7 @@ obj-$(CONFIG_HID)		+= hid/
 obj-$(CONFIG_PPC_PS3)		+= ps3/
 obj-$(CONFIG_OF)		+= of/
 obj-$(CONFIG_SSB)		+= ssb/
+obj-$(CONFIG_VIRT_CORE)		+= virt/
 obj-$(CONFIG_VIRTIO)		+= virtio/
 obj-$(CONFIG_STAGING)		+= staging/
 obj-y				+= platform/
diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig
new file mode 100644
index 0000000..ace7b2e
--- /dev/null
+++ b/drivers/virt/Kconfig
@@ -0,0 +1,5 @@
+config VIRT_CORE
+	tristate "Generic virtual device binding support"
+	---help---
+	  Core support for binding kernel drivers to virtual devices.
+	  Make sure to also select any drivers you wish to use.
diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile
new file mode 100644
index 0000000..7a77047
--- /dev/null
+++ b/drivers/virt/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIRT_CORE) += virt_core.o
diff --git a/drivers/virt/virt_core.c b/drivers/virt/virt_core.c
new file mode 100644
index 0000000..89e1f53
--- /dev/null
+++ b/drivers/virt/virt_core.c
@@ -0,0 +1,111 @@
+/*
+ * Virt core support: this is the glue that binds together drivers for virtual devices
+ * and hypervisors.
+ *
+ * Copyright (c) 2009 Red Hat Inc.
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/list.h>
+#include <linux/anon_inodes.h>
+
+#include <linux/virt.h>
+
+/* virt bus implementation */
+static LIST_HEAD(driver_list);
+static LIST_HEAD(hypervisor_list);
+static DEFINE_MUTEX(driver_lock);
+static DEFINE_MUTEX(hypervisor_lock);
+
+static int virt_close(struct inode *inode, struct file *filp)
+{
+	struct virt_dev *dev = filp->private_data;
+	dev->driver->device_remove(dev->driver, dev);
+	dev->hypervisor->put(dev->hypervisor);
+	kfree(dev);
+	return 0;
+}
+
+void virt_driver_register(struct virt_driver *driver,
+			     struct file_operations *fops)
+{
+	BUG_ON(fops->release);
+	BUG_ON(fops->open);
+	driver->f_ops = fops;
+	fops->release = virt_close;
+	mutex_lock(&driver_lock);
+	list_add(&driver->list, &driver_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(virt_driver_register);
+
+void virt_driver_unregister(struct virt_driver *driver)
+{
+	mutex_lock(&driver_lock);
+	list_del(&driver->list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(virt_driver_unregister);
+
+void virt_hypervisor_register(struct virt_hypervisor *hypervisor)
+{
+	mutex_lock(&hypervisor_lock);
+	list_add(&hypervisor->list, &hypervisor_list);
+	mutex_unlock(&hypervisor_lock);
+}
+EXPORT_SYMBOL_GPL(virt_hypervisor_register);
+
+void virt_hypervisor_unregister(struct virt_hypervisor *hypervisor)
+{
+	mutex_lock(&hypervisor_lock);
+	list_del(&hypervisor->list);
+	mutex_unlock(&hypervisor_lock);
+}
+EXPORT_SYMBOL_GPL(virt_hypervisor_unregister);
+
+int virt_device_create(struct virt_hypervisor *hypervisor, int flags,
+		       const u8 *id, int id_len)
+{
+	struct virt_dev *dev = kmalloc(sizeof *dev, GFP_KERNEL);
+	struct virt_driver *driver;
+	const struct file_operations *fops;
+	int ret;
+	if (!dev)
+		return -ENOMEM;
+	dev->hypervisor = hypervisor;
+	hypervisor->get(hypervisor);
+
+	mutex_lock(&driver_lock);
+	ret = -ENODEV;
+	list_for_each_entry(driver, &driver_list, list) {
+		ret = driver->device_probe(driver, dev, id, id_len);
+		if (ret != -ENODEV)
+			break;
+	}
+	mutex_unlock(&driver_lock);
+	if (ret)
+		goto err_probe;
+
+	fops = fops_get(driver->f_ops);
+	if (!fops) {
+		ret = -ENOMEM;
+		goto err_file;
+	}
+
+	ret = anon_inode_getfd(driver->name, fops, dev, flags);
+	if (ret < 0)
+		goto err_file;
+	return ret;
+err_file:
+	driver->device_remove(driver, dev);
+err_probe:
+	hypervisor->put(hypervisor);
+	kfree(dev);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(virt_device_create);
diff --git a/include/linux/virt.h b/include/linux/virt.h
new file mode 100644
index 0000000..ecc0aef
--- /dev/null
+++ b/include/linux/virt.h
@@ -0,0 +1,94 @@
+#ifndef LINUX_VIRT_H
+#define LINUX_VIRT_H
+/* Virt core support.
+ * Each driver and hypervisor register with core, and then drivers can find
+ * and interfact with hypervisors through the virt_dev object.
+ */
+
+#include <linux/list.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+
+struct virt_dev;
+
+struct virt_hypervisor {
+	/* Fields set by virt core. */
+	struct list_head list;
+	/* Fields set by hypervisor. */
+	/* Operations supported by this hypervisor. */
+	/* core uses reference counting to make sure hypervisor does not
+	 * go away while there are devices attached to it. */
+	void (*get)(struct virt_hypervisor *);
+	void (*put)(struct virt_hypervisor *);
+	/* Optional interrupt injection support */
+	int (*set_irq)(struct virt_hypervisor *,
+		       int source, int irq, int level);
+};
+
+struct virt_driver {
+	/* Fields set by virt core. */
+	struct list_head list;
+	const struct file_operations *f_ops;
+	/* Fields set by driver. */
+	const char *name;
+	/* Check whether this driver wants this device,
+	 * and bind driver to device if yes.
+	 * Return -ENODEV if driver does not match this device,
+	 * 0 on success, any other value is an error. */
+	int  (*device_probe)(struct virt_driver *, struct virt_dev *,
+			     const void *id, int id_len);
+	/* Called on removal of all devices where this driver's probe
+	 * returned success */
+	void (*device_remove)(struct virt_driver *, struct virt_dev *);
+};
+
+/* Device objects are allocated and freed by virt core. */
+struct virt_dev {
+	struct virt_driver *driver;
+	void *driver_ctx;
+	struct virt_hypervisor *hypervisor;
+};
+
+/* Utility function for file_operations in drivers. */
+static inline struct virt_dev *virt_dev_get(struct file *file)
+{
+	return file->private_data;
+}
+
+#ifdef CONFIG_VIRT_CORE_MODULE
+void virt_driver_register(struct virt_driver *, struct file_operations *);
+void virt_driver_unregister(struct virt_driver *);
+
+void virt_hypervisor_register(struct virt_hypervisor *);
+void virt_hypervisor_unregister(struct virt_hypervisor *);
+
+
+/* Users add devices by making a hypervisor-specific system call (e.g. ioctl)
+ * the result of which is a file descriptor controlling the device.
+ * Hypervisors implement this by calling virt_device_create to get struct file
+ * and install it in file descriptor table.  Device is later released by
+ * closing the file descriptor */
+/* Associate a file descriptor with the device and return it.
+ * Returns a value < 0 on failure. */
+int virt_device_create(struct virt_hypervisor *, int flags,
+		       const u8 *id, int id_len);
+#else
+static inline
+void virt_driver_register(struct virt_driver *d, struct file_operations *o) {}
+static inline
+void virt_driver_unregister(struct virt_driver *d) {}
+
+static inline
+void virt_hypervisor_register(struct virt_hypervisor *h) {}
+static inline
+void virt_hypervisor_unregister(struct virt_hypervisor *h) {}
+
+static inline
+struct file *virt_device_create(struct virt_hypervisor *h, int flags,
+				const u8 *id, int id_len)
+{
+	return ERR_PTR(-ENOTTY);
+}
+#endif
+
+#endif
-- 
1.6.3.1.175.g3be7e0

[PATCH 2/3] kvm: virtual device support

[Michael S. Tsirkin]

Implement virt_hypervisor in kvm, and add support for creating virtual devices.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 include/linux/kvm.h      |   13 ++++++++++++
 include/linux/kvm_host.h |    3 ++
 virt/kvm/kvm_main.c      |   47 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 0 deletions(-)

diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 8cc1379..8f6cc79 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -487,6 +487,8 @@ struct kvm_irq_routing {
 #define KVM_REINJECT_CONTROL      _IO(KVMIO, 0x71)
 #define KVM_DEASSIGN_PCI_DEVICE _IOW(KVMIO, 0x72, \
 				     struct kvm_assigned_pci_dev)
+#define KVM_CREATE_VIRT_DEVICE  _IOW(KVMIO, 0x73, \
+				     struct kvm_create_virt_device)
 
 /*
  * ioctls for vcpu fds
@@ -602,6 +604,17 @@ struct kvm_assigned_irq {
 	};
 };
 
+struct kvm_create_virt_device {
+	__s32 flags; /* flags from open(2). Only O_CLOEXEC supported for now. */
+	__u32 id_len;
+	union {
+		__u32 reserved[12];
+	};
+	/* Followed by up to MAX_ID_LEN byte id. */
+};
+
+#define KVM_VIRT_DEVICE_MAX_ID_LEN	200
+
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 
 #define KVM_DEV_IRQ_ASSIGN_MSI_ACTION	KVM_DEV_IRQ_ASSIGN_ENABLE_MSI
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 894a56e..7131851 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -26,6 +26,8 @@
 
 #include <asm/kvm_host.h>
 
+#include <linux/virt.h>
+
 /*
  * vcpu->requests bit members
  */
@@ -152,6 +154,7 @@ struct kvm {
 	unsigned long mmu_notifier_seq;
 	long mmu_notifier_count;
 #endif
+	struct virt_hypervisor hypervisor;
 };
 
 /* The guest did something we don't support. */
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 1ecbe23..c0487f0 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -766,6 +766,24 @@ static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 	kvm_arch_flush_shadow(kvm);
 }
 
+static void kvm_hypervisor_get(struct virt_hypervisor *h)
+{
+	struct kvm *kvm = container_of(h, struct kvm, hypervisor);
+	kvm_get_kvm(kvm);
+}
+
+static void kvm_hypervisor_put(struct virt_hypervisor *h)
+{
+	struct kvm *kvm = container_of(h, struct kvm, hypervisor);
+	kvm_put_kvm(kvm);
+}
+static int kvm_hypervisor_set_irq(struct virt_hypervisor *h,
+				  int source, int irq, int level)
+{
+	struct kvm *kvm = container_of(h, struct kvm, hypervisor);
+	return kvm_set_irq(kvm, source, irq, level);
+}
+
 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
 	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
@@ -828,6 +846,10 @@ static struct kvm *kvm_create_vm(void)
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	kvm_coalesced_mmio_init(kvm);
 #endif
+	kvm->hypervisor.get = kvm_hypervisor_get;
+	kvm->hypervisor.put = kvm_hypervisor_put;
+	kvm->hypervisor.set_irq = kvm_hypervisor_set_irq;
+	virt_hypervisor_register(&kvm->hypervisor);
 out:
 	return kvm;
 }
@@ -865,6 +887,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 {
 	struct mm_struct *mm = kvm->mm;
 
+	virt_hypervisor_unregister(&kvm->hypervisor);
 	kvm_arch_sync_events(kvm);
 	spin_lock(&kvm_lock);
 	list_del(&kvm->vm_list);
@@ -1917,6 +1940,30 @@ static long kvm_vm_ioctl(struct file *filp,
 		vfree(entries);
 		break;
 	}
+	case KVM_CREATE_VIRT_DEVICE: {
+		struct kvm_create_virt_device d;
+		u8 *id;
+		r = -EFAULT;
+		if (copy_from_user(&d, argp, sizeof d))
+			goto out;
+		r = -EINVAL;
+		if (d.id_len > KVM_VIRT_DEVICE_MAX_ID_LEN)
+			goto out;
+		if (d.flags & ~O_CLOEXEC)
+			goto out;
+		r = -ENOMEM;
+		id = kmalloc(d.id_len, GFP_KERNEL);
+		if (!id)
+			goto out_free_id;
+		r = -EFAULT;
+		if (copy_from_user(id, argp + sizeof d, d.id_len))
+			goto out_free_id;
+		r = virt_device_create(&kvm->hypervisor, d.flags,
+				       id, d.id_len);
+	out_free_id:
+		kfree(id);
+		break;
+	}
 #endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
-- 
1.6.3.1.175.g3be7e0

[PATCH 3/3] virt_irq: virtual device for injecting interrupts

[Michael S. Tsirkin]

virt_irq is an alternative to irqfd interface, based
on the virt core infrastructure, which also serves
as an example of virt core usage.

The main advantage here compared to irqfd is the use of fd
created by the virt core, which avoids any possibility of
deadlock issues with eventfd and kvm file descriptors
referencing each other.

As a minor positive side effect, we don't need an
extra lock and don't need to schedule work to inject
the interrupt.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 drivers/virt/Kconfig     |    6 +++
 drivers/virt/Makefile    |    1 +
 drivers/virt/virt_irq.c  |   78 ++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virt_irq.h |   19 +++++++++++
 4 files changed, 104 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virt/virt_irq.c
 create mode 100644 include/linux/virt_irq.h

diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig
index ace7b2e..060c8da 100644
--- a/drivers/virt/Kconfig
+++ b/drivers/virt/Kconfig
@@ -3,3 +3,9 @@ config VIRT_CORE
 	---help---
 	  Core support for binding kernel drivers to virtual devices.
 	  Make sure to also select any drivers you wish to use.
+
+config VIRT_IRQ
+	tristate "Virtual device for injecting interrupts from userspace"
+	depends on VIRT_CORE
+	---help---
+	  Simple virtual device that supports injecting interrupts.
diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile
index 7a77047..0072530 100644
--- a/drivers/virt/Makefile
+++ b/drivers/virt/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_VIRT_CORE) += virt_core.o
+obj-$(CONFIG_VIRT_IRQ) += virt_irq.o
diff --git a/drivers/virt/virt_irq.c b/drivers/virt/virt_irq.c
new file mode 100644
index 0000000..c10087e
--- /dev/null
+++ b/drivers/virt/virt_irq.c
@@ -0,0 +1,78 @@
+/*
+ * Virt irq device: simple virtual device for interrupt injection.
+ *
+ * Copyright (c) 2009 Red Hat Inc.
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/virt.h>
+#include <linux/virt_irq.h>
+
+struct virt_irq_dev {
+	int irq;
+};
+
+static ssize_t virt_irq_write(struct file *f, const char __user *p, size_t s,
+			      loff_t *o)
+{
+	struct virt_dev *dev = virt_dev_get(f);
+	struct virt_irq_dev *vdev = dev->driver_ctx;
+	int r = dev->hypervisor->set_irq(dev->hypervisor, vdev->irq, 0, 1);
+	dev->hypervisor->set_irq(dev->hypervisor, vdev->irq, 0, 0);
+	return r < 0 ? r : 0;
+}
+
+int virt_irq_probe(struct virt_driver *driver, struct virt_dev *dev,
+		    const void *id, int id_len)
+{
+	struct virt_irq_dev *vdev;
+	const struct virt_irq_id *irq_id;
+	if (!dev->hypervisor->set_irq)
+		return -ENODEV;
+	if (id_len != sizeof id)
+		return -ENODEV;
+	irq_id = id;
+	if (memcmp(irq_id->name, "irq", sizeof irq_id->name))
+		return -ENODEV;
+	
+	vdev = kmalloc(sizeof *vdev, GFP_KERNEL);
+	if (!vdev)
+		return -ENOMEM;
+	vdev->irq = irq_id->irq;
+	dev->driver_ctx = vdev;
+	return 0;
+}
+
+void virt_irq_remove(struct virt_driver *driver, struct virt_dev *dev)
+{
+	kfree(dev->driver_ctx);
+}
+
+static struct file_operations virt_irq_fops = {
+	.owner	 = THIS_MODULE,
+	.write   = virt_irq_write,
+};
+
+static struct virt_driver virt_irq_driver = {
+	.name = "virt_irq",
+	.device_probe = virt_irq_probe,
+	.device_remove = virt_irq_remove,
+};
+
+static int __init virt_irq_init(void)
+{
+	virt_driver_register(&virt_irq_driver, &virt_irq_fops);
+	return 0;
+}
+
+static void __exit virt_irq_cleanup(void)
+{
+	virt_driver_unregister(&virt_irq_driver);
+}
+
+module_init(virt_irq_init);
+module_exit(virt_irq_cleanup);
diff --git a/include/linux/virt_irq.h b/include/linux/virt_irq.h
new file mode 100644
index 0000000..6b3421d
--- /dev/null
+++ b/include/linux/virt_irq.h
@@ -0,0 +1,19 @@
+#ifndef LINUX_VIRT_IRQ_H
+#define LINUX_VIRT_IRQ_H
+
+#include <linux/types.h>
+
+/* Format for IRQ device id */
+
+struct virt_irq_id {
+	__u8 name[4]; /* Must be "irq\0" */
+	__u32 irq;
+};
+
+static inline void virt_irq_id_init(struct virt_irq_id *id, int irq)
+{
+	memcpy(id->name, "irq", sizeof id->name);
+	id->irq = irq;
+}
+
+#endif
-- 
1.6.3.1.175.g3be7e0