[PATCH] cpu hotplug driver: kernel module

[PATCH] cpu hotplug driver: kernel module

[Glauber de Oliveira Costa]

Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
---
 drivers/virtio/Kconfig      |    6 +
 drivers/virtio/Makefile     |    1 +
 drivers/virtio/virtio_cpu.c |  226 +++++++++++++++++++++++++++++++++++++++++++
 drivers/virtio/virtio_pci.c |    1 +
 kernel/cpu.c                |    2 +
 5 files changed, 236 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virtio/virtio_cpu.c

diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 73a414c..6665d4d 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -23,3 +23,9 @@ config VIRTIO_PCI
 
 	  If unsure, say M.
 
+config VIRTIO_CPU
+	tristate "virtio-cpu"
+	depends on VIRTIO_PCI && EXPERIMENTAL
+	---help---
+	  I'll obviously put a better description later
+
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index cc84999..057ebcc 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_VIRTIO) += virtio.o
 obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_CPU) += virtio_cpu.o
diff --git a/drivers/virtio/virtio_cpu.c b/drivers/virtio/virtio_cpu.c
new file mode 100644
index 0000000..8dc0c7d
--- /dev/null
+++ b/drivers/virtio/virtio_cpu.c
@@ -0,0 +1,226 @@
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+
+#include <linux/cpu.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/freezer.h>
+
+#define VIRTIO_ID_CPU 4
+
+#define CMD_CPU_SET 1
+
+struct virtio_hotplug_hdr {
+	u8 cmd;
+	u8 status;
+};
+
+struct hotplug_buf {
+	struct virtio_hotplug_hdr hdr;
+	u32 arg;
+};
+
+struct virtio_hotplug {
+	struct virtio_device *dev;
+	struct virtqueue *cmd_receive;
+	struct task_struct *hotplug_thread;
+	wait_queue_head_t hotplug_wait;
+	u32 hotplug_target;
+	int thread_started;
+};
+
+
+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_CPU, VIRTIO_DEV_ANY_ID},
+	{ 0 },
+};
+
+void *__alloc_virtio_buf(int size)
+{
+	return kzalloc(size, GFP_KERNEL);
+}
+
+#define alloc_hotplug_buf() (__alloc_virtio_buf(sizeof(struct hotplug_buf)))
+
+static int __send_hotplug_buf(u8 cmd, struct hotplug_buf *buf,
+			      struct virtqueue *vq)
+{
+	struct scatterlist sg[2];
+	int err = 0;
+	
+	sg_init_table(sg, 2);
+
+	sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+	sg_set_buf(&sg[1], &buf->arg, sizeof(buf->arg));
+	
+	err = vq->vq_ops->add_buf(vq, sg, 0, 2, buf);
+	if (err)
+		return err;
+
+	vq->vq_ops->kick(vq);
+
+	return 0;
+} 
+
+/* Prepare the virtio driver to receive a command. */
+static int prepare_to_receive(struct virtio_hotplug *v)
+{
+	void *buf = alloc_hotplug_buf();
+
+	if (!buf)
+		return -ENOMEM;
+
+	return __send_hotplug_buf(0, buf, v->cmd_receive);
+}
+
+static void hotplug_add_cpus(struct virtio_hotplug *v)
+{
+	struct sys_device *dev;
+	struct virtqueue *vq = v->cmd_receive;
+	struct hotplug_buf *buf;
+	int i, len;
+	int err;
+
+	while ((buf = vq->vq_ops->get_buf(vq, &len)) != NULL) {
+		switch (buf->hdr.cmd) {
+		case CMD_CPU_SET:
+			v->hotplug_target = buf->arg;
+
+			if (v->hotplug_target > num_present_cpus())
+				v->hotplug_target = num_present_cpus();
+			break;
+		default:
+			printk("%s: Unrecognized command %d\n",
+				__func__, buf->hdr.cmd);
+			break;	
+		}
+	}
+	kfree(buf);
+
+
+	while ((i = num_online_cpus()) < v->hotplug_target) {
+		err = cpu_up(i);
+		dev = get_cpu_sysdev(i);
+		if (!err) {
+			if (dev)
+				kobject_uevent(&dev->kobj, KOBJ_ONLINE);
+		}
+	}
+
+	while ((i = num_online_cpus()) > v->hotplug_target) {
+		i--;
+		dev = get_cpu_sysdev(i);
+		err = cpu_down(i);
+		if (!err) {
+			if (dev)
+				kobject_uevent(&dev->kobj, KOBJ_ONLINE);
+		}
+	}
+}
+
+static int virtio_do_hotplug(void *p)
+{
+	struct virtio_hotplug *v = p;
+	DEFINE_WAIT(wait);
+	set_freezable();
+
+	for (;;) {
+		prepare_to_receive(v);
+		prepare_to_wait(&v->hotplug_wait, &wait, TASK_UNINTERRUPTIBLE);
+		v->thread_started = 1;
+		schedule();
+		finish_wait(&v->hotplug_wait, &wait);
+
+		try_to_freeze();
+
+		if (kthread_should_stop())
+			break;
+
+		hotplug_add_cpus(v);
+		v->hotplug_target = 0;
+	}
+	return 0;
+}
+
+/* We need a separate thread because cpu_up() and cpu_down() cannot
+ * be called in interrupt context */
+static bool hotplug_get_cmd(struct virtqueue *vq)
+{
+	struct virtio_hotplug *v = vq->vdev->priv;
+
+	wake_up(&v->hotplug_wait);
+	return true;
+
+}
+
+static int virtio_cpu_probe(struct virtio_device *vdev)
+{
+	int err;
+	struct virtio_hotplug *v;
+
+	v = kzalloc(sizeof(*v), GFP_KERNEL);
+	if (!v)
+		return -ENOMEM;
+
+	
+	if (IS_ERR(v->cmd_receive)) {
+		printk("%s: Couldn't find cmd_receive queue\n", __func__);
+		return PTR_ERR(v->cmd_receive);
+	}
+	
+	vdev->priv = v;
+
+	init_waitqueue_head(&v->hotplug_wait);
+
+	v->thread_started = 0;
+	v->hotplug_thread = kthread_run(virtio_do_hotplug, v, 
+				     "virtio_hotplugd\n");
+
+	if (IS_ERR(v->hotplug_thread)) {
+		err = PTR_ERR(v->hotplug_thread);
+		goto out;
+	}
+
+	v->cmd_receive = vdev->config->find_vq(vdev, 0, hotplug_get_cmd);
+
+	printk("Registered virtio hotplug driver\n");
+	return 0;
+out:
+	vdev->config->del_vq(v->cmd_receive);
+	return err;
+}
+
+static void virtio_cpu_remove(struct virtio_device *vdev)
+{
+	struct virtio_hotplug *v = vdev->priv;
+	kthread_stop(v->hotplug_thread);
+	vdev->config->del_vq(v->cmd_receive);
+	kfree(v);
+}
+
+
+static struct virtio_driver virtio_cpu = {
+	.driver.name = KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table = id_table,
+	.probe = virtio_cpu_probe,
+	.remove = __devexit_p(virtio_cpu_remove),
+};
+
+static int __init virtio_cpu_init(void)
+{
+	return register_virtio_driver(&virtio_cpu);
+}
+
+static void __exit virtio_cpu_exit(void)
+{
+	unregister_virtio_driver(&virtio_cpu);
+}
+
+module_init(virtio_cpu_init);
+module_exit(virtio_cpu_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Glauber de Oliveira Costa");
+MODULE_DESCRIPTION("virtio driver for cpu hotplugging");
+MODULE_VERSION("1");
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
index e1fa68d..8d5a719 100644
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_id_table[] = {
 	{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
 	{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+	{ 0x2523, 0x2325, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* cpu hotplug driver */
 	{ 0 },
 };
 
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 6b3a0c1..c432c28 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -219,6 +219,7 @@ int cpu_down(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
 /* Requires cpu_add_remove_lock to be held */
@@ -284,6 +285,7 @@ int __cpuinit cpu_up(unsigned int cpu)
 	mutex_unlock(&cpu_add_remove_lock);
 	return err;
 }
+EXPORT_SYMBOL_GPL(cpu_up);
 
 #ifdef CONFIG_PM_SLEEP_SMP
 static cpumask_t frozen_cpus;
-- 
1.5.0.6

[PATCH] cpu hotplug driver: userspace back end

[Glauber de Oliveira Costa]

Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
---
 qemu/Makefile.target     |    2 +-
 qemu/hw/pc.c             |    4 +-
 qemu/hw/pc.h             |    3 +
 qemu/hw/virtio-hotplug.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++
 qemu/monitor.c           |   11 +++++
 qemu/qemu-kvm.h          |    2 +
 6 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 qemu/hw/virtio-hotplug.c

diff --git a/qemu/Makefile.target b/qemu/Makefile.target
index bb7be0f..4d5679a 100644
--- a/qemu/Makefile.target
+++ b/qemu/Makefile.target
@@ -464,7 +464,7 @@ VL_OBJS += rtl8139.o
 VL_OBJS+= hypercall.o
 
 # virtio devices
-VL_OBJS += virtio.o virtio-net.o virtio-blk.o
+VL_OBJS += virtio.o virtio-net.o virtio-blk.o virtio-hotplug.o
 
 ifeq ($(TARGET_BASE_ARCH), i386)
 # Hardware support
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index 3972ab4..c9d8f89 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -1029,7 +1029,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,
         }
     }
 
-#define USE_HYPERCALL
+    virtio_hotplug_init(pci_bus);
+
+#undef USE_HYPERCALL
 #ifdef USE_HYPERCALL
     pci_hypercall_init(pci_bus);
 #endif
diff --git a/qemu/hw/pc.h b/qemu/hw/pc.h
index 7d1832f..3e2cfb4 100644
--- a/qemu/hw/pc.h
+++ b/qemu/hw/pc.h
@@ -151,6 +151,9 @@ void virtio_net_poll(void);
 void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
 		      BlockDriverState *bs);
 
+/* virtio-hotplug.c */
+void *virtio_hotplug_init(PCIBus *bus);
+
 /* extboot.c */
 
 void extboot_init(BlockDriverState *bs, int cmd);
diff --git a/qemu/hw/virtio-hotplug.c b/qemu/hw/virtio-hotplug.c
new file mode 100644
index 0000000..e51f26f
--- /dev/null
+++ b/qemu/hw/virtio-hotplug.c
@@ -0,0 +1,111 @@
+#include "virtio.h"
+#include "net.h"
+#include "pc.h"
+#include "sysemu.h"
+
+typedef struct VirtIOHotplug {
+    VirtIODevice vdev;
+    VirtQueue *cmd_vq;
+    int buffer_ready;
+    struct VirtIOHotplug *next;
+    int job_status;
+} VirtIOHotplug;
+
+typedef struct VirtioHotplugHdr {
+    uint8_t cmd;
+    uint8_t status;
+} VirtioHotplugHdr;
+
+VirtIOHotplug *virtio_hotplug;
+uint32_t attempt_buffer;
+
+#define VIRTIO_ID_HOTPLUG 4 /* arbitrary */
+
+#define CMD_CPU_SET 1
+
+static VirtIOHotplug *to_virtio_hotplug(VirtIODevice *vdev)
+{
+    return (VirtIOHotplug *)vdev;
+}
+
+static void virtio_hotplug_update_config(VirtIODevice *vdev, uint8_t *config)
+{
+    /* nothing to do */
+}
+
+static uint32_t virtio_hotplug_get_features(VirtIODevice *vdev)
+{
+    /* no features yet */
+    return 0;
+}
+
+static void virtio_hotplug_send(VirtIODevice *vdev, VirtQueue *vq, uint8_t cmd,
+    			const uint32_t arg)
+{
+    VirtQueueElement elem;
+    VirtioHotplugHdr *hdr;
+    uint32_t *data;
+
+    if (virtqueue_pop(vq, &elem) == 0) {
+        fprintf(stderr, "pop failure\n"); 
+        return;
+    }
+    
+    hdr = (void *)elem.in_sg[0].iov_base;
+    hdr->cmd = cmd;
+    
+    data = (int *)elem.in_sg[1].iov_base;
+    *data = arg;
+     
+    virtqueue_push(vq, &elem, sizeof(*hdr) + elem.in_sg[1].iov_len);
+    virtio_notify(vdev, vq);
+}
+
+int hotplug_send_cmd(int value)
+{
+   
+    if (!virtio_hotplug->buffer_ready) {
+        attempt_buffer = value;
+        return;
+    }
+
+    virtio_hotplug_send(&virtio_hotplug->vdev, virtio_hotplug->cmd_vq, 
+                        CMD_CPU_SET, value);
+
+    return 0;
+}
+
+/* TX */
+static void virtio_hotplug_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOHotplug *n = to_virtio_hotplug(vdev);
+
+    n->buffer_ready = 1;
+
+    if (attempt_buffer) {
+        uint32_t value = attempt_buffer;
+        attempt_buffer = 0;
+        hotplug_send_cmd(value);
+    }
+
+}
+
+void *virtio_hotplug_init(PCIBus *bus)
+{
+    VirtIOHotplug *n;
+
+    n = (VirtIOHotplug *)virtio_init_pci(bus, "virtio-hotplug", 0x2523, 0x2325,
+    			     0, VIRTIO_ID_HOTPLUG,
+    			     0xff, 0x80, 0x00,
+    			     6, sizeof(VirtIOHotplug));
+
+    n->vdev.update_config = virtio_hotplug_update_config;
+    n->vdev.get_features = virtio_hotplug_get_features;
+    n->cmd_vq = virtio_add_queue(&n->vdev, 128, virtio_hotplug_handle_cmd);
+
+    n->buffer_ready = 0;
+
+    virtio_hotplug = n;
+        
+    return &n->vdev;
+}
diff --git a/qemu/monitor.c b/qemu/monitor.c
index e03c473..f0c55c6 100644
--- a/qemu/monitor.c
+++ b/qemu/monitor.c
@@ -346,6 +346,16 @@ static void do_cpu_set(int index)
         term_printf("Invalid CPU index\n");
 }
 
+static void do_cpu_set_nr(int value)
+{
+    if ((value < 1)) {
+       term_printf("value out of range\n");
+       return;
+    }
+
+    hotplug_send_cmd(value);
+}
+
 static void do_info_jit(void)
 {
     dump_exec_info(NULL, monitor_fprintf);
@@ -1339,6 +1349,7 @@ static term_cmd_t term_cmds[] = {
       "", "cancel the current VM migration" },
     { "migrate_set_speed", "s", do_migrate_set_speed,
       "value", "set maximum speed (in bytes) for migrations" },
+    { "cpu_set", "i", do_cpu_set_nr, "value", "number of cpus in the guest" },
     { NULL, NULL, },
 };
 
diff --git a/qemu/qemu-kvm.h b/qemu/qemu-kvm.h
index e4aeb3a..6bf234d 100644
--- a/qemu/qemu-kvm.h
+++ b/qemu/qemu-kvm.h
@@ -48,6 +48,8 @@ void kvm_tpr_access_report(CPUState *env, uint64_t rip, int is_write);
 int handle_tpr_access(void *opaque, int vcpu,
 			     uint64_t rip, int is_write);
 
+int hotplug_send_smd(int cpus);
+
 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 #define BITMAP_SIZE(m) (ALIGN(((m)>>TARGET_PAGE_BITS), HOST_LONG_BITS) / 8)
 
-- 
1.5.0.6

Re: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Avi Kivity]

Glauber de Oliveira Costa wrote:
> I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio
> The first patch is the kernel module, while the second, the userspace pci device.
>
> The host boots with the maximum cpus it should ever use, through the -smp parameter.
> Due to real machine constraints (which qemu copies), i386 does not allow for any addition
> of cpus after boot, so this is the most general way.
>
> I do however, include an "attempt_buffer" in the userspace part. It's purpose is to
> allow tools like virt-manager to set a max_cpus (-smp), and a desired number of cpus
> in their configuration files. (and AFAICT, there's is no easy way for them to tell when the
> backend driver is up and running)
>
> Other than that, it should be pretty much straightforward.
>
> Looking forward for your comments
>
>   

I would much prefer to see cpu hotplug implemented via acpi.  Such an 
implementation would work on older kernels without change, and will also 
work with other operating systems.  It isn't a high-speed interface so 
virtio doesn't buy us anything.

Linux appears to support it (CONFIG_ACPI_HOTPLUG_CPU) so all that's 
needed is the host side support (likely qemu/bios only).  Of course 
hacking on acpi is fun, if you're the kind of person than enjoys dental 
surgery.

-- 
error compiling committee.c: too many arguments to function

Re: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Christian Borntraeger]

Am Mittwoch, 9. Januar 2008 schrieb Glauber de Oliveira Costa:
> I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio
> The first patch is the kernel module, while the second, the userspace pci 
device.

I personally prefer to use non paravirtualized cpu hotplug and implement the 
existing interfaces instead in the hypervisor. (sigp/sclp for s390 and 
maybe acpi for x86 etc.). This is not a performance critical operation.


Christian

Re: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Glauber de Oliveira Costa]

Avi Kivity wrote:
> Glauber de Oliveira Costa wrote:
>> I'm sending a first draft of my proposed cpu hotplug driver for 
>> kvm/virtio
>> The first patch is the kernel module, while the second, the userspace 
>> pci device.
>>
>> The host boots with the maximum cpus it should ever use, through the 
>> -smp parameter.
>> Due to real machine constraints (which qemu copies), i386 does not 
>> allow for any addition
>> of cpus after boot, so this is the most general way.
>>
>> I do however, include an "attempt_buffer" in the userspace part. It's 
>> purpose is to
>> allow tools like virt-manager to set a max_cpus (-smp), and a desired 
>> number of cpus
>> in their configuration files. (and AFAICT, there's is no easy way for 
>> them to tell when the
>> backend driver is up and running)
>>
>> Other than that, it should be pretty much straightforward.
>>
>> Looking forward for your comments
>>
>>   
> 
> I would much prefer to see cpu hotplug implemented via acpi.  Such an 
> implementation would work on older kernels without change, and will also 
> work with other operating systems.  It isn't a high-speed interface so 
> virtio doesn't buy us anything.
> 
> Linux appears to support it (CONFIG_ACPI_HOTPLUG_CPU) so all that's 
> needed is the host side support (likely qemu/bios only).  Of course 
> hacking on acpi is fun, if you're the kind of person than enjoys dental 
> surgery.
> 
Being acpi so that fun, how can it be that virtio don't by us _anything_ ?

Re: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Glauber de Oliveira Costa]

Christian Borntraeger wrote:
> Am Mittwoch, 9. Januar 2008 schrieb Glauber de Oliveira Costa:
>> I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio
>> The first patch is the kernel module, while the second, the userspace pci 
> device.
> 
> I personally prefer to use non paravirtualized cpu hotplug and implement the 
> existing interfaces instead in the hypervisor. (sigp/sclp for s390 and 
> maybe acpi for x86 etc.). This is not a performance critical operation.
> 
> 
> Christian
It's barely "paravirtualized". It's a device driver talking to a 
hardware from the guest POV, not much different than acpi itself is, but 
sanner. The only difference is that acpi support "already exists", but 
virtio is a lot easier, and it wouldn't take long to write a driver like 
that for windows or anything else (not that I am volunteering ;-)).

Also, in the future the interface can be extended in ways acpi can't 
handle. (and just because nobody will change the hardware to embrace new 
things that fast)

That said, if acpi is really the preference here, and this patches have 
chance, no problem. But it will take me a little more time to implement 
them ;-)

Re: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Avi Kivity]

Glauber de Oliveira Costa wrote:
>
> That said, if acpi is really the preference here, and this patches 
> have chance, no problem. But it will take me a little more time to 
> implement them ;-)

The power button support that was recently added at least proves that 
the host->guest notification path works, so that's one tooth that does 
not need pulling out.  The real trouble with acpi is understanding the 
spec which is very badly written.

_AVI

-- 
error compiling committee.c: too many arguments to function

RE: [kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver

[Dong, Eddie]

Avi Kivity wrote:
> Glauber de Oliveira Costa wrote:
>> 
>> That said, if acpi is really the preference here, and this patches
>> have chance, no problem. But it will take me a little more time to
>> implement them ;-)
> 
> The power button support that was recently added at least proves that
> the host->guest notification path works, so that's one tooth that
> does not need pulling out.  The real trouble with acpi is
> understanding the spec which is very badly written.   
> 
> _AVI

Edwin implemented device hotplug support for Xen/Qemu, probably we
can reuse the effort. 

Edwin:
	Can u share the device hotplug patch with Glauber?
thx,eddie