[PATCH 1/3] Extend virtio config routines to support ballooning

[PATCH 1/3] Extend virtio config routines to support ballooning

[Anthony Liguori]

This patch updates the virtio device methods to provide hooks for setting and
getting the config.  This is needed for ballooning since the balloon driver
uses config changes to signal changes in the balloon amount.  We also add a
method to signal that we have changed the config.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

diff --git a/qemu/hw/virtio-blk.c b/qemu/hw/virtio-blk.c
index 88dc086..da9e795 100644
--- a/qemu/hw/virtio-blk.c
+++ b/qemu/hw/virtio-blk.c
@@ -286,7 +286,7 @@ void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
     if (!s)
 	return NULL;
 
-    s->vdev.update_config = virtio_blk_update_config;
+    s->vdev.get_config = virtio_blk_update_config;
     s->vdev.get_features = virtio_blk_get_features;
     s->vdev.reset = virtio_blk_reset;
     s->bs = bs;
diff --git a/qemu/hw/virtio-net.c b/qemu/hw/virtio-net.c
index 409960f..bc2ede6 100644
--- a/qemu/hw/virtio-net.c
+++ b/qemu/hw/virtio-net.c
@@ -332,7 +332,7 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     if (!n)
 	return NULL;
 
-    n->vdev.update_config = virtio_net_update_config;
+    n->vdev.get_config = virtio_net_update_config;
     n->vdev.get_features = virtio_net_get_features;
     n->vdev.set_features = virtio_net_set_features;
     n->rx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_rx);
diff --git a/qemu/hw/virtio.c b/qemu/hw/virtio.c
index e035e4e..e675f43 100644
--- a/qemu/hw/virtio.c
+++ b/qemu/hw/virtio.c
@@ -313,6 +313,8 @@ static uint32_t virtio_config_readb(void *opaque, uint32_t addr)
     VirtIODevice *vdev = opaque;
     uint8_t val;
 
+    vdev->get_config(vdev, vdev->config);
+
     addr -= vdev->addr + VIRTIO_PCI_CONFIG;
     if (addr > (vdev->config_len - sizeof(val)))
 	return (uint32_t)-1;
@@ -326,6 +328,8 @@ static uint32_t virtio_config_readw(void *opaque, uint32_t addr)
     VirtIODevice *vdev = opaque;
     uint16_t val;
 
+    vdev->get_config(vdev, vdev->config);
+
     addr -= vdev->addr + VIRTIO_PCI_CONFIG;
     if (addr > (vdev->config_len - sizeof(val)))
 	return (uint32_t)-1;
@@ -339,6 +343,8 @@ static uint32_t virtio_config_readl(void *opaque, uint32_t addr)
     VirtIODevice *vdev = opaque;
     uint32_t val;
 
+    vdev->get_config(vdev, vdev->config);
+
     addr -= vdev->addr + VIRTIO_PCI_CONFIG;
     if (addr > (vdev->config_len - sizeof(val)))
 	return (uint32_t)-1;
@@ -357,6 +363,9 @@ static void virtio_config_writeb(void *opaque, uint32_t addr, uint32_t data)
 	return;
 
     memcpy(vdev->config + addr, &val, sizeof(val));
+
+    if (vdev->set_config)
+        vdev->set_config(vdev, vdev->config);
 }
 
 static void virtio_config_writew(void *opaque, uint32_t addr, uint32_t data)
@@ -369,6 +378,9 @@ static void virtio_config_writew(void *opaque, uint32_t addr, uint32_t data)
 	return;
 
     memcpy(vdev->config + addr, &val, sizeof(val));
+
+    if (vdev->set_config)
+        vdev->set_config(vdev, vdev->config);
 }
 
 static void virtio_config_writel(void *opaque, uint32_t addr, uint32_t data)
@@ -381,6 +393,9 @@ static void virtio_config_writel(void *opaque, uint32_t addr, uint32_t data)
 	return;
 
     memcpy(vdev->config + addr, &val, sizeof(val));
+
+    if (vdev->set_config)
+        vdev->set_config(vdev, vdev->config);
 }
 
 static void virtio_map(PCIDevice *pci_dev, int region_num,
@@ -409,7 +424,7 @@ static void virtio_map(PCIDevice *pci_dev, int region_num,
 	register_ioport_read(addr + 20, vdev->config_len, 4,
 			     virtio_config_readl, vdev);
 
-	vdev->update_config(vdev, vdev->config);
+	vdev->get_config(vdev, vdev->config);
     }
 }
 
@@ -439,7 +454,13 @@ void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
 	(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 	return;
 
-    vdev->isr = 1;
+    vdev->isr |= 0x01;
+    virtio_update_irq(vdev);
+}
+
+void virtio_notify_config(VirtIODevice *vdev)
+{
+    vdev->isr |= 0x03;
     virtio_update_irq(vdev);
 }
 
diff --git a/qemu/hw/virtio.h b/qemu/hw/virtio.h
index 1adaed3..0dcedbf 100644
--- a/qemu/hw/virtio.h
+++ b/qemu/hw/virtio.h
@@ -120,7 +120,8 @@ struct VirtIODevice
     void *config;
     uint32_t (*get_features)(VirtIODevice *vdev);
     void (*set_features)(VirtIODevice *vdev, uint32_t val);
-    void (*update_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*get_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
     void (*reset)(VirtIODevice *vdev);
     VirtQueue vq[VIRTIO_PCI_QUEUE_MAX];
 };
@@ -147,4 +148,6 @@ void virtio_save(VirtIODevice *vdev, QEMUFile *f);
 
 void virtio_load(VirtIODevice *vdev, QEMUFile *f);
 
+void virtio_notify_config(VirtIODevice *vdev);
+
 #endif

[PATCH 2/3] add ballooning infrastructure to QEMU

[Anthony Liguori]

This patch adds a ballooning infrastructure to QEMU.  This includes a pair of
monitor commands, balloon and info balloon, to balloon a guest and to query
the guest's balloon status.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

diff --git a/qemu/balloon.h b/qemu/balloon.h
new file mode 100644
index 0000000..60b4a5d
--- /dev/null
+++ b/qemu/balloon.h
@@ -0,0 +1,27 @@
+/*
+ * Balloon
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef _QEMU_BALLOON_H
+#define _QEMU_BALLOON_H
+
+#include "cpu-defs.h"
+
+typedef ram_addr_t (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
+
+void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque);
+
+void qemu_balloon(ram_addr_t target);
+
+ram_addr_t qemu_balloon_status(void);
+
+#endif
diff --git a/qemu/monitor.c b/qemu/monitor.c
index 4acf346..2240d8e 100644
--- a/qemu/monitor.c
+++ b/qemu/monitor.c
@@ -35,6 +35,7 @@
 #include "audio/audio.h"
 #include "disas.h"
 #include "migration.h"
+#include "balloon.h"
 #include <dirent.h>
 #include "qemu-timer.h"
 
@@ -1399,6 +1400,23 @@ static void do_inject_nmi(int cpu_index)
 }
 #endif
 
+static void do_balloon(int value)
+{
+    ram_addr_t target = value;
+    qemu_balloon(target << 20);
+}
+
+static void do_info_balloon(void)
+{
+    ram_addr_t actual;
+
+    actual = qemu_balloon_status();
+    if (actual == 0)
+	term_printf("Ballooning not activated in VM\n");
+    else
+	term_printf("balloon: actual=%d\n", (int)(actual >> 20));
+}
+
 static term_cmd_t term_cmds[] = {
     { "help|?", "s?", do_help,
       "[cmd]", "show the help" },
@@ -1494,6 +1512,8 @@ static term_cmd_t term_cmds[] = {
     { "pci_add", "iss", device_hot_add, "bus nic|storage [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]...", "hot-add PCI device" },
     { "pci_del", "ii", device_hot_remove, "bus slot-number", "hot remove PCI device" },
 #endif
+    { "balloon", "i", do_balloon,
+      "target", "request VM to change it's memory allocation (in MB)" },
     { NULL, NULL, },
 };
 
@@ -1558,6 +1578,8 @@ static term_cmd_t info_cmds[] = {
 #endif
     { "migration", "", do_info_migration,
       "", "show migration information" },
+    { "balloon", "", do_info_balloon,
+      "", "show balloon information" },
     { NULL, NULL, },
 };
 
diff --git a/qemu/vl.c b/qemu/vl.c
index 2dc1311..ec89921 100644
--- a/qemu/vl.c
+++ b/qemu/vl.c
@@ -38,6 +38,7 @@
 #include "block.h"
 #include "audio/audio.h"
 #include "migration.h"
+#include "balloon.h"
 #include "qemu-kvm.h"
 
 #include <unistd.h>
@@ -530,6 +531,31 @@ void hw_error(const char *fmt, ...)
     va_end(ap);
     abort();
 }
+ 
+/***************/
+/* ballooning */
+
+static QEMUBalloonEvent *qemu_balloon_event;
+void *qemu_balloon_event_opaque;
+
+void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque)
+{
+    qemu_balloon_event = func;
+    qemu_balloon_event_opaque = opaque;
+}
+
+void qemu_balloon(ram_addr_t target)
+{
+    if (qemu_balloon_event)
+	qemu_balloon_event(qemu_balloon_event_opaque, target);
+}
+
+ram_addr_t qemu_balloon_status(void)
+{
+    if (qemu_balloon_event)
+	return qemu_balloon_event(qemu_balloon_event_opaque, 0);
+    return 0;
+}
 
 /***********************************************************/
 /* keyboard/mouse */

[PATCH 3/3] Add virtio balloon driver (v2)

[Anthony Liguori]

This patch adds the virtio balloon driver.  It doesn't actually reduce the RSS
size when using KVM (although it does when using -no-kvm).  This is because
even with MMU notifiers, we're holding a reference to each page in the shadow
page table.  Once we switch to not taking a reference for pages in the shadow
page table, the balloon driver should work as expected.

Since v1, fixed the madvise bug spotted by Andrea and updated to use 
kvm_has_sync_mmu.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

diff --git a/qemu/Makefile.target b/qemu/Makefile.target
index 2332fe3..89814fd 100644
--- a/qemu/Makefile.target
+++ b/qemu/Makefile.target
@@ -601,7 +601,7 @@ OBJS += rtl8139.o
 OBJS += e1000.o
 
 # virtio devices
-OBJS += virtio.o virtio-net.o virtio-blk.o
+OBJS += virtio.o virtio-net.o virtio-blk.o virtio-balloon.o
 
 OBJS += device-hotplug.o
 
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index 10d7269..fdb0a9c 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -1165,6 +1165,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,
 
 	extboot_init(info->bdrv, 1);
     }
+
+    if (pci_enabled)
+        virtio_balloon_init(pci_bus);
 }
 
 static void pc_init_pci(ram_addr_t ram_size, int vga_ram_size,
diff --git a/qemu/hw/pc.h b/qemu/hw/pc.h
index aa74d53..a5cc553 100644
--- a/qemu/hw/pc.h
+++ b/qemu/hw/pc.h
@@ -162,4 +162,7 @@ void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
 
 void extboot_init(BlockDriverState *bs, int cmd);
 
+/* virtio-balloon.h */
+void *virtio_balloon_init(PCIBus *bus);
+
 #endif
diff --git a/qemu/hw/virtio-balloon.c b/qemu/hw/virtio-balloon.c
new file mode 100644
index 0000000..64a2e0f
--- /dev/null
+++ b/qemu/hw/virtio-balloon.c
@@ -0,0 +1,194 @@
+/*
+ * Virtio Block Device
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "virtio.h"
+#include "pc.h"
+#include "sysemu.h"
+#include "cpu.h"
+#include "balloon.h"
+#include "virtio-balloon.h"
+#include "qemu-kvm.h"
+
+#if defined(__linux__)
+#include <sys/mman.h>
+#endif
+
+typedef struct VirtIOBalloon
+{
+    VirtIODevice vdev;
+    VirtQueue *ivq, *dvq;
+    uint32_t num_pages;
+    uint32_t actual;
+} VirtIOBalloon;
+
+static VirtIOBalloon *to_virtio_balloon(VirtIODevice *vdev)
+{
+    return (VirtIOBalloon *)vdev;
+}
+
+static void balloon_page(void *addr, int deflate)
+{
+#if defined(__linux__)
+    if (!kvm_enabled() || qemu_kvm_has_sync_mmu())
+        madvise(addr, TARGET_PAGE_SIZE,
+                deflate ? MADV_WILLNEED : MADV_DONTNEED);
+#endif
+}
+
+/* FIXME: once we do a virtio refactoring, this will get subsumed into common
+ * code */
+static size_t memcpy_from_iovector(void *data, size_t offset, size_t size,
+                                   struct iovec *iov, int iovlen)
+{
+    int i;
+    uint8_t *ptr = data;
+    size_t iov_off = 0;
+    size_t data_off = 0;
+
+    for (i = 0; i < iovlen && size; i++) {
+        if (offset < (iov_off + iov[i].iov_len)) {
+            size_t len = MIN((iov_off + iov[i].iov_len) - offset , size);
+
+            memcpy(ptr + data_off, iov[i].iov_base + (offset - iov_off), len);
+
+            data_off += len;
+            offset += len;
+            size -= len;
+        }
+
+        iov_off += iov[i].iov_len;
+    }
+
+    return data_off;
+}
+
+static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOBalloon *s = to_virtio_balloon(vdev);
+    VirtQueueElement elem;
+
+    while (virtqueue_pop(vq, &elem)) {
+	size_t offset = 0;
+	uint32_t pfn;
+
+	while (memcpy_from_iovector(&pfn, offset, 4, 
+                                    elem.out_sg, elem.out_num) == 4) {
+	    ram_addr_t pa;
+	    ram_addr_t addr;
+
+	    pa = (ram_addr_t)ldl_p(&pfn) << TARGET_PAGE_BITS;
+	    offset += 4;
+
+	    addr = cpu_get_physical_page_desc(pa);
+	    if ((addr & ~TARGET_PAGE_MASK) != IO_MEM_RAM)
+		continue;
+
+	    balloon_page(phys_ram_base + addr, !!(vq == s->dvq));
+	}
+
+	virtqueue_push(vq, &elem, offset);
+	virtio_notify(vdev, vq);
+    }
+}
+
+static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
+{
+    VirtIOBalloon *dev = to_virtio_balloon(vdev);
+    struct virtio_balloon_config config;
+
+    config.num_pages = cpu_to_le32(dev->num_pages);
+    config.actual = cpu_to_le32(dev->actual);
+
+    memcpy(config_data, &config, 8);
+}
+
+static void virtio_balloon_set_config(VirtIODevice *vdev,
+				      const uint8_t *config_data)
+{
+    VirtIOBalloon *dev = to_virtio_balloon(vdev);
+    struct virtio_balloon_config config;
+    memcpy(&config, config_data, 8);
+    dev->actual = config.actual;
+}
+
+static uint32_t virtio_balloon_get_features(VirtIODevice *vdev)
+{
+    return 0;
+}
+
+static ram_addr_t virtio_balloon_to_target(void *opaque, ram_addr_t target)
+{
+    VirtIOBalloon *dev = opaque;
+
+    if (target > ram_size)
+	target = ram_size;
+
+    if (target) {
+	dev->num_pages = (ram_size - target) >> TARGET_PAGE_BITS;
+	virtio_notify_config(&dev->vdev);
+    }
+
+    return ram_size - (dev->actual << TARGET_PAGE_BITS);
+}
+
+static void virtio_balloon_save(QEMUFile *f, void *opaque)
+{
+    VirtIOBalloon *s = opaque;
+
+    virtio_save(&s->vdev, f);
+
+    qemu_put_be32(f, s->num_pages);
+    qemu_put_be32(f, s->actual);
+}
+
+static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIOBalloon *s = opaque;
+
+    if (version_id != 1)
+        return -EINVAL;
+
+    virtio_load(&s->vdev, f);
+
+    s->num_pages = qemu_get_be32(f);
+    s->actual = qemu_get_be32(f);
+
+    return 0;
+}
+
+void *virtio_balloon_init(PCIBus *bus)
+{
+    VirtIOBalloon *s;
+
+    s = (VirtIOBalloon *)virtio_init_pci(bus, "virtio-balloon",
+					 6900, 0x1002,
+					 0, VIRTIO_ID_BALLOON,
+					 0x05, 0x00, 0x00,
+					 8, sizeof(VirtIOBalloon));
+    if (s == NULL)
+        return NULL;
+
+    s->vdev.get_config = virtio_balloon_get_config;
+    s->vdev.set_config = virtio_balloon_set_config;
+    s->vdev.get_features = virtio_balloon_get_features;
+
+    s->ivq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
+    s->dvq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
+
+    qemu_add_balloon_handler(virtio_balloon_to_target, s);
+
+    register_savevm("virtio-balloon", -1, 1, virtio_balloon_save, virtio_balloon_load, s);
+
+    return &s->vdev;
+}
diff --git a/qemu/hw/virtio-balloon.h b/qemu/hw/virtio-balloon.h
new file mode 100644
index 0000000..27d6985
--- /dev/null
+++ b/qemu/hw/virtio-balloon.h
@@ -0,0 +1,34 @@
+/*
+ * Virtio Support
+ *
+ * Copyright IBM, Corp. 2007-2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Rusty Russell     <rusty@rustcorp.com.au>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef _QEMU_VIRTIO_BALLOON_H
+#define _QEMU_VIRTIO_BALLOON_H
+
+/* from Linux's linux/virtio_balloon.h */
+
+/* The ID for virtio_balloon */
+#define VIRTIO_ID_BALLOON	5
+
+/* The feature bitmap for virtio balloon */
+#define VIRTIO_BALLOON_F_MUST_TELL_HOST	0 /* Tell before reclaiming pages */
+
+struct virtio_balloon_config
+{
+    /* Number of pages host wants Guest to give up. */
+    uint32_t num_pages;
+    /* Number of pages we've actually got in balloon. */
+    uint32_t actual;
+};
+
+#endif
diff --git a/qemu/monitor.c b/qemu/monitor.c
index 18843c4..f65aa18 100644
--- a/qemu/monitor.c
+++ b/qemu/monitor.c
@@ -1420,7 +1420,9 @@ static void do_info_balloon(void)
     ram_addr_t actual;
 
     actual = qemu_balloon_status();
-    if (actual == 0)
+    if (kvm_enabled() && !qemu_kvm_has_sync_mmu())
+        term_printf("Using KVM without synchronous MMU, ballooning disabled\n");
+    else if (actual == 0)
 	term_printf("Ballooning not activated in VM\n");
     else
 	term_printf("balloon: actual=%d\n", (int)(actual >> 20));
diff --git a/qemu/qemu-kvm.h b/qemu/qemu-kvm.h
index 33bef5a..3e40a7d 100644
--- a/qemu/qemu-kvm.h
+++ b/qemu/qemu-kvm.h
@@ -111,10 +111,12 @@ extern kvm_context_t kvm_context;
 #define kvm_enabled() (kvm_allowed)
 #define qemu_kvm_irqchip_in_kernel() kvm_irqchip_in_kernel(kvm_context)
 #define qemu_kvm_pit_in_kernel() kvm_pit_in_kernel(kvm_context)
+#define qemu_kvm_has_sync_mmu() kvm_has_sync_mmu(kvm_context)
 #else
 #define kvm_enabled() (0)
 #define qemu_kvm_irqchip_in_kernel() (0)
 #define qemu_kvm_pit_in_kernel() (0)
+#define qemu_kvm_has_sync_mmu() (0)
 #endif
 
 void kvm_mutex_unlock(void);

Re: [PATCH 3/3] Add virtio balloon driver (v2)

[Anthony Liguori]

Anthony Liguori wrote:
> This patch adds the virtio balloon driver.  It doesn't actually reduce the RSS
> size when using KVM (although it does when using -no-kvm).  This is because
> even with MMU notifiers, we're holding a reference to each page in the shadow
> page table.  Once we switch to not taking a reference for pages in the shadow
> page table, the balloon driver should work as expected.
>   

Too late in the day for me.  This comment is totally bogus.  Avi: can 
you just ignore everything but the first sentence? :-)

Regards,

Anthony Liguori

Re: [PATCH 3/3] Add virtio balloon driver (v2)

[Avi Kivity]

Anthony Liguori wrote:
> Anthony Liguori wrote:
>> This patch adds the virtio balloon driver.  It doesn't actually 
>> reduce the RSS
>> size when using KVM (although it does when using -no-kvm).  This is 
>> because
>> even with MMU notifiers, we're holding a reference to each page in 
>> the shadow
>> page table.  Once we switch to not taking a reference for pages in 
>> the shadow
>> page table, the balloon driver should work as expected.
>>   
>
> Too late in the day for me.  This comment is totally bogus.  Avi: can 
> you just ignore everything but the first sentence? :-)
>

Actually, the last word in the first sentence is incorrect as well.

-- 
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.

Re: [PATCH 1/3] Extend virtio config routines to support ballooning

[Avi Kivity]

Anthony Liguori wrote:
> This patch updates the virtio device methods to provide hooks for setting and
> getting the config.  This is needed for ballooning since the balloon driver
> uses config changes to signal changes in the balloon amount.  We also add a
> method to signal that we have changed the config.
>   

Applied all three, thanks.

-- 
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.