[Qemu-devel] virtio: Add memory statistics reporting to the balloon driver (V7)

[Qemu-devel] virtio: Add memory statistics reporting to the balloon driver (V7)

[Adam Litke]

This version improves support for multiple monitors and has been ported up to
HEAD as of 01/14.

Changes since V6:
 - Integrated with virtio qdev feature bit changes
   (specifically: Use VirtIODevice 'guest_features' to check if memory stats
   is a negotiated feature)
 - Track which monitor initiated the most recent stats request.  Now it does the
   Right Thing(tm) with multiple monitors making parallel requests.

Changes since V5:
 - Asynchronous query-balloon mode for QMP
 - Add timeout to prevent hanging the user monitor in synchronous mode

Changes since V4:
 - Virtio spec updated: http://ozlabs.org/~rusty/virtio-spec/virtio-spec-0.8.2.pdf
 - Guest-side Linux implementation applied by Rusty
 - Start using the QObject infrastructure
 - All endian conversions done in the host
 - Report stats that reference a quantity of memory in bytes

Changes since V3:
 - Increase stat field size to 64 bits
 - Report all sizes in kb (not pages)
 - Drop anon_pages stat

Changes since V2:
 - Use a virtqueue for communication instead of the device config space

Changes since V1:
 - In the monitor, print all stats on one line with less abbreviated names
 - Coding style changes

When using ballooning to manage overcommitted memory on a host, a system for
guests to communicate their memory usage to the host can provide information
that will minimize the impact of ballooning on the guests.  The current method
employs a daemon running in each guest that communicates memory statistics to a
host daemon at a specified time interval.  The host daemon aggregates this
information and inflates and/or deflates balloons according to the level of
host memory pressure.  This approach is effective but overly complex since a
daemon must be installed inside each guest and coordinated to communicate with
the host.  A simpler approach is to collect memory statistics in the virtio
balloon driver and communicate them directly to the hypervisor.

Signed-off-by: Adam Litke <agl@us.ibm.com>
To: Anthony Liguori <aliguori@us.ibm.com>
Cc: Avi Kivity <avi@redhat.com>
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: qemu-devel@nongnu.org

diff --git a/balloon.h b/balloon.h
index 60b4a5d..684ea9e 100644
--- a/balloon.h
+++ b/balloon.h
@@ -16,12 +16,15 @@
 
 #include "cpu-defs.h"
 
-typedef ram_addr_t (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
+/* Timeout for synchronous stats requests (in seconds) */
+#define QEMU_BALLOON_SYNC_TIMEOUT 5
+
+typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target, Monitor *mon);
 
 void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque);
 
-void qemu_balloon(ram_addr_t target);
+int qemu_balloon(ram_addr_t target);
 
-ram_addr_t qemu_balloon_status(void);
+int qemu_balloon_status(Monitor *mon);
 
 #endif
diff --git a/hw/virtio-balloon.c b/hw/virtio-balloon.c
index e17880f..b68bad7 100644
--- a/hw/virtio-balloon.c
+++ b/hw/virtio-balloon.c
@@ -19,6 +19,10 @@
 #include "balloon.h"
 #include "virtio-balloon.h"
 #include "kvm.h"
+#include "monitor.h"
+#include "qlist.h"
+#include "qint.h"
+#include "qstring.h"
 
 #if defined(__linux__)
 #include <sys/mman.h>
@@ -27,9 +31,16 @@
 typedef struct VirtIOBalloon
 {
     VirtIODevice vdev;
-    VirtQueue *ivq, *dvq;
+    VirtQueue *ivq, *dvq, *svq;
     uint32_t num_pages;
     uint32_t actual;
+    uint64_t stats[VIRTIO_BALLOON_S_NR];
+    VirtQueueElement stats_vq_elem;
+    size_t stats_vq_offset;
+    QEMUTimer *stats_timer;
+    uint64_t stats_updated;
+    Monitor *stats_mon;
+    bool stats_requested;
 } VirtIOBalloon;
 
 static VirtIOBalloon *to_virtio_balloon(VirtIODevice *vdev)
@@ -46,6 +57,50 @@ static void balloon_page(void *addr, int deflate)
 #endif
 }
 
+/*
+ * reset_stats - Mark all items in the stats array as unset
+ *
+ * This function needs to be called at device intialization and before
+ * before updating to a set of newly-generated stats.  This will ensure that no
+ * stale values stick around in case the guest reports a subset of the supported
+ * statistics.
+ */
+static inline void reset_stats(VirtIOBalloon *dev)
+{
+    int i;
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; dev->stats[i++] = -1);
+    dev->stats_updated = qemu_get_clock(host_clock);
+}
+
+static void stat_put(QDict *dict, const char *label, uint64_t val)
+{
+    if (val != -1)
+        qdict_put(dict, label, qint_from_int(val));
+}
+
+static QObject *get_stats_qobject(VirtIOBalloon *dev)
+{
+    QDict *dict = qdict_new();
+    uint32_t actual = ram_size - (dev->actual << VIRTIO_BALLOON_PFN_SHIFT);
+    uint64_t age;
+
+    stat_put(dict, "actual", actual);
+    stat_put(dict, "mem_swapped_in", dev->stats[VIRTIO_BALLOON_S_SWAP_IN]);
+    stat_put(dict, "mem_swapped_out", dev->stats[VIRTIO_BALLOON_S_SWAP_OUT]);
+    stat_put(dict, "major_page_faults", dev->stats[VIRTIO_BALLOON_S_MAJFLT]);
+    stat_put(dict, "minor_page_faults", dev->stats[VIRTIO_BALLOON_S_MINFLT]);
+    stat_put(dict, "free_mem", dev->stats[VIRTIO_BALLOON_S_MEMFREE]);
+    stat_put(dict, "total_mem", dev->stats[VIRTIO_BALLOON_S_MEMTOT]);
+
+    /* If age is over the timeout threshold, report it */
+    age = (qemu_get_clock(host_clock) - dev->stats_updated) /
+          (get_ticks_per_sec() / 1000);
+    if (age >= QEMU_BALLOON_SYNC_TIMEOUT * 1000)
+        stat_put(dict, "age", age);
+
+    return QOBJECT(dict);
+}
+
 /* FIXME: once we do a virtio refactoring, this will get subsumed into common
  * code */
 static size_t memcpy_from_iovector(void *data, size_t offset, size_t size,
@@ -104,6 +159,73 @@ static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
     }
 }
 
+/*
+ * complete_stats_request - Clean up and report statistics.
+ */
+static void complete_stats_request(VirtIOBalloon *vb)
+{
+    QObject *stats;
+
+    /* Only respond to incoming stats if we requested them */
+    if (!vb->stats_requested) {
+        return;
+    }
+
+    stats = get_stats_qobject(vb);
+    if (!monitor_ctrl_mode(vb->stats_mon)) {
+        qemu_del_timer(vb->stats_timer);
+        monitor_print_balloon(vb->stats_mon, stats);
+        monitor_resume(vb->stats_mon);
+    } else {
+        monitor_protocol_event(QEVENT_BALLOON, stats);
+    }
+
+    vb->stats_mon = NULL;
+    vb->stats_requested = false;
+}
+
+/*
+ * stats_request_timeout - Timer callback for sychronous request timeout
+ *
+ * In the case of a synchronous timeout, just report the old statistics.
+ */
+static void stats_request_timeout(void *opaque)
+{
+    VirtIOBalloon *vb = (VirtIOBalloon *)opaque;
+    complete_stats_request(vb);
+}
+
+static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOBalloon *s = DO_UPCAST(VirtIOBalloon, vdev, vdev);
+    VirtQueueElement *elem = &s->stats_vq_elem;
+    VirtIOBalloonStat stat;
+    size_t offset = 0;
+
+    if (!virtqueue_pop(vq, elem)) {
+        return;
+    }
+
+    /* Initialize the stats to get rid of any stale values.  This is only
+     * needed to handle the case where a guest supports fewer stats than it
+     * used to (ie. it has booted into an old kernel).
+     */
+    reset_stats(s);
+
+    while (memcpy_from_iovector(&stat, offset, sizeof(stat), elem->out_sg,
+                                elem->out_num) == sizeof(stat)) {
+        uint16_t tag = tswap16(stat.tag);
+        uint64_t val = tswap64(stat.val);
+
+        offset += sizeof(stat);
+        if (tag < VIRTIO_BALLOON_S_NR)
+            s->stats[tag] = val;
+    }
+    s->stats_vq_offset = offset;
+
+    complete_stats_request(s);
+}
+
 static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
 {
     VirtIOBalloon *dev = to_virtio_balloon(vdev);
@@ -126,10 +248,56 @@ static void virtio_balloon_set_config(VirtIODevice *vdev,
 
 static uint32_t virtio_balloon_get_features(VirtIODevice *vdev, uint32_t f)
 {
+    f |= (1 << VIRTIO_BALLOON_F_STATS_VQ);
     return f;
 }
 
-static ram_addr_t virtio_balloon_to_target(void *opaque, ram_addr_t target)
+/*
+ * request_stats - Initiate a request for memory statistics
+ *
+ * If stats are not supported, just report 'actual', otherwise, two types of
+ * requests are supported:
+ *
+ * Synchronous requests are intended to be used with the user monitor.  The
+ * monitor is suspended until the stats are received or the timer expires.
+ * Either the newly-update stats or the current stats (in the case of a timeout)
+ * are printed to the monitor and the monitor is resumed.
+ *
+ * Asynchronous requests are intended for QMP.  A QMP event will be generated
+ * only if the stats are updated.
+ */
+static void request_stats(VirtIOBalloon *vb, Monitor *mon)
+{
+    /* If a user-monitor is already waiting, resume that one first */
+    if (vb->stats_requested && !monitor_ctrl_mode(vb->stats_mon)) {
+            qemu_del_timer(vb->stats_timer);
+            monitor_resume(vb->stats_mon);
+    }
+    vb->stats_requested = true;
+    vb->stats_mon = mon;
+
+    /* Set up a synchronous request for a user-monitor. */
+    if (!monitor_ctrl_mode(mon)) {
+        uint64_t later = qemu_get_clock(vm_clock) +
+                             QEMU_BALLOON_SYNC_TIMEOUT * get_ticks_per_sec();
+        monitor_suspend(mon);
+        qemu_mod_timer(vb->stats_timer, later);
+    }
+
+    if (vb->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ)) {
+        virtqueue_push(vb->svq, &vb->stats_vq_elem, vb->stats_vq_offset);
+        virtio_notify(&vb->vdev, vb->svq);
+    } else {
+        /* Stats are not supported.  Clear out any stale values that might have
+         * been set by a more featureful guest kernel.
+         */
+        reset_stats(vb);
+        complete_stats_request(vb);
+    }
+}
+
+static void virtio_balloon_to_target(void *opaque, ram_addr_t target,
+                                     Monitor *mon)
 {
     VirtIOBalloon *dev = opaque;
 
@@ -139,9 +307,9 @@ static ram_addr_t virtio_balloon_to_target(void *opaque, ram_addr_t target)
     if (target) {
         dev->num_pages = (ram_size - target) >> VIRTIO_BALLOON_PFN_SHIFT;
         virtio_notify_config(&dev->vdev);
+    } else {
+        request_stats(dev, mon);
     }
-
-    return ram_size - (dev->actual << VIRTIO_BALLOON_PFN_SHIFT);
 }
 
 static void virtio_balloon_save(QEMUFile *f, void *opaque)
@@ -152,6 +320,11 @@ static void virtio_balloon_save(QEMUFile *f, void *opaque)
 
     qemu_put_be32(f, s->num_pages);
     qemu_put_be32(f, s->actual);
+    qemu_put_buffer(f, (uint8_t *)&s->stats_vq_elem, sizeof(VirtQueueElement));
+    qemu_put_buffer(f, (uint8_t *)&s->stats_vq_offset, sizeof(size_t));
+    qemu_put_timer(f, s->stats_timer);
+    qemu_put_be32(f, s->stats_updated);
+/* XXX save monitor and stats_requested */
 }
 
 static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
@@ -165,6 +338,11 @@ static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
 
     s->num_pages = qemu_get_be32(f);
     s->actual = qemu_get_be32(f);
+    qemu_get_buffer(f, (uint8_t *)&s->stats_vq_elem, sizeof(VirtQueueElement));
+    qemu_get_buffer(f, (uint8_t *)&s->stats_vq_offset, sizeof(size_t));
+    qemu_get_timer(f, s->stats_timer);
+    s->stats_updated = qemu_get_be32(f);
+/* XXX Load monitor and stats_requested */
 
     return 0;
 }
@@ -183,7 +361,10 @@ VirtIODevice *virtio_balloon_init(DeviceState *dev)
 
     s->ivq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
     s->dvq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
+    s->svq = virtio_add_queue(&s->vdev, 128, virtio_balloon_receive_stats);
 
+    s->stats_timer = qemu_new_timer(vm_clock, stats_request_timeout, s);
+    reset_stats(s);
     qemu_add_balloon_handler(virtio_balloon_to_target, s);
 
     register_savevm("virtio-balloon", -1, 1, virtio_balloon_save, virtio_balloon_load, s);
diff --git a/hw/virtio-balloon.h b/hw/virtio-balloon.h
index 9a0d119..e20cf6b 100644
--- a/hw/virtio-balloon.h
+++ b/hw/virtio-balloon.h
@@ -25,6 +25,7 @@
 
 /* The feature bitmap for virtio balloon */
 #define VIRTIO_BALLOON_F_MUST_TELL_HOST 0 /* Tell before reclaiming pages */
+#define VIRTIO_BALLOON_F_STATS_VQ 1       /* Memory stats virtqueue */
 
 /* Size of a PFN in the balloon interface. */
 #define VIRTIO_BALLOON_PFN_SHIFT 12
@@ -37,4 +38,18 @@ struct virtio_balloon_config
     uint32_t actual;
 };
 
+/* Memory Statistics */
+#define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
+#define VIRTIO_BALLOON_S_SWAP_OUT 1   /* Amount of memory swapped out */
+#define VIRTIO_BALLOON_S_MAJFLT   2   /* Number of major faults */
+#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
+#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
+#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
+#define VIRTIO_BALLOON_S_NR       6
+
+typedef struct VirtIOBalloonStat {
+    uint16_t tag;
+    uint64_t val;
+} __attribute__((packed)) VirtIOBalloonStat;
+
 #endif
diff --git a/monitor.c b/monitor.c
index b824e7c..134ed15 100644
--- a/monitor.c
+++ b/monitor.c
@@ -133,7 +133,7 @@ static void monitor_command_cb(Monitor *mon, const char *cmdline,
                                void *opaque);
 
 /* Return true if in control mode, false otherwise */
-static inline int monitor_ctrl_mode(const Monitor *mon)
+int monitor_ctrl_mode(const Monitor *mon)
 {
     return (mon->flags & MONITOR_USE_CONTROL);
 }
@@ -357,6 +357,9 @@ void monitor_protocol_event(MonitorEvent event, QObject *data)
         case QEVENT_STOP:
             event_name = "STOP";
             break;
+        case QEVENT_BALLOON:
+            event_name = "BALLOON";
+            break;
         default:
             abort();
             break;
@@ -2058,43 +2061,27 @@ static void do_info_status(Monitor *mon, QObject **ret_data)
                                     vm_running, singlestep);
 }
 
-static ram_addr_t balloon_get_value(void)
+static void print_balloon_stat(const char *key, QObject *obj, void *opaque)
 {
-    ram_addr_t actual;
-
-    if (kvm_enabled() && !kvm_has_sync_mmu()) {
-        qemu_error_new(QERR_KVM_MISSING_CAP, "synchronous MMU", "balloon");
-        return 0;
-    }
-
-    actual = qemu_balloon_status();
-    if (actual == 0) {
-        qemu_error_new(QERR_DEVICE_NOT_ACTIVE, "balloon");
-        return 0;
-    }
-
-    return actual;
-}
+    Monitor *mon = opaque;
 
-/**
- * do_balloon(): Request VM to change its memory allocation
- */
-static void do_balloon(Monitor *mon, const QDict *qdict, QObject **ret_data)
-{
-    if (balloon_get_value()) {
-        /* ballooning is active */
-        qemu_balloon(qdict_get_int(qdict, "value"));
-    }
+    if (strcmp(key, "actual"))
+        monitor_printf(mon, ",%s=%" PRId64, key,
+                            qint_get_int(qobject_to_qint(obj)));
 }
 
-static void monitor_print_balloon(Monitor *mon, const QObject *data)
+void monitor_print_balloon(Monitor *mon, const QObject *data)
 {
     QDict *qdict;
 
     qdict = qobject_to_qdict(data);
+    if (!qdict_haskey(qdict, "actual"))
+        return;
 
-    monitor_printf(mon, "balloon: actual=%" PRId64 "\n",
-                        qdict_get_int(qdict, "balloon") >> 20);
+    monitor_printf(mon, "balloon: actual=%" PRId64,
+                        qdict_get_int(qdict, "actual") >> 20);
+    qdict_iter(qdict, print_balloon_stat, mon);
+    monitor_printf(mon, "\n");
 }
 
 /**
@@ -2102,21 +2089,52 @@ static void monitor_print_balloon(Monitor *mon, const QObject *data)
  *
  * Return a QDict with the following information:
  *
- * - "balloon": current balloon value in bytes
+ * - "actual": current balloon value in bytes
+ * The following fields may or may not be present:
+ * - "mem_swapped_in": Amount of memory swapped in (bytes)
+ * - "mem_swapped_out": Amount of memory swapped out (bytes)
+ * - "major_page_faults": Number of major faults
+ * - "minor_page_faults": Number of minor faults
+ * - "free_mem": Total amount of free and unused memory (bytes)
+ * - "total_mem": Total amount of available memory (bytes)
  *
  * Example:
  *
- * { "balloon": 1073741824 }
+ * { "actual": 1073741824, "mem_swapped_in": 0, "mem_swapped_out": 0,
+ *   "major_page_faults": 142, "minor_page_faults": 239245,
+ *   "free_mem": 1014185984, "total_mem": 1044668416 }
  */
 static void do_info_balloon(Monitor *mon, QObject **ret_data)
 {
-    ram_addr_t actual;
+    int ret;
+
+    if (kvm_enabled() && !kvm_has_sync_mmu()) {
+        qemu_error_new(QERR_KVM_MISSING_CAP, "synchronous MMU", "balloon");
+        return;
+    }
+
+    ret = qemu_balloon_status(mon);
+    if (!ret) {
+        qemu_error_new(QERR_DEVICE_NOT_ACTIVE, "balloon");
+        return;
+    }
+}
+
+/**
+ * do_balloon(): Request VM to change its memory allocation
+ */
+static void do_balloon(Monitor *mon, const QDict *qdict, QObject **ret_data)
+{
+    int ret;
 
-    actual = balloon_get_value();
-    if (actual != 0) {
-        *ret_data = qobject_from_jsonf("{ 'balloon': %" PRId64 "}",
-                                       (int64_t) actual);
+    if (kvm_enabled() && !kvm_has_sync_mmu()) {
+        qemu_error_new(QERR_KVM_MISSING_CAP, "synchronous MMU", "balloon");
+        return;
     }
+
+    ret = qemu_balloon(qdict_get_int(qdict, "value"));
+    if (ret == 0)
+        qemu_error_new(QERR_DEVICE_NOT_ACTIVE, "balloon");
 }
 
 static qemu_acl *find_acl(Monitor *mon, const char *name)
diff --git a/monitor.h b/monitor.h
index 6ed117a..556239a 100644
--- a/monitor.h
+++ b/monitor.h
@@ -20,6 +20,7 @@ typedef enum MonitorEvent {
     QEVENT_RESET,
     QEVENT_POWERDOWN,
     QEVENT_STOP,
+    QEVENT_BALLOON,
     QEVENT_MAX,
 } MonitorEvent;
 
@@ -32,6 +33,7 @@ void monitor_resume(Monitor *mon);
 void monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
                                  BlockDriverCompletionFunc *completion_cb,
                                  void *opaque);
+void monitor_print_balloon(Monitor *mon, const QObject *data);
 
 int monitor_get_fd(Monitor *mon, const char *fdname);
 
@@ -41,4 +43,6 @@ void monitor_printf(Monitor *mon, const char *fmt, ...)
 void monitor_print_filename(Monitor *mon, const char *filename);
 void monitor_flush(Monitor *mon);
 
+extern int monitor_ctrl_mode(const Monitor *mon);
+
 #endif /* !MONITOR_H */
diff --git a/vl.c b/vl.c
index b048e89..e4bf434 100644
--- a/vl.c
+++ b/vl.c
@@ -362,17 +362,24 @@ void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque)
     qemu_balloon_event_opaque = opaque;
 }
 
-void qemu_balloon(ram_addr_t target)
+int qemu_balloon(ram_addr_t target)
 {
-    if (qemu_balloon_event)
-        qemu_balloon_event(qemu_balloon_event_opaque, target);
+    if (qemu_balloon_event) {
+        qemu_balloon_event(qemu_balloon_event_opaque, target, NULL);
+        return 1;
+    } else {
+        return 0;
+    }
 }
 
-ram_addr_t qemu_balloon_status(void)
+int qemu_balloon_status(Monitor *mon)
 {
-    if (qemu_balloon_event)
-        return qemu_balloon_event(qemu_balloon_event_opaque, 0);
-    return 0;
+    if (qemu_balloon_event) {
+        qemu_balloon_event(qemu_balloon_event_opaque, 0, mon);
+        return 1;
+    } else {
+        return 0;
+    }
 }
 
 /***********************************************************/


-- 
Thanks,
Adam

[Qemu-devel] Re: virtio: Add memory statistics reporting to the balloon driver (V7)

[Luiz Capitulino]

On Fri, 15 Jan 2010 13:54:29 -0600
Adam Litke <agl@us.ibm.com> wrote:

> This version improves support for multiple monitors and has been ported up to
> HEAD as of 01/14.

 Overall review on the Monitor related changes seems ok, but I'm not sure
how I should enable it.

[Qemu-devel] Re: virtio: Add memory statistics reporting to the balloon driver (V7)

[Adam Litke]

On Mon, 2010-01-18 at 12:12 -0200, Luiz Capitulino wrote:
> On Fri, 15 Jan 2010 13:54:29 -0600
> Adam Litke <agl@us.ibm.com> wrote:
> 
> > This version improves support for multiple monitors and has been ported up to
> > HEAD as of 01/14.
> 
>  Overall review on the Monitor related changes seems ok, but I'm not sure
> how I should enable it.

You must run a Linux guest with the virtio balloon enabled (-balloon
virtio) and the guest kernel must support the stats API.  Rusty Russell
has accepted my patch into his tree and will be pushing it upstream.  In
the meantime, you could build a 2.6.33-rc3 kernel with the following
patch applied...

commit b9bb81c2f7ce82d636d47089d6aa279d9be704f2
Author: litke@us.ibm.com <aglitke@kernel.beaverton.ibm.com>
Date:   Fri Jan 8 14:17:45 2010 -0800

    balloon memory stats patch

diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 9dd5880..f95be86 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -28,7 +28,7 @@
 struct virtio_balloon
 {
 	struct virtio_device *vdev;
-	struct virtqueue *inflate_vq, *deflate_vq;
+	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
 
 	/* Where the ballooning thread waits for config to change. */
 	wait_queue_head_t config_change;
@@ -49,6 +49,10 @@ struct virtio_balloon
 	/* The array of pfns we tell the Host about. */
 	unsigned int num_pfns;
 	u32 pfns[256];
+
+	/* Memory statistics */
+	int need_stats_update;
+	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
 };
 
 static struct virtio_device_id id_table[] = {
@@ -154,6 +158,72 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num)
 	}
 }
 
+static inline void update_stat(struct virtio_balloon *vb, int idx,
+			       u16 tag, u64 val)
+{
+	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
+	vb->stats[idx].tag = tag;
+	vb->stats[idx].val = val;
+}
+
+#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
+
+static void update_balloon_stats(struct virtio_balloon *vb)
+{
+	unsigned long events[NR_VM_EVENT_ITEMS];
+	struct sysinfo i;
+	int idx = 0;
+
+	all_vm_events(events);
+	si_meminfo(&i);
+
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
+				pages_to_bytes(events[PSWPIN]));
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
+				pages_to_bytes(events[PSWPOUT]));
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
+				pages_to_bytes(i.freeram));
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
+				pages_to_bytes(i.totalram));
+}
+
+/*
+ * While most virtqueues communicate guest-initiated requests to the hypervisor,
+ * the stats queue operates in reverse.  The driver initializes the virtqueue
+ * with a single buffer.  From that point forward, all conversations consist of
+ * a hypervisor request (a call to this function) which directs us to refill
+ * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
+ * we notify our kthread which does the actual work via stats_handle_request().
+ */
+static void stats_request(struct virtqueue *vq)
+{
+	struct virtio_balloon *vb;
+	unsigned int len;
+
+	vb = vq->vq_ops->get_buf(vq, &len);
+	if (!vb)
+		return;
+	vb->need_stats_update = 1;
+	wake_up(&vb->config_change);
+}
+
+static void stats_handle_request(struct virtio_balloon *vb)
+{
+	struct virtqueue *vq;
+	struct scatterlist sg;
+
+	vb->need_stats_update = 0;
+	update_balloon_stats(vb);
+
+	vq = vb->stats_vq;
+	sg_init_one(&sg, vb->stats, sizeof(vb->stats));
+	if (vq->vq_ops->add_buf(vq, &sg, 1, 0, vb) < 0)
+		BUG();
+	vq->vq_ops->kick(vq);
+}
+
 static void virtballoon_changed(struct virtio_device *vdev)
 {
 	struct virtio_balloon *vb = vdev->priv;
@@ -190,8 +260,11 @@ static int balloon(void *_vballoon)
 		try_to_freeze();
 		wait_event_interruptible(vb->config_change,
 					 (diff = towards_target(vb)) != 0
+					 || vb->need_stats_update
 					 || kthread_should_stop()
 					 || freezing(current));
+		if (vb->need_stats_update)
+			stats_handle_request(vb);
 		if (diff > 0)
 			fill_balloon(vb, diff);
 		else if (diff < 0)
@@ -204,10 +277,10 @@ static int balloon(void *_vballoon)
 static int virtballoon_probe(struct virtio_device *vdev)
 {
 	struct virtio_balloon *vb;
-	struct virtqueue *vqs[2];
-	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack };
-	const char *names[] = { "inflate", "deflate" };
-	int err;
+	struct virtqueue *vqs[3];
+	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
+	const char *names[] = { "inflate", "deflate", "stats" };
+	int err, nvqs;
 
 	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
 	if (!vb) {
@@ -220,13 +293,29 @@ static int virtballoon_probe(struct virtio_device *vdev)
 	init_waitqueue_head(&vb->config_change);
 	vb->vdev = vdev;
 
-	/* We expect two virtqueues. */
-	err = vdev->config->find_vqs(vdev, 2, vqs, callbacks, names);
+	/* We expect two virtqueues: inflate and deflate,
+	 * and optionally stat. */
+	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
+	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
 	if (err)
 		goto out_free_vb;
 
 	vb->inflate_vq = vqs[0];
 	vb->deflate_vq = vqs[1];
+	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
+		struct scatterlist sg;
+		vb->stats_vq = vqs[2];
+
+		/*
+		 * Prime this virtqueue with one buffer so the hypervisor can
+		 * use it to signal us later.
+		 */
+		sg_init_one(&sg, vb->stats, sizeof vb->stats);
+		if (vb->stats_vq->vq_ops->add_buf(vb->stats_vq,
+						  &sg, 1, 0, vb) < 0)
+			BUG();
+		vb->stats_vq->vq_ops->kick(vb->stats_vq);
+	}
 
 	vb->thread = kthread_run(balloon, vb, "vballoon");
 	if (IS_ERR(vb->thread)) {
@@ -264,7 +353,10 @@ static void __devexit virtballoon_remove(struct virtio_device *vdev)
 	kfree(vb);
 }
 
-static unsigned int features[] = { VIRTIO_BALLOON_F_MUST_TELL_HOST };
+static unsigned int features[] = {
+	VIRTIO_BALLOON_F_MUST_TELL_HOST,
+	VIRTIO_BALLOON_F_STATS_VQ,
+};
 
 static struct virtio_driver virtio_balloon = {
 	.feature_table = features,
diff --git a/include/linux/virtio_balloon.h b/include/linux/virtio_balloon.h
index 1418f04..a50ecd1 100644
--- a/include/linux/virtio_balloon.h
+++ b/include/linux/virtio_balloon.h
@@ -7,6 +7,7 @@
 
 /* The feature bitmap for virtio balloon */
 #define VIRTIO_BALLOON_F_MUST_TELL_HOST	0 /* Tell before reclaiming pages */
+#define VIRTIO_BALLOON_F_STATS_VQ	1 /* Memory Stats virtqueue */
 
 /* Size of a PFN in the balloon interface. */
 #define VIRTIO_BALLOON_PFN_SHIFT 12
@@ -18,4 +19,18 @@ struct virtio_balloon_config
 	/* Number of pages we've actually got in balloon. */
 	__le32 actual;
 };
+
+#define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
+#define VIRTIO_BALLOON_S_SWAP_OUT 1   /* Amount of memory swapped out */
+#define VIRTIO_BALLOON_S_MAJFLT   2   /* Number of major faults */
+#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
+#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
+#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
+#define VIRTIO_BALLOON_S_NR       6
+
+struct virtio_balloon_stat {
+	u16 tag;
+	u64 val;
+} __attribute__((packed));
+
 #endif /* _LINUX_VIRTIO_BALLOON_H */


-- 
Thanks,
Adam