[RFC] virtio: use mandatory barriers for remote processor vdevs

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

Virtio is using memory barriers to control the ordering of
references to the vrings on SMP systems. When the guest is compiled
with SMP support, virtio is only using SMP barriers in order to
avoid incurring the overhead involved with mandatory barriers.

Lately, though, virtio is being increasingly used with inter-processor
communication scenarios too, which involve running two (separate)
instances of operating systems on two (separate) processors, each of
which might either be UP or SMP.

To control the ordering of memory references when the vrings are shared
between two external processors, we must always use mandatory barriers.

A trivial, albeit sub-optimal, solution would be to simply revert
commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
that's going to have a negative impact on performance of SMP-based
virtualization use cases.

A different approach, as demonstrated by this patch, would pick the type
of memory barriers, in run time, according to the requirements of the
virtio device. This way, both SMP virtualization scenarios and inter-
processor communication use cases would run correctly, without making
any performance compromises (except for those incurred by an additional
branch or level of indirection).

This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
feature, which should be used by virtio devices that run on remote
processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
---
Alternatively, we can also introduce some kind of virtio_mb_ops and set it
according to the nature of the vdev with handlers that just do the right
thing, instead of introducting that branch.

Though I also wonder how big really is the perforamnce gain of d57ed95 ?

 drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
 include/linux/virtio_ring.h  |    6 +++
 2 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c7a2c20..cf66a2d 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,24 +23,6 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
-/* virtio guest is communicating with a virtual "device" that actually runs on
- * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
-/* Where possible, use SMP barriers which are more lightweight than mandatory
- * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
-#else
-/* We must force memory ordering even if guest is UP since host could be
- * running on another CPU, but SMP barriers are defined to barrier() in that
- * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
-#endif
-
 #ifdef DEBUG
 /* For development, we want to crash whenever the ring is screwed. */
 #define BAD_RING(_vq, fmt, args...)				\
@@ -86,6 +68,9 @@ struct vring_virtqueue
 	/* Host publishes avail event idx */
 	bool event;
 
+	/* Host runs on a remote processor */
+	bool rproc;
+
 	/* Number of free buffers */
 	unsigned int num_free;
 	/* Head of free buffer list. */
@@ -108,6 +93,48 @@ struct vring_virtqueue
 	void *data[];
 };
 
+/*
+ * virtio guest is communicating with a virtual "device" that may either run
+ * on the host processor, or on an external processor. The former requires
+ * memory barriers in order to control SMP effects, but the latter must
+ * use mandatory barriers.
+ */
+#ifdef CONFIG_SMP
+/* Where possible, use SMP barriers which are more lightweight than mandatory
+ * barriers, because mandatory barriers control MMIO effects on accesses
+ * through relaxed memory I/O windows. */
+static inline void virtio_mb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		mb();
+	else
+		smp_mb();
+}
+
+static inline void virtio_rmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		rmb();
+	else
+		smp_rmb();
+}
+
+static inline void virtio_wmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		wmb();
+	else
+		smp_wmb();
+}
+#else
+/* We must force memory ordering even if guest is UP since host could be
+ * running on another CPU, but SMP barriers are defined to barrier() in that
+ * configuration. So fall back to mandatory barriers instead. */
+static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
+static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
+static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
+#endif
+
 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 
 /* Set up an indirect table of descriptors and add it to the queue. */
@@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
 
 	/* No callback?  Tell other side not to bother us. */
 	if (!callback)
@@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
 			break;
 		case VIRTIO_RING_F_EVENT_IDX:
 			break;
+		case VIRTIO_RING_F_REMOTEPROC:
+			break;
 		default:
 			/* We don't understand this bit. */
 			clear_bit(i, vdev->features);
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 36be0f6..9839593 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -58,6 +58,12 @@
  * at the end of the used ring. Guest should ignore the used->flags field. */
 #define VIRTIO_RING_F_EVENT_IDX		29
 
+/*
+ * The device we're talking to resides on a remote processor, so we must always
+ * use mandatory memory barriers.
+ */
+#define VIRTIO_RING_F_REMOTEPROC	30
+
 /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
 struct vring_desc {
 	/* Address (guest-physical). */
-- 
1.7.5.4

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

Virtio is using memory barriers to control the ordering of
references to the vrings on SMP systems. When the guest is compiled
with SMP support, virtio is only using SMP barriers in order to
avoid incurring the overhead involved with mandatory barriers.

Lately, though, virtio is being increasingly used with inter-processor
communication scenarios too, which involve running two (separate)
instances of operating systems on two (separate) processors, each of
which might either be UP or SMP.

To control the ordering of memory references when the vrings are shared
between two external processors, we must always use mandatory barriers.

A trivial, albeit sub-optimal, solution would be to simply revert
commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
that's going to have a negative impact on performance of SMP-based
virtualization use cases.

A different approach, as demonstrated by this patch, would pick the type
of memory barriers, in run time, according to the requirements of the
virtio device. This way, both SMP virtualization scenarios and inter-
processor communication use cases would run correctly, without making
any performance compromises (except for those incurred by an additional
branch or level of indirection).

This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
feature, which should be used by virtio devices that run on remote
processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
---
Alternatively, we can also introduce some kind of virtio_mb_ops and set it
according to the nature of the vdev with handlers that just do the right
thing, instead of introducting that branch.

Though I also wonder how big really is the perforamnce gain of d57ed95 ?

 drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
 include/linux/virtio_ring.h  |    6 +++
 2 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c7a2c20..cf66a2d 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,24 +23,6 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
-/* virtio guest is communicating with a virtual "device" that actually runs on
- * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
-/* Where possible, use SMP barriers which are more lightweight than mandatory
- * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
-#else
-/* We must force memory ordering even if guest is UP since host could be
- * running on another CPU, but SMP barriers are defined to barrier() in that
- * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
-#endif
-
 #ifdef DEBUG
 /* For development, we want to crash whenever the ring is screwed. */
 #define BAD_RING(_vq, fmt, args...)				\
@@ -86,6 +68,9 @@ struct vring_virtqueue
 	/* Host publishes avail event idx */
 	bool event;
 
+	/* Host runs on a remote processor */
+	bool rproc;
+
 	/* Number of free buffers */
 	unsigned int num_free;
 	/* Head of free buffer list. */
@@ -108,6 +93,48 @@ struct vring_virtqueue
 	void *data[];
 };
 
+/*
+ * virtio guest is communicating with a virtual "device" that may either run
+ * on the host processor, or on an external processor. The former requires
+ * memory barriers in order to control SMP effects, but the latter must
+ * use mandatory barriers.
+ */
+#ifdef CONFIG_SMP
+/* Where possible, use SMP barriers which are more lightweight than mandatory
+ * barriers, because mandatory barriers control MMIO effects on accesses
+ * through relaxed memory I/O windows. */
+static inline void virtio_mb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		mb();
+	else
+		smp_mb();
+}
+
+static inline void virtio_rmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		rmb();
+	else
+		smp_rmb();
+}
+
+static inline void virtio_wmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		wmb();
+	else
+		smp_wmb();
+}
+#else
+/* We must force memory ordering even if guest is UP since host could be
+ * running on another CPU, but SMP barriers are defined to barrier() in that
+ * configuration. So fall back to mandatory barriers instead. */
+static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
+static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
+static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
+#endif
+
 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 
 /* Set up an indirect table of descriptors and add it to the queue. */
@@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
 
 	/* No callback?  Tell other side not to bother us. */
 	if (!callback)
@@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
 			break;
 		case VIRTIO_RING_F_EVENT_IDX:
 			break;
+		case VIRTIO_RING_F_REMOTEPROC:
+			break;
 		default:
 			/* We don't understand this bit. */
 			clear_bit(i, vdev->features);
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 36be0f6..9839593 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -58,6 +58,12 @@
  * at the end of the used ring. Guest should ignore the used->flags field. */
 #define VIRTIO_RING_F_EVENT_IDX		29
 
+/*
+ * The device we're talking to resides on a remote processor, so we must always
+ * use mandatory memory barriers.
+ */
+#define VIRTIO_RING_F_REMOTEPROC	30
+
 /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
 struct vring_desc {
 	/* Address (guest-physical). */
-- 
1.7.5.4

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.

Is that using virtio-mmio? If yes, would the extra serialization belongs
at that layer?

> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.

Sorry, could you pls explain what are 'two external processors'?
I think I know that if two x86 CPUs in an SMP system run kernels built
in an SMP configuration, smp_*mb barriers are enough.

Documentation/memory-barriers.txt says:
	Mandatory barriers should not be used to control SMP effects ...
	They may, however, be used to control MMIO effects on accesses through
	relaxed memory I/O windows.

We don't control MMIO/relaxed memory I/O windows here, so what exactly
is the issue?

Could you please give an example of a setup that is currently broken?

> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.
> 
> A different approach, as demonstrated by this patch, would pick the type
> of memory barriers, in run time, according to the requirements of the
> virtio device. This way, both SMP virtualization scenarios and inter-
> processor communication use cases would run correctly, without making
> any performance compromises (except for those incurred by an additional
> branch or level of indirection).

Is an extra branch faster or slower than reverting d57ed95?

> 
> This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
> feature, which should be used by virtio devices that run on remote
> processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
> to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

One wonders how the remote side knows enough to set this flag?

> 
> Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
> ---
> Alternatively, we can also introduce some kind of virtio_mb_ops and set it
> according to the nature of the vdev with handlers that just do the right
> thing, instead of introducting that branch.
> 
> Though I also wonder how big really is the perforamnce gain of d57ed95 ?

Want to check and tell us?

>  drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
>  include/linux/virtio_ring.h  |    6 +++
>  2 files changed, 60 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index c7a2c20..cf66a2d 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,24 +23,6 @@
>  #include <linux/slab.h>
>  #include <linux/module.h>
>  
> -/* virtio guest is communicating with a virtual "device" that actually runs on
> - * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> -/* Where possible, use SMP barriers which are more lightweight than mandatory
> - * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> -#else
> -/* We must force memory ordering even if guest is UP since host could be
> - * running on another CPU, but SMP barriers are defined to barrier() in that
> - * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> -#endif
> -
>  #ifdef DEBUG
>  /* For development, we want to crash whenever the ring is screwed. */
>  #define BAD_RING(_vq, fmt, args...)				\
> @@ -86,6 +68,9 @@ struct vring_virtqueue
>  	/* Host publishes avail event idx */
>  	bool event;
>  
> +	/* Host runs on a remote processor */
> +	bool rproc;
> +
>  	/* Number of free buffers */
>  	unsigned int num_free;
>  	/* Head of free buffer list. */
> @@ -108,6 +93,48 @@ struct vring_virtqueue
>  	void *data[];
>  };
>  
> +/*
> + * virtio guest is communicating with a virtual "device" that may either run
> + * on the host processor, or on an external processor. The former requires
> + * memory barriers in order to control SMP effects, but the latter must
> + * use mandatory barriers.
> + */
> +#ifdef CONFIG_SMP
> +/* Where possible, use SMP barriers which are more lightweight than mandatory
> + * barriers, because mandatory barriers control MMIO effects on accesses
> + * through relaxed memory I/O windows. */
> +static inline void virtio_mb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		mb();
> +	else
> +		smp_mb();
> +}
> +
> +static inline void virtio_rmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		rmb();
> +	else
> +		smp_rmb();
> +}
> +
> +static inline void virtio_wmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		wmb();
> +	else
> +		smp_wmb();
> +}
> +#else
> +/* We must force memory ordering even if guest is UP since host could be
> + * running on another CPU, but SMP barriers are defined to barrier() in that
> + * configuration. So fall back to mandatory barriers instead. */
> +static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
> +static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
> +static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
> +#endif
> +
>  #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
>  
>  /* Set up an indirect table of descriptors and add it to the queue. */
> @@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
>  
>  	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
>  	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
> +	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
>  
>  	/* No callback?  Tell other side not to bother us. */
>  	if (!callback)
> @@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
>  			break;
>  		case VIRTIO_RING_F_EVENT_IDX:
>  			break;
> +		case VIRTIO_RING_F_REMOTEPROC:
> +			break;
>  		default:
>  			/* We don't understand this bit. */
>  			clear_bit(i, vdev->features);
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> index 36be0f6..9839593 100644
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -58,6 +58,12 @@
>   * at the end of the used ring. Guest should ignore the used->flags field. */
>  #define VIRTIO_RING_F_EVENT_IDX		29
>  
> +/*
> + * The device we're talking to resides on a remote processor, so we must always
> + * use mandatory memory barriers.
> + */
> +#define VIRTIO_RING_F_REMOTEPROC	30
> +
>  /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
>  struct vring_desc {
>  	/* Address (guest-physical). */
> -- 
> 1.7.5.4

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.

Is that using virtio-mmio? If yes, would the extra serialization belongs
at that layer?

> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.

Sorry, could you pls explain what are 'two external processors'?
I think I know that if two x86 CPUs in an SMP system run kernels built
in an SMP configuration, smp_*mb barriers are enough.

Documentation/memory-barriers.txt says:
	Mandatory barriers should not be used to control SMP effects ...
	They may, however, be used to control MMIO effects on accesses through
	relaxed memory I/O windows.

We don't control MMIO/relaxed memory I/O windows here, so what exactly
is the issue?

Could you please give an example of a setup that is currently broken?

> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.
> 
> A different approach, as demonstrated by this patch, would pick the type
> of memory barriers, in run time, according to the requirements of the
> virtio device. This way, both SMP virtualization scenarios and inter-
> processor communication use cases would run correctly, without making
> any performance compromises (except for those incurred by an additional
> branch or level of indirection).

Is an extra branch faster or slower than reverting d57ed95?

> 
> This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
> feature, which should be used by virtio devices that run on remote
> processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
> to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

One wonders how the remote side knows enough to set this flag?

> 
> Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
> ---
> Alternatively, we can also introduce some kind of virtio_mb_ops and set it
> according to the nature of the vdev with handlers that just do the right
> thing, instead of introducting that branch.
> 
> Though I also wonder how big really is the perforamnce gain of d57ed95 ?

Want to check and tell us?

>  drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
>  include/linux/virtio_ring.h  |    6 +++
>  2 files changed, 60 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index c7a2c20..cf66a2d 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,24 +23,6 @@
>  #include <linux/slab.h>
>  #include <linux/module.h>
>  
> -/* virtio guest is communicating with a virtual "device" that actually runs on
> - * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> -/* Where possible, use SMP barriers which are more lightweight than mandatory
> - * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> -#else
> -/* We must force memory ordering even if guest is UP since host could be
> - * running on another CPU, but SMP barriers are defined to barrier() in that
> - * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> -#endif
> -
>  #ifdef DEBUG
>  /* For development, we want to crash whenever the ring is screwed. */
>  #define BAD_RING(_vq, fmt, args...)				\
> @@ -86,6 +68,9 @@ struct vring_virtqueue
>  	/* Host publishes avail event idx */
>  	bool event;
>  
> +	/* Host runs on a remote processor */
> +	bool rproc;
> +
>  	/* Number of free buffers */
>  	unsigned int num_free;
>  	/* Head of free buffer list. */
> @@ -108,6 +93,48 @@ struct vring_virtqueue
>  	void *data[];
>  };
>  
> +/*
> + * virtio guest is communicating with a virtual "device" that may either run
> + * on the host processor, or on an external processor. The former requires
> + * memory barriers in order to control SMP effects, but the latter must
> + * use mandatory barriers.
> + */
> +#ifdef CONFIG_SMP
> +/* Where possible, use SMP barriers which are more lightweight than mandatory
> + * barriers, because mandatory barriers control MMIO effects on accesses
> + * through relaxed memory I/O windows. */
> +static inline void virtio_mb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		mb();
> +	else
> +		smp_mb();
> +}
> +
> +static inline void virtio_rmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		rmb();
> +	else
> +		smp_rmb();
> +}
> +
> +static inline void virtio_wmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		wmb();
> +	else
> +		smp_wmb();
> +}
> +#else
> +/* We must force memory ordering even if guest is UP since host could be
> + * running on another CPU, but SMP barriers are defined to barrier() in that
> + * configuration. So fall back to mandatory barriers instead. */
> +static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
> +static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
> +static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
> +#endif
> +
>  #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
>  
>  /* Set up an indirect table of descriptors and add it to the queue. */
> @@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
>  
>  	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
>  	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
> +	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
>  
>  	/* No callback?  Tell other side not to bother us. */
>  	if (!callback)
> @@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
>  			break;
>  		case VIRTIO_RING_F_EVENT_IDX:
>  			break;
> +		case VIRTIO_RING_F_REMOTEPROC:
> +			break;
>  		default:
>  			/* We don't understand this bit. */
>  			clear_bit(i, vdev->features);
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> index 36be0f6..9839593 100644
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -58,6 +58,12 @@
>   * at the end of the used ring. Guest should ignore the used->flags field. */
>  #define VIRTIO_RING_F_EVENT_IDX		29
>  
> +/*
> + * The device we're talking to resides on a remote processor, so we must always
> + * use mandatory memory barriers.
> + */
> +#define VIRTIO_RING_F_REMOTEPROC	30
> +
>  /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
>  struct vring_desc {
>  	/* Address (guest-physical). */
> -- 
> 1.7.5.4

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.

Is that using virtio-mmio? If yes, would the extra serialization belongs
at that layer?

> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.

Sorry, could you pls explain what are 'two external processors'?
I think I know that if two x86 CPUs in an SMP system run kernels built
in an SMP configuration, smp_*mb barriers are enough.

Documentation/memory-barriers.txt says:
	Mandatory barriers should not be used to control SMP effects ...
	They may, however, be used to control MMIO effects on accesses through
	relaxed memory I/O windows.

We don't control MMIO/relaxed memory I/O windows here, so what exactly
is the issue?

Could you please give an example of a setup that is currently broken?

> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.
> 
> A different approach, as demonstrated by this patch, would pick the type
> of memory barriers, in run time, according to the requirements of the
> virtio device. This way, both SMP virtualization scenarios and inter-
> processor communication use cases would run correctly, without making
> any performance compromises (except for those incurred by an additional
> branch or level of indirection).

Is an extra branch faster or slower than reverting d57ed95?

> 
> This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
> feature, which should be used by virtio devices that run on remote
> processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
> to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

One wonders how the remote side knows enough to set this flag?

> 
> Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
> ---
> Alternatively, we can also introduce some kind of virtio_mb_ops and set it
> according to the nature of the vdev with handlers that just do the right
> thing, instead of introducting that branch.
> 
> Though I also wonder how big really is the perforamnce gain of d57ed95 ?

Want to check and tell us?

>  drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
>  include/linux/virtio_ring.h  |    6 +++
>  2 files changed, 60 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index c7a2c20..cf66a2d 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,24 +23,6 @@
>  #include <linux/slab.h>
>  #include <linux/module.h>
>  
> -/* virtio guest is communicating with a virtual "device" that actually runs on
> - * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> -/* Where possible, use SMP barriers which are more lightweight than mandatory
> - * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> -#else
> -/* We must force memory ordering even if guest is UP since host could be
> - * running on another CPU, but SMP barriers are defined to barrier() in that
> - * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> -#endif
> -
>  #ifdef DEBUG
>  /* For development, we want to crash whenever the ring is screwed. */
>  #define BAD_RING(_vq, fmt, args...)				\
> @@ -86,6 +68,9 @@ struct vring_virtqueue
>  	/* Host publishes avail event idx */
>  	bool event;
>  
> +	/* Host runs on a remote processor */
> +	bool rproc;
> +
>  	/* Number of free buffers */
>  	unsigned int num_free;
>  	/* Head of free buffer list. */
> @@ -108,6 +93,48 @@ struct vring_virtqueue
>  	void *data[];
>  };
>  
> +/*
> + * virtio guest is communicating with a virtual "device" that may either run
> + * on the host processor, or on an external processor. The former requires
> + * memory barriers in order to control SMP effects, but the latter must
> + * use mandatory barriers.
> + */
> +#ifdef CONFIG_SMP
> +/* Where possible, use SMP barriers which are more lightweight than mandatory
> + * barriers, because mandatory barriers control MMIO effects on accesses
> + * through relaxed memory I/O windows. */
> +static inline void virtio_mb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		mb();
> +	else
> +		smp_mb();
> +}
> +
> +static inline void virtio_rmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		rmb();
> +	else
> +		smp_rmb();
> +}
> +
> +static inline void virtio_wmb(struct vring_virtqueue *vq)
> +{
> +	if (vq->rproc)
> +		wmb();
> +	else
> +		smp_wmb();
> +}
> +#else
> +/* We must force memory ordering even if guest is UP since host could be
> + * running on another CPU, but SMP barriers are defined to barrier() in that
> + * configuration. So fall back to mandatory barriers instead. */
> +static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
> +static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
> +static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
> +#endif
> +
>  #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
>  
>  /* Set up an indirect table of descriptors and add it to the queue. */
> @@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
>  
>  	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
>  	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
> +	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
>  
>  	/* No callback?  Tell other side not to bother us. */
>  	if (!callback)
> @@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
>  			break;
>  		case VIRTIO_RING_F_EVENT_IDX:
>  			break;
> +		case VIRTIO_RING_F_REMOTEPROC:
> +			break;
>  		default:
>  			/* We don't understand this bit. */
>  			clear_bit(i, vdev->features);
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> index 36be0f6..9839593 100644
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -58,6 +58,12 @@
>   * at the end of the used ring. Guest should ignore the used->flags field. */
>  #define VIRTIO_RING_F_EVENT_IDX		29
>  
> +/*
> + * The device we're talking to resides on a remote processor, so we must always
> + * use mandatory memory barriers.
> + */
> +#define VIRTIO_RING_F_REMOTEPROC	30
> +
>  /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
>  struct vring_desc {
>  	/* Address (guest-physical). */
> -- 
> 1.7.5.4

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
>> Virtio is using memory barriers to control the ordering of
>> references to the vrings on SMP systems. When the guest is compiled
>> with SMP support, virtio is only using SMP barriers in order to
>> avoid incurring the overhead involved with mandatory barriers.
>>
>> Lately, though, virtio is being increasingly used with inter-processor
>> communication scenarios too, which involve running two (separate)
>> instances of operating systems on two (separate) processors, each of
>> which might either be UP or SMP.
>
> Is that using virtio-mmio?

No, I'm using this:

https://lkml.org/lkml/2011/10/25/139

> Sorry, could you pls explain what are 'two external processors'?
> I think I know that if two x86 CPUs in an SMP system run kernels built
> in an SMP configuration, smp_*mb barriers are enough.

Sure:

My setup is not SMP-based; it's two separate processors running in AMP
configuration. The processors have completely different architectures,
are not cache coherent, and only simply share some memory, which is
used for communications using virtio as the shared memory "wire"
protocol (i.e. we're not even doing virtualization: we have Linux on
one processor, and some RTOS on another processor, and they use virtio
to send and receive buffers).

So it's not SMP effects we're controlling; we're pretty much doing
MMIO and must use mandatory barriers (otherwise we see breakage).

> Is an extra branch faster or slower than reverting d57ed95?

Sorry, unfortunately I have no way to measure this, as I don't have
any virtualization/x86 setup. I'm developing on ARM SoCs, where
virtualization hardware is coming, but not here yet.

> One wonders how the remote side knows enough to set this flag?

The remote side is a dedicated firmware loaded in order to boot the
other processor, so it really is intimate with the platform and its
requirements (and it publishes the virtio device features for every
supported virtio device).

>> Though I also wonder how big really is the perforamnce gain of d57ed95 ?
>
> Want to check and tell us?

Sorry, as I said, I have no way to do that... Any chance you did some
measurements back then when d57ed95 was introduced ?

I just guessed it did improve performance, otherwise you probably
wouldn't have bothered. But if not, then reverting it is surely
preferable to adding more complexity.

Thanks!
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
>> Virtio is using memory barriers to control the ordering of
>> references to the vrings on SMP systems. When the guest is compiled
>> with SMP support, virtio is only using SMP barriers in order to
>> avoid incurring the overhead involved with mandatory barriers.
>>
>> Lately, though, virtio is being increasingly used with inter-processor
>> communication scenarios too, which involve running two (separate)
>> instances of operating systems on two (separate) processors, each of
>> which might either be UP or SMP.
>
> Is that using virtio-mmio?

No, I'm using this:

https://lkml.org/lkml/2011/10/25/139

> Sorry, could you pls explain what are 'two external processors'?
> I think I know that if two x86 CPUs in an SMP system run kernels built
> in an SMP configuration, smp_*mb barriers are enough.

Sure:

My setup is not SMP-based; it's two separate processors running in AMP
configuration. The processors have completely different architectures,
are not cache coherent, and only simply share some memory, which is
used for communications using virtio as the shared memory "wire"
protocol (i.e. we're not even doing virtualization: we have Linux on
one processor, and some RTOS on another processor, and they use virtio
to send and receive buffers).

So it's not SMP effects we're controlling; we're pretty much doing
MMIO and must use mandatory barriers (otherwise we see breakage).

> Is an extra branch faster or slower than reverting d57ed95?

Sorry, unfortunately I have no way to measure this, as I don't have
any virtualization/x86 setup. I'm developing on ARM SoCs, where
virtualization hardware is coming, but not here yet.

> One wonders how the remote side knows enough to set this flag?

The remote side is a dedicated firmware loaded in order to boot the
other processor, so it really is intimate with the platform and its
requirements (and it publishes the virtio device features for every
supported virtio device).

>> Though I also wonder how big really is the perforamnce gain of d57ed95 ?
>
> Want to check and tell us?

Sorry, as I said, I have no way to do that... Any chance you did some
measurements back then when d57ed95 was introduced ?

I just guessed it did improve performance, otherwise you probably
wouldn't have bothered. But if not, then reverting it is surely
preferable to adding more complexity.

Thanks!
Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
>> Virtio is using memory barriers to control the ordering of
>> references to the vrings on SMP systems. When the guest is compiled
>> with SMP support, virtio is only using SMP barriers in order to
>> avoid incurring the overhead involved with mandatory barriers.
>>
>> Lately, though, virtio is being increasingly used with inter-processor
>> communication scenarios too, which involve running two (separate)
>> instances of operating systems on two (separate) processors, each of
>> which might either be UP or SMP.
>
> Is that using virtio-mmio?

No, I'm using this:

https://lkml.org/lkml/2011/10/25/139

> Sorry, could you pls explain what are 'two external processors'?
> I think I know that if two x86 CPUs in an SMP system run kernels built
> in an SMP configuration, smp_*mb barriers are enough.

Sure:

My setup is not SMP-based; it's two separate processors running in AMP
configuration. The processors have completely different architectures,
are not cache coherent, and only simply share some memory, which is
used for communications using virtio as the shared memory "wire"
protocol (i.e. we're not even doing virtualization: we have Linux on
one processor, and some RTOS on another processor, and they use virtio
to send and receive buffers).

So it's not SMP effects we're controlling; we're pretty much doing
MMIO and must use mandatory barriers (otherwise we see breakage).

> Is an extra branch faster or slower than reverting d57ed95?

Sorry, unfortunately I have no way to measure this, as I don't have
any virtualization/x86 setup. I'm developing on ARM SoCs, where
virtualization hardware is coming, but not here yet.

> One wonders how the remote side knows enough to set this flag?

The remote side is a dedicated firmware loaded in order to boot the
other processor, so it really is intimate with the platform and its
requirements (and it publishes the virtio device features for every
supported virtio device).

>> Though I also wonder how big really is the perforamnce gain of d57ed95 ?
>
> Want to check and tell us?

Sorry, as I said, I have no way to do that... Any chance you did some
measurements back then when d57ed95 was introduced ?

I just guessed it did improve performance, otherwise you probably
wouldn't have bothered. But if not, then reverting it is surely
preferable to adding more complexity.

Thanks!
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> >> Virtio is using memory barriers to control the ordering of
> >> references to the vrings on SMP systems. When the guest is compiled
> >> with SMP support, virtio is only using SMP barriers in order to
> >> avoid incurring the overhead involved with mandatory barriers.
> >>
> >> Lately, though, virtio is being increasingly used with inter-processor
> >> communication scenarios too, which involve running two (separate)
> >> instances of operating systems on two (separate) processors, each of
> >> which might either be UP or SMP.
> >
> > Is that using virtio-mmio?
> 
> No, I'm using this:
> 
> https://lkml.org/lkml/2011/10/25/139

This mentions iommu - is there a need to use dma api to let
the firmware acess the rings? Or does it have access to all
of memory?

> > Sorry, could you pls explain what are 'two external processors'?
> > I think I know that if two x86 CPUs in an SMP system run kernels built
> > in an SMP configuration, smp_*mb barriers are enough.
> 
> Sure:
> 
> My setup is not SMP-based; it's two separate processors running in AMP
> configuration. The processors have completely different architectures,
> are not cache coherent, and only simply share some memory, which is
> used for communications using virtio as the shared memory "wire"
> protocol (i.e. we're not even doing virtualization: we have Linux on
> one processor, and some RTOS on another processor, and they use virtio
> to send and receive buffers).

I'd like to make sure I understand the memory model some more.
Is there cache snooping? If yes access from an external device
typically works mostly in the same way as smp ...


> So it's not SMP effects we're controlling; we're pretty much doing
> MMIO and must use mandatory barriers

So you put virtio rings in MMIO memory?

> (otherwise we see breakage).

Could you please give a couple of examples of breakage?

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> >> Virtio is using memory barriers to control the ordering of
> >> references to the vrings on SMP systems. When the guest is compiled
> >> with SMP support, virtio is only using SMP barriers in order to
> >> avoid incurring the overhead involved with mandatory barriers.
> >>
> >> Lately, though, virtio is being increasingly used with inter-processor
> >> communication scenarios too, which involve running two (separate)
> >> instances of operating systems on two (separate) processors, each of
> >> which might either be UP or SMP.
> >
> > Is that using virtio-mmio?
> 
> No, I'm using this:
> 
> https://lkml.org/lkml/2011/10/25/139

This mentions iommu - is there a need to use dma api to let
the firmware acess the rings? Or does it have access to all
of memory?

> > Sorry, could you pls explain what are 'two external processors'?
> > I think I know that if two x86 CPUs in an SMP system run kernels built
> > in an SMP configuration, smp_*mb barriers are enough.
> 
> Sure:
> 
> My setup is not SMP-based; it's two separate processors running in AMP
> configuration. The processors have completely different architectures,
> are not cache coherent, and only simply share some memory, which is
> used for communications using virtio as the shared memory "wire"
> protocol (i.e. we're not even doing virtualization: we have Linux on
> one processor, and some RTOS on another processor, and they use virtio
> to send and receive buffers).

I'd like to make sure I understand the memory model some more.
Is there cache snooping? If yes access from an external device
typically works mostly in the same way as smp ...


> So it's not SMP effects we're controlling; we're pretty much doing
> MMIO and must use mandatory barriers

So you put virtio rings in MMIO memory?

> (otherwise we see breakage).

Could you please give a couple of examples of breakage?

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 3:11 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 02:31:26PM +0200, Ohad Ben-Cohen wrote:
> >> Virtio is using memory barriers to control the ordering of
> >> references to the vrings on SMP systems. When the guest is compiled
> >> with SMP support, virtio is only using SMP barriers in order to
> >> avoid incurring the overhead involved with mandatory barriers.
> >>
> >> Lately, though, virtio is being increasingly used with inter-processor
> >> communication scenarios too, which involve running two (separate)
> >> instances of operating systems on two (separate) processors, each of
> >> which might either be UP or SMP.
> >
> > Is that using virtio-mmio?
> 
> No, I'm using this:
> 
> https://lkml.org/lkml/2011/10/25/139

This mentions iommu - is there a need to use dma api to let
the firmware acess the rings? Or does it have access to all
of memory?

> > Sorry, could you pls explain what are 'two external processors'?
> > I think I know that if two x86 CPUs in an SMP system run kernels built
> > in an SMP configuration, smp_*mb barriers are enough.
> 
> Sure:
> 
> My setup is not SMP-based; it's two separate processors running in AMP
> configuration. The processors have completely different architectures,
> are not cache coherent, and only simply share some memory, which is
> used for communications using virtio as the shared memory "wire"
> protocol (i.e. we're not even doing virtualization: we have Linux on
> one processor, and some RTOS on another processor, and they use virtio
> to send and receive buffers).

I'd like to make sure I understand the memory model some more.
Is there cache snooping? If yes access from an external device
typically works mostly in the same way as smp ...


> So it's not SMP effects we're controlling; we're pretty much doing
> MMIO and must use mandatory barriers

So you put virtio rings in MMIO memory?

> (otherwise we see breakage).

Could you please give a couple of examples of breakage?

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> This mentions iommu - is there a need to use dma api to let
> the firmware acess the rings? Or does it have access to all
> of memory?

IOMMU may or may not be used, it really depends on the hardware (my
personal SoC does employ one, while others don't).

The vrings are created in non-cacheable memory, which is allocated
using dma_alloc_coherent, but that isn't necessarily controlling the
remote processor access to the memory (a notable example is an
iommu-less remote processor which can directly access the physical
memory).

> Is there cache snooping? If yes access from an external device
> typically works mostly in the same way as smp ...

No, nothing fancy like that. Every processor has its own cache, with
no coherency protocol. The remote processor should really be treated
as a device, and not as a processor that is part of an SMP
configuration, and we must prohibit both the compiler and the CPU from
reordering memory operations.

> So you put virtio rings in MMIO memory?

I'll be precise: the vrings are created in non-cacheable memory, which
both processors have access to.

> Could you please give a couple of examples of breakage?

Sure. Basically, the order of the vring memory operations appear
differently to the observing processor. For example, avail->idx gets
updated before the new entry is put in the available array...

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> This mentions iommu - is there a need to use dma api to let
> the firmware acess the rings? Or does it have access to all
> of memory?

IOMMU may or may not be used, it really depends on the hardware (my
personal SoC does employ one, while others don't).

The vrings are created in non-cacheable memory, which is allocated
using dma_alloc_coherent, but that isn't necessarily controlling the
remote processor access to the memory (a notable example is an
iommu-less remote processor which can directly access the physical
memory).

> Is there cache snooping? If yes access from an external device
> typically works mostly in the same way as smp ...

No, nothing fancy like that. Every processor has its own cache, with
no coherency protocol. The remote processor should really be treated
as a device, and not as a processor that is part of an SMP
configuration, and we must prohibit both the compiler and the CPU from
reordering memory operations.

> So you put virtio rings in MMIO memory?

I'll be precise: the vrings are created in non-cacheable memory, which
both processors have access to.

> Could you please give a couple of examples of breakage?

Sure. Basically, the order of the vring memory operations appear
differently to the observing processor. For example, avail->idx gets
updated before the new entry is put in the available array...

Thanks,
Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> This mentions iommu - is there a need to use dma api to let
> the firmware acess the rings? Or does it have access to all
> of memory?

IOMMU may or may not be used, it really depends on the hardware (my
personal SoC does employ one, while others don't).

The vrings are created in non-cacheable memory, which is allocated
using dma_alloc_coherent, but that isn't necessarily controlling the
remote processor access to the memory (a notable example is an
iommu-less remote processor which can directly access the physical
memory).

> Is there cache snooping? If yes access from an external device
> typically works mostly in the same way as smp ...

No, nothing fancy like that. Every processor has its own cache, with
no coherency protocol. The remote processor should really be treated
as a device, and not as a processor that is part of an SMP
configuration, and we must prohibit both the compiler and the CPU from
reordering memory operations.

> So you put virtio rings in MMIO memory?

I'll be precise: the vrings are created in non-cacheable memory, which
both processors have access to.

> Could you please give a couple of examples of breakage?

Sure. Basically, the order of the vring memory operations appear
differently to the observing processor. For example, avail->idx gets
updated before the new entry is put in the available array...

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:45:05PM +0200, Ohad Ben-Cohen wrote:
> > So you put virtio rings in MMIO memory?
> 
> I'll be precise: the vrings are created in non-cacheable memory, which
> both processors have access to.
> 
> > Could you please give a couple of examples of breakage?
> 
> Sure. Basically, the order of the vring memory operations appear
> differently to the observing processor. For example, avail->idx gets
> updated before the new entry is put in the available array...

I see. And this happens because the ARM processor reorders
memory writes to this uncacheable memory?
And in an SMP configuration, writes are somehow not reordered?

For example, if we had such an AMP configuration with and x86
processor, wmb() (sfence) would be wrong and smp_wmb() would be sufficient.

Just checking that this is not a bug in the smp_wmb implementation
for the specific platform.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:45:05PM +0200, Ohad Ben-Cohen wrote:
> > So you put virtio rings in MMIO memory?
> 
> I'll be precise: the vrings are created in non-cacheable memory, which
> both processors have access to.
> 
> > Could you please give a couple of examples of breakage?
> 
> Sure. Basically, the order of the vring memory operations appear
> differently to the observing processor. For example, avail->idx gets
> updated before the new entry is put in the available array...

I see. And this happens because the ARM processor reorders
memory writes to this uncacheable memory?
And in an SMP configuration, writes are somehow not reordered?

For example, if we had such an AMP configuration with and x86
processor, wmb() (sfence) would be wrong and smp_wmb() would be sufficient.

Just checking that this is not a bug in the smp_wmb implementation
for the specific platform.

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:45:05PM +0200, Ohad Ben-Cohen wrote:
> > So you put virtio rings in MMIO memory?
> 
> I'll be precise: the vrings are created in non-cacheable memory, which
> both processors have access to.
> 
> > Could you please give a couple of examples of breakage?
> 
> Sure. Basically, the order of the vring memory operations appear
> differently to the observing processor. For example, avail->idx gets
> updated before the new entry is put in the available array...

I see. And this happens because the ARM processor reorders
memory writes to this uncacheable memory?
And in an SMP configuration, writes are somehow not reordered?

For example, if we had such an AMP configuration with and x86
processor, wmb() (sfence) would be wrong and smp_wmb() would be sufficient.

Just checking that this is not a bug in the smp_wmb implementation
for the specific platform.

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> I see. And this happens because the ARM processor reorders
> memory writes

Yes.

> And in an SMP configuration, writes are somehow not reordered?

They are, but then the smp memory barriers are enough to control these
effects. It's not enough to control reordering as seen by a device
(which is what our AMP processors are) though.

(btw, the difference between an SMP processor and a device here lies
in how the memory is mapped: normal memory vs. device memory
attributes. it's an ARM thingy).

> Just checking that this is not a bug in the smp_wmb implementation
> for the specific platform.

No, it's not.

ARM's smp memory barriers use ARM's DMB instruction, which is enough
to control SMP effects, whereas ARM's mandatory memory barriers use
ARM's DSB instruction, which is required to ensure the ordering
between Device and Normal memory accesses.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> I see. And this happens because the ARM processor reorders
> memory writes

Yes.

> And in an SMP configuration, writes are somehow not reordered?

They are, but then the smp memory barriers are enough to control these
effects. It's not enough to control reordering as seen by a device
(which is what our AMP processors are) though.

(btw, the difference between an SMP processor and a device here lies
in how the memory is mapped: normal memory vs. device memory
attributes. it's an ARM thingy).

> Just checking that this is not a bug in the smp_wmb implementation
> for the specific platform.

No, it's not.

ARM's smp memory barriers use ARM's DMB instruction, which is enough
to control SMP effects, whereas ARM's mandatory memory barriers use
ARM's DSB instruction, which is required to ensure the ordering
between Device and Normal memory accesses.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 06:04:56PM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > I see. And this happens because the ARM processor reorders
> > memory writes
> 
> Yes.
> 
> > And in an SMP configuration, writes are somehow not reordered?
> 
> They are, but then the smp memory barriers are enough to control these
> effects. It's not enough to control reordering as seen by a device
> (which is what our AMP processors are) though.
> 
> (btw, the difference between an SMP processor and a device here lies
> in how the memory is mapped: normal memory vs. device memory
> attributes. it's an ARM thingy).

How are the rings mapped? normal memory, right?
We allocate them with plan alloc_pages_exact in virtio_pci.c ...

> > Just checking that this is not a bug in the smp_wmb implementation
> > for the specific platform.
> 
> No, it's not.
> 
> ARM's smp memory barriers use ARM's DMB instruction, which is enough
> to control SMP effects, whereas ARM's mandatory memory barriers use
> ARM's DSB instruction, which is required to ensure the ordering
> between Device and Normal memory accesses.
> 
> Thanks,
> Ohad.

Yes wmb() is required to ensure ordering for MMIO.
But here both accesses: index and ring - are for
memory, not MMIO.

I could understand ring kick bypassing index write, maybe ...
But you described an index write bypassing descriptor write.
Is this something you see in practice?


-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 06:04:56PM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > I see. And this happens because the ARM processor reorders
> > memory writes
> 
> Yes.
> 
> > And in an SMP configuration, writes are somehow not reordered?
> 
> They are, but then the smp memory barriers are enough to control these
> effects. It's not enough to control reordering as seen by a device
> (which is what our AMP processors are) though.
> 
> (btw, the difference between an SMP processor and a device here lies
> in how the memory is mapped: normal memory vs. device memory
> attributes. it's an ARM thingy).

How are the rings mapped? normal memory, right?
We allocate them with plan alloc_pages_exact in virtio_pci.c ...

> > Just checking that this is not a bug in the smp_wmb implementation
> > for the specific platform.
> 
> No, it's not.
> 
> ARM's smp memory barriers use ARM's DMB instruction, which is enough
> to control SMP effects, whereas ARM's mandatory memory barriers use
> ARM's DSB instruction, which is required to ensure the ordering
> between Device and Normal memory accesses.
> 
> Thanks,
> Ohad.

Yes wmb() is required to ensure ordering for MMIO.
But here both accesses: index and ring - are for
memory, not MMIO.

I could understand ring kick bypassing index write, maybe ...
But you described an index write bypassing descriptor write.
Is this something you see in practice?


-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 06:04:56PM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > I see. And this happens because the ARM processor reorders
> > memory writes
> 
> Yes.
> 
> > And in an SMP configuration, writes are somehow not reordered?
> 
> They are, but then the smp memory barriers are enough to control these
> effects. It's not enough to control reordering as seen by a device
> (which is what our AMP processors are) though.
> 
> (btw, the difference between an SMP processor and a device here lies
> in how the memory is mapped: normal memory vs. device memory
> attributes. it's an ARM thingy).

How are the rings mapped? normal memory, right?
We allocate them with plan alloc_pages_exact in virtio_pci.c ...

> > Just checking that this is not a bug in the smp_wmb implementation
> > for the specific platform.
> 
> No, it's not.
> 
> ARM's smp memory barriers use ARM's DMB instruction, which is enough
> to control SMP effects, whereas ARM's mandatory memory barriers use
> ARM's DSB instruction, which is required to ensure the ordering
> between Device and Normal memory accesses.
> 
> Thanks,
> Ohad.

Yes wmb() is required to ensure ordering for MMIO.
But here both accesses: index and ring - are for
memory, not MMIO.

I could understand ring kick bypassing index write, maybe ...
But you described an index write bypassing descriptor write.
Is this something you see in practice?


-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> How are the rings mapped? normal memory, right?

No, device memory.

> We allocate them with plan alloc_pages_exact in virtio_pci.c ...

I'm not using virtio_pci.c; remoteproc is allocating the rings using
the DMA API.

> Yes wmb() is required to ensure ordering for MMIO.
> But here both accesses: index and ring - are for
> memory, not MMIO.

I'm doing IO with a device over shared memory. It does require
mandatory barriers as I explained.

> Is this something you see in practice?

Yes. These bugs are very real.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> How are the rings mapped? normal memory, right?

No, device memory.

> We allocate them with plan alloc_pages_exact in virtio_pci.c ...

I'm not using virtio_pci.c; remoteproc is allocating the rings using
the DMA API.

> Yes wmb() is required to ensure ordering for MMIO.
> But here both accesses: index and ring - are for
> memory, not MMIO.

I'm doing IO with a device over shared memory. It does require
mandatory barriers as I explained.

> Is this something you see in practice?

Yes. These bugs are very real.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> How are the rings mapped? normal memory, right?

No, device memory.

> We allocate them with plan alloc_pages_exact in virtio_pci.c ...

I'm not using virtio_pci.c; remoteproc is allocating the rings using
the DMA API.

> Yes wmb() is required to ensure ordering for MMIO.
> But here both accesses: index and ring - are for
> memory, not MMIO.

I'm doing IO with a device over shared memory. It does require
mandatory barriers as I explained.

> Is this something you see in practice?

Yes. These bugs are very real.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> How are the rings mapped? normal memory, right?
>
> No, device memory.

Ok, I have more info.

Originally remoteproc was mapping the rings using ioremap, and that
meant ARM Device memory.

Recently, though, we moved to CMA (allocating memory for the rings via
dma_alloc_coherent), and that isn't Device memory anymore: it's
uncacheable Normal memory (on ARM v6+).

We still require mandatory barriers though: one very reproducible
problem I personally face is that the avail index doesn't get updated
before the kick. As a result, the remote processor misses a buffer
that was just added (the kick wakes it up only to find that the avail
index wasn't changed yet). In this case, it probably happens because
the mailbox, used to kick the remote processor, is mapped as Device
memory, and therefore the kick can be reordered before the updates to
the ring  can be observed.

I did get two additional reports about reordering issues, on different
setups than my own, and which I can't personally reproduce: the one
I've described earlier (avail index gets updated before the avail
array) and one in the receive path (reading a used buffer which we
already read). I couldn't personally verify those, but both issues
were reported to be gone when mandatory barriers were used.

I expect those reports only to increase: the diversity of platforms
that are now looking into adopting virtio for this kind of
inter-process communication is quite huge, with several different
architectures and even more hardware implementations on the way (not
only ARM).

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> How are the rings mapped? normal memory, right?
>
> No, device memory.

Ok, I have more info.

Originally remoteproc was mapping the rings using ioremap, and that
meant ARM Device memory.

Recently, though, we moved to CMA (allocating memory for the rings via
dma_alloc_coherent), and that isn't Device memory anymore: it's
uncacheable Normal memory (on ARM v6+).

We still require mandatory barriers though: one very reproducible
problem I personally face is that the avail index doesn't get updated
before the kick. As a result, the remote processor misses a buffer
that was just added (the kick wakes it up only to find that the avail
index wasn't changed yet). In this case, it probably happens because
the mailbox, used to kick the remote processor, is mapped as Device
memory, and therefore the kick can be reordered before the updates to
the ring  can be observed.

I did get two additional reports about reordering issues, on different
setups than my own, and which I can't personally reproduce: the one
I've described earlier (avail index gets updated before the avail
array) and one in the receive path (reading a used buffer which we
already read). I couldn't personally verify those, but both issues
were reported to be gone when mandatory barriers were used.

I expect those reports only to increase: the diversity of platforms
that are now looking into adopting virtio for this kind of
inter-process communication is quite huge, with several different
architectures and even more hardware implementations on the way (not
only ARM).

Thanks,
Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> How are the rings mapped? normal memory, right?
>
> No, device memory.

Ok, I have more info.

Originally remoteproc was mapping the rings using ioremap, and that
meant ARM Device memory.

Recently, though, we moved to CMA (allocating memory for the rings via
dma_alloc_coherent), and that isn't Device memory anymore: it's
uncacheable Normal memory (on ARM v6+).

We still require mandatory barriers though: one very reproducible
problem I personally face is that the avail index doesn't get updated
before the kick. As a result, the remote processor misses a buffer
that was just added (the kick wakes it up only to find that the avail
index wasn't changed yet). In this case, it probably happens because
the mailbox, used to kick the remote processor, is mapped as Device
memory, and therefore the kick can be reordered before the updates to
the ring  can be observed.

I did get two additional reports about reordering issues, on different
setups than my own, and which I can't personally reproduce: the one
I've described earlier (avail index gets updated before the avail
array) and one in the receive path (reading a used buffer which we
already read). I couldn't personally verify those, but both issues
were reported to be gone when mandatory barriers were used.

I expect those reports only to increase: the diversity of platforms
that are now looking into adopting virtio for this kind of
inter-process communication is quite huge, with several different
architectures and even more hardware implementations on the way (not
only ARM).

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 01:27:10AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> > On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> >> How are the rings mapped? normal memory, right?
> >
> > No, device memory.
> 
> Ok, I have more info.
> 
> Originally remoteproc was mapping the rings using ioremap, and that
> meant ARM Device memory.
> 
> Recently, though, we moved to CMA (allocating memory for the rings via
> dma_alloc_coherent), and that isn't Device memory anymore: it's
> uncacheable Normal memory (on ARM v6+).

And these accesses need to be ordered with DSB? Or DMB?

> We still require mandatory barriers though: one very reproducible
> problem I personally face is that the avail index doesn't get updated
> before the kick.

Aha! The *kick* really is MMIO. So I think we do need a mandatory barrier
before the kick.  Maybe we need it for virtio-pci as well
(not on kvm, naturally :) Off-hand this seems to belong in the transport
layer but need to think about it.

> As a result, the remote processor misses a buffer
> that was just added (the kick wakes it up only to find that the avail
> index wasn't changed yet). In this case, it probably happens because
> the mailbox, used to kick the remote processor, is mapped as Device
> memory, and therefore the kick can be reordered before the updates to
> the ring  can be observed.
> 
> I did get two additional reports about reordering issues, on different
> setups than my own, and which I can't personally reproduce: the one
> I've described earlier (avail index gets updated before the avail
> array) and one in the receive path (reading a used buffer which we
> already read). I couldn't personally verify those, but both issues
> were reported to be gone when mandatory barriers were used.

Hmm. So it's a hint that something is wrong with memory
but not what's wrong exactly.

> I expect those reports only to increase: the diversity of platforms
> that are now looking into adopting virtio for this kind of
> inter-process communication is quite huge, with several different
> architectures and even more hardware implementations on the way (not
> only ARM).
> 
> Thanks,
> Ohad.

Right. We need to be very careful with memory,
it's a tricky field. One known problem with virtio
is its insistance on using native endian-ness
for some fields. If power is used, we'll have to fix this.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 01:27:10AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> > On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> >> How are the rings mapped? normal memory, right?
> >
> > No, device memory.
> 
> Ok, I have more info.
> 
> Originally remoteproc was mapping the rings using ioremap, and that
> meant ARM Device memory.
> 
> Recently, though, we moved to CMA (allocating memory for the rings via
> dma_alloc_coherent), and that isn't Device memory anymore: it's
> uncacheable Normal memory (on ARM v6+).

And these accesses need to be ordered with DSB? Or DMB?

> We still require mandatory barriers though: one very reproducible
> problem I personally face is that the avail index doesn't get updated
> before the kick.

Aha! The *kick* really is MMIO. So I think we do need a mandatory barrier
before the kick.  Maybe we need it for virtio-pci as well
(not on kvm, naturally :) Off-hand this seems to belong in the transport
layer but need to think about it.

> As a result, the remote processor misses a buffer
> that was just added (the kick wakes it up only to find that the avail
> index wasn't changed yet). In this case, it probably happens because
> the mailbox, used to kick the remote processor, is mapped as Device
> memory, and therefore the kick can be reordered before the updates to
> the ring  can be observed.
> 
> I did get two additional reports about reordering issues, on different
> setups than my own, and which I can't personally reproduce: the one
> I've described earlier (avail index gets updated before the avail
> array) and one in the receive path (reading a used buffer which we
> already read). I couldn't personally verify those, but both issues
> were reported to be gone when mandatory barriers were used.

Hmm. So it's a hint that something is wrong with memory
but not what's wrong exactly.

> I expect those reports only to increase: the diversity of platforms
> that are now looking into adopting virtio for this kind of
> inter-process communication is quite huge, with several different
> architectures and even more hardware implementations on the way (not
> only ARM).
> 
> Thanks,
> Ohad.

Right. We need to be very careful with memory,
it's a tricky field. One known problem with virtio
is its insistance on using native endian-ness
for some fields. If power is used, we'll have to fix this.

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 01:27:10AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 6:24 PM, Ohad Ben-Cohen <ohad@wizery.com> wrote:
> > On Wed, Nov 30, 2011 at 6:15 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> >> How are the rings mapped? normal memory, right?
> >
> > No, device memory.
> 
> Ok, I have more info.
> 
> Originally remoteproc was mapping the rings using ioremap, and that
> meant ARM Device memory.
> 
> Recently, though, we moved to CMA (allocating memory for the rings via
> dma_alloc_coherent), and that isn't Device memory anymore: it's
> uncacheable Normal memory (on ARM v6+).

And these accesses need to be ordered with DSB? Or DMB?

> We still require mandatory barriers though: one very reproducible
> problem I personally face is that the avail index doesn't get updated
> before the kick.

Aha! The *kick* really is MMIO. So I think we do need a mandatory barrier
before the kick.  Maybe we need it for virtio-pci as well
(not on kvm, naturally :) Off-hand this seems to belong in the transport
layer but need to think about it.

> As a result, the remote processor misses a buffer
> that was just added (the kick wakes it up only to find that the avail
> index wasn't changed yet). In this case, it probably happens because
> the mailbox, used to kick the remote processor, is mapped as Device
> memory, and therefore the kick can be reordered before the updates to
> the ring  can be observed.
> 
> I did get two additional reports about reordering issues, on different
> setups than my own, and which I can't personally reproduce: the one
> I've described earlier (avail index gets updated before the avail
> array) and one in the receive path (reading a used buffer which we
> already read). I couldn't personally verify those, but both issues
> were reported to be gone when mandatory barriers were used.

Hmm. So it's a hint that something is wrong with memory
but not what's wrong exactly.

> I expect those reports only to increase: the diversity of platforms
> that are now looking into adopting virtio for this kind of
> inter-process communication is quite huge, with several different
> architectures and even more hardware implementations on the way (not
> only ARM).
> 
> Thanks,
> Ohad.

Right. We need to be very careful with memory,
it's a tricky field. One known problem with virtio
is its insistance on using native endian-ness
for some fields. If power is used, we'll have to fix this.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:43 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> And these accesses need to be ordered with DSB? Or DMB?

DMB (i.e. smp barriers) should be enough within Normal memory
accesses, though the other issues that were reported to me are a bit
concerning. I'm still trying to get more information about them, in
the hopes that I can eventually reproduce them myself.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:43 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> And these accesses need to be ordered with DSB? Or DMB?

DMB (i.e. smp barriers) should be enough within Normal memory
accesses, though the other issues that were reported to me are a bit
concerning. I'm still trying to get more information about them, in
the hopes that I can eventually reproduce them myself.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:43 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> And these accesses need to be ordered with DSB? Or DMB?

DMB (i.e. smp barriers) should be enough within Normal memory
accesses, though the other issues that were reported to me are a bit
concerning. I'm still trying to get more information about them, in
the hopes that I can eventually reproduce them myself.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:59 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> I see. And this happens because the ARM processor reorders
> memory writes

Yes.

> And in an SMP configuration, writes are somehow not reordered?

They are, but then the smp memory barriers are enough to control these
effects. It's not enough to control reordering as seen by a device
(which is what our AMP processors are) though.

(btw, the difference between an SMP processor and a device here lies
in how the memory is mapped: normal memory vs. device memory
attributes. it's an ARM thingy).

> Just checking that this is not a bug in the smp_wmb implementation
> for the specific platform.

No, it's not.

ARM's smp memory barriers use ARM's DMB instruction, which is enough
to control SMP effects, whereas ARM's mandatory memory barriers use
ARM's DSB instruction, which is required to ensure the ordering
between Device and Normal memory accesses.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> > Is an extra branch faster or slower than reverting d57ed95?
> 
> Sorry, unfortunately I have no way to measure this, as I don't have
> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> virtualization hardware is coming, but not here yet.

You can try using the micro-benchmark in tools/virtio/.
That does not need virtualization.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> > Is an extra branch faster or slower than reverting d57ed95?
> 
> Sorry, unfortunately I have no way to measure this, as I don't have
> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> virtualization hardware is coming, but not here yet.

You can try using the micro-benchmark in tools/virtio/.
That does not need virtualization.

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> > Is an extra branch faster or slower than reverting d57ed95?
> 
> Sorry, unfortunately I have no way to measure this, as I don't have
> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> virtualization hardware is coming, but not here yet.

You can try using the micro-benchmark in tools/virtio/.
That does not need virtualization.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
>> > Is an extra branch faster or slower than reverting d57ed95?
>>
>> Sorry, unfortunately I have no way to measure this, as I don't have
>> any virtualization/x86 setup. I'm developing on ARM SoCs, where
>> virtualization hardware is coming, but not here yet.
>
> You can try using the micro-benchmark in tools/virtio/.

Hmm, care to show me exactly what do you mean ?

Though I somewhat suspect that any micro-benchmarking I'll do with my
random ARM SoC will not have much value to real virtualization/x86
workloads.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
>> > Is an extra branch faster or slower than reverting d57ed95?
>>
>> Sorry, unfortunately I have no way to measure this, as I don't have
>> any virtualization/x86 setup. I'm developing on ARM SoCs, where
>> virtualization hardware is coming, but not here yet.
>
> You can try using the micro-benchmark in tools/virtio/.

Hmm, care to show me exactly what do you mean ?

Though I somewhat suspect that any micro-benchmarking I'll do with my
random ARM SoC will not have much value to real virtualization/x86
workloads.

Thanks,
Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
>> > Is an extra branch faster or slower than reverting d57ed95?
>>
>> Sorry, unfortunately I have no way to measure this, as I don't have
>> any virtualization/x86 setup. I'm developing on ARM SoCs, where
>> virtualization hardware is coming, but not here yet.
>
> You can try using the micro-benchmark in tools/virtio/.

Hmm, care to show me exactly what do you mean ?

Though I somewhat suspect that any micro-benchmarking I'll do with my
random ARM SoC will not have much value to real virtualization/x86
workloads.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:55:53PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> >> > Is an extra branch faster or slower than reverting d57ed95?
> >>
> >> Sorry, unfortunately I have no way to measure this, as I don't have
> >> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> >> virtualization hardware is coming, but not here yet.
> >
> > You can try using the micro-benchmark in tools/virtio/.
> 
> Hmm, care to show me exactly what do you mean ?

make headers_install
make -C tools/virtio/
(you'll need an empty stub for tools/virtio/linux/module.h,
 I just sent a patch to add that)
sudo insmod tools/virtio/vhost_test/vhost_test.ko
./tools/virtio/virtio_test

> Though I somewhat suspect that any micro-benchmarking I'll do with my
> random ARM SoC will not have much value to real virtualization/x86
> workloads.
> 
> Thanks,
> Ohad.

Real virtualization/x86 can keep using current smp_XX barriers, right?
We can have some config for your kind of setup.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:55:53PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> >> > Is an extra branch faster or slower than reverting d57ed95?
> >>
> >> Sorry, unfortunately I have no way to measure this, as I don't have
> >> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> >> virtualization hardware is coming, but not here yet.
> >
> > You can try using the micro-benchmark in tools/virtio/.
> 
> Hmm, care to show me exactly what do you mean ?

make headers_install
make -C tools/virtio/
(you'll need an empty stub for tools/virtio/linux/module.h,
 I just sent a patch to add that)
sudo insmod tools/virtio/vhost_test/vhost_test.ko
./tools/virtio/virtio_test

> Though I somewhat suspect that any micro-benchmarking I'll do with my
> random ARM SoC will not have much value to real virtualization/x86
> workloads.
> 
> Thanks,
> Ohad.

Real virtualization/x86 can keep using current smp_XX barriers, right?
We can have some config for your kind of setup.

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Wed, Nov 30, 2011 at 01:55:53PM +0200, Ohad Ben-Cohen wrote:
> On Tue, Nov 29, 2011 at 5:19 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Nov 29, 2011 at 03:57:19PM +0200, Ohad Ben-Cohen wrote:
> >> > Is an extra branch faster or slower than reverting d57ed95?
> >>
> >> Sorry, unfortunately I have no way to measure this, as I don't have
> >> any virtualization/x86 setup. I'm developing on ARM SoCs, where
> >> virtualization hardware is coming, but not here yet.
> >
> > You can try using the micro-benchmark in tools/virtio/.
> 
> Hmm, care to show me exactly what do you mean ?

make headers_install
make -C tools/virtio/
(you'll need an empty stub for tools/virtio/linux/module.h,
 I just sent a patch to add that)
sudo insmod tools/virtio/vhost_test/vhost_test.ko
./tools/virtio/virtio_test

> Though I somewhat suspect that any micro-benchmarking I'll do with my
> random ARM SoC will not have much value to real virtualization/x86
> workloads.
> 
> Thanks,
> Ohad.

Real virtualization/x86 can keep using current smp_XX barriers, right?
We can have some config for your kind of setup.

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> make headers_install
> make -C tools/virtio/
> (you'll need an empty stub for tools/virtio/linux/module.h,
>  I just sent a patch to add that)
> sudo insmod tools/virtio/vhost_test/vhost_test.ko
> ./tools/virtio/virtio_test

Ok, I gave this a spin.

I've tried to see if reverting d57ed95 has any measurable effect on
the execution time of virtio_test's run_test(), but I couldn't see any
(several attempts with and without d57ed95 yielded very similar range
of execution times).

YMMV though, especially with real workloads.

> Real virtualization/x86 can keep using current smp_XX barriers, right?

Yes, sure. ARM virtualization can too, since smp_XX barriers are
enough for that scenario.

> We can have some config for your kind of setup.

Please note that it can't be a compile-time decision though (unless
we're willing to effectively revert d57ed95 when this config kicks
in): it's not unlikely that one would want to have both use cases
running on the same time.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> make headers_install
> make -C tools/virtio/
> (you'll need an empty stub for tools/virtio/linux/module.h,
>  I just sent a patch to add that)
> sudo insmod tools/virtio/vhost_test/vhost_test.ko
> ./tools/virtio/virtio_test

Ok, I gave this a spin.

I've tried to see if reverting d57ed95 has any measurable effect on
the execution time of virtio_test's run_test(), but I couldn't see any
(several attempts with and without d57ed95 yielded very similar range
of execution times).

YMMV though, especially with real workloads.

> Real virtualization/x86 can keep using current smp_XX barriers, right?

Yes, sure. ARM virtualization can too, since smp_XX barriers are
enough for that scenario.

> We can have some config for your kind of setup.

Please note that it can't be a compile-time decision though (unless
we're willing to effectively revert d57ed95 when this config kicks
in): it's not unlikely that one would want to have both use cases
running on the same time.

Thanks,
Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> make headers_install
> make -C tools/virtio/
> (you'll need an empty stub for tools/virtio/linux/module.h,
> ?I just sent a patch to add that)
> sudo insmod tools/virtio/vhost_test/vhost_test.ko
> ./tools/virtio/virtio_test

Ok, I gave this a spin.

I've tried to see if reverting d57ed95 has any measurable effect on
the execution time of virtio_test's run_test(), but I couldn't see any
(several attempts with and without d57ed95 yielded very similar range
of execution times).

YMMV though, especially with real workloads.

> Real virtualization/x86 can keep using current smp_XX barriers, right?

Yes, sure. ARM virtualization can too, since smp_XX barriers are
enough for that scenario.

> We can have some config for your kind of setup.

Please note that it can't be a compile-time decision though (unless
we're willing to effectively revert d57ed95 when this config kicks
in): it's not unlikely that one would want to have both use cases
running on the same time.

Thanks,
Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:43:08AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > make headers_install
> > make -C tools/virtio/
> > (you'll need an empty stub for tools/virtio/linux/module.h,
> >  I just sent a patch to add that)
> > sudo insmod tools/virtio/vhost_test/vhost_test.ko
> > ./tools/virtio/virtio_test
> 
> Ok, I gave this a spin.
> 
> I've tried to see if reverting d57ed95 has any measurable effect on
> the execution time of virtio_test's run_test(), but I couldn't see any
> (several attempts with and without d57ed95 yielded very similar range
> of execution times).
> 
> YMMV though, especially with real workloads.
> 
> > Real virtualization/x86 can keep using current smp_XX barriers, right?
> 
> Yes, sure. ARM virtualization can too, since smp_XX barriers are
> enough for that scenario.
> 
> > We can have some config for your kind of setup.
> 
> Please note that it can't be a compile-time decision though (unless
> we're willing to effectively revert d57ed95 when this config kicks
> in): it's not unlikely that one would want to have both use cases
> running on the same time.
> 
> Thanks,
> Ohad.

For x86, stores into memory are ordered. So I think that yes, smp_XXX
can be selected at compile time.

So let's forget the virtio strangeness for a minute,

To me it starts looking like we need some new kind of barrier
that handles accesses to DMA coherent memory. dma_Xmb()?
dma_coherent_Xmb()?  For example, on x86, dma_wmb() can be barrier(),
but on your system it needs to do DSB.

We can set the rule that dma barriers are guaranteed stronger
than smp ones, and we can just use dma_ everywhere.
So the strength will be:

smp < dma < mandatory

And now virtio can use DMA barriers and instead of adding
overhead for x86, x86 will actually gain from this,
as we'll drop mandatory barriers on UP systems.

Hmm?

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:43:08AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > make headers_install
> > make -C tools/virtio/
> > (you'll need an empty stub for tools/virtio/linux/module.h,
> >  I just sent a patch to add that)
> > sudo insmod tools/virtio/vhost_test/vhost_test.ko
> > ./tools/virtio/virtio_test
> 
> Ok, I gave this a spin.
> 
> I've tried to see if reverting d57ed95 has any measurable effect on
> the execution time of virtio_test's run_test(), but I couldn't see any
> (several attempts with and without d57ed95 yielded very similar range
> of execution times).
> 
> YMMV though, especially with real workloads.
> 
> > Real virtualization/x86 can keep using current smp_XX barriers, right?
> 
> Yes, sure. ARM virtualization can too, since smp_XX barriers are
> enough for that scenario.
> 
> > We can have some config for your kind of setup.
> 
> Please note that it can't be a compile-time decision though (unless
> we're willing to effectively revert d57ed95 when this config kicks
> in): it's not unlikely that one would want to have both use cases
> running on the same time.
> 
> Thanks,
> Ohad.

For x86, stores into memory are ordered. So I think that yes, smp_XXX
can be selected at compile time.

So let's forget the virtio strangeness for a minute,

To me it starts looking like we need some new kind of barrier
that handles accesses to DMA coherent memory. dma_Xmb()?
dma_coherent_Xmb()?  For example, on x86, dma_wmb() can be barrier(),
but on your system it needs to do DSB.

We can set the rule that dma barriers are guaranteed stronger
than smp ones, and we can just use dma_ everywhere.
So the strength will be:

smp < dma < mandatory

And now virtio can use DMA barriers and instead of adding
overhead for x86, x86 will actually gain from this,
as we'll drop mandatory barriers on UP systems.

Hmm?

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:43:08AM +0200, Ohad Ben-Cohen wrote:
> On Wed, Nov 30, 2011 at 4:50 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > make headers_install
> > make -C tools/virtio/
> > (you'll need an empty stub for tools/virtio/linux/module.h,
> > ?I just sent a patch to add that)
> > sudo insmod tools/virtio/vhost_test/vhost_test.ko
> > ./tools/virtio/virtio_test
> 
> Ok, I gave this a spin.
> 
> I've tried to see if reverting d57ed95 has any measurable effect on
> the execution time of virtio_test's run_test(), but I couldn't see any
> (several attempts with and without d57ed95 yielded very similar range
> of execution times).
> 
> YMMV though, especially with real workloads.
> 
> > Real virtualization/x86 can keep using current smp_XX barriers, right?
> 
> Yes, sure. ARM virtualization can too, since smp_XX barriers are
> enough for that scenario.
> 
> > We can have some config for your kind of setup.
> 
> Please note that it can't be a compile-time decision though (unless
> we're willing to effectively revert d57ed95 when this config kicks
> in): it's not unlikely that one would want to have both use cases
> running on the same time.
> 
> Thanks,
> Ohad.

For x86, stores into memory are ordered. So I think that yes, smp_XXX
can be selected at compile time.

So let's forget the virtio strangeness for a minute,

To me it starts looking like we need some new kind of barrier
that handles accesses to DMA coherent memory. dma_Xmb()?
dma_coherent_Xmb()?  For example, on x86, dma_wmb() can be barrier(),
but on your system it needs to do DSB.

We can set the rule that dma barriers are guaranteed stronger
than smp ones, and we can just use dma_ everywhere.
So the strength will be:

smp < dma < mandatory

And now virtio can use DMA barriers and instead of adding
overhead for x86, x86 will actually gain from this,
as we'll drop mandatory barriers on UP systems.

Hmm?

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.
> 
> So let's forget the virtio strangeness for a minute,

Hmm, we got away with light barriers because we knew we were not
*really* talking to a device.  But now with virtio-mmio, turns out we
are :)

I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
optimization later if it proves worthwhile.

Thoughts?
Rusty. 

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.
> 
> So let's forget the virtio strangeness for a minute,

Hmm, we got away with light barriers because we knew we were not
*really* talking to a device.  But now with virtio-mmio, turns out we
are :)

I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
optimization later if it proves worthwhile.

Thoughts?
Rusty. 

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.
> 
> So let's forget the virtio strangeness for a minute,

Hmm, we got away with light barriers because we knew we were not
*really* talking to a device.  But now with virtio-mmio, turns out we
are :)

I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
optimization later if it proves worthwhile.

Thoughts?
Rusty. 

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 4:28 AM, Rusty Russell <rusty@rustcorp.com.au> wrote:
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device.  But now with virtio-mmio, turns out we
> are :)
>
> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.

+1

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 4:28 AM, Rusty Russell <rusty@rustcorp.com.au> wrote:
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device.  But now with virtio-mmio, turns out we
> are :)
>
> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.

+1

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 4:28 AM, Rusty Russell <rusty@rustcorp.com.au> wrote:
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device. ?But now with virtio-mmio, turns out we
> are :)
>
> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.

+1

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> > 
> > So let's forget the virtio strangeness for a minute,
> 
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device.  But now with virtio-mmio, turns out we
> are :)

You think virtio-mmio this issue too?  It's reported on remoteproc...

> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.
> 
> Thoughts?
> Rusty. 

Generally it does seem the best we can do for 3.2.

Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
to find some real setups (as in - not my laptop) to run some benchmarks
on, to verify there's no major problem.
I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Further, if we do revert, need to remember to apply the following
beforehand, to avoid breaking virtio tool:

tools/virtio: implement mandatory barriers for x86

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 68b8b8d..1bf0e80 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -172,11 +172,18 @@ struct virtqueue {
 #define MODULE_LICENSE(__MODULE_LICENSE_value) \
 	const char *__MODULE_LICENSE_name = __MODULE_LICENSE_value
 
 #define CONFIG_SMP
 
 #if defined(__i386__) || defined(__x86_64__)
 #define barrier() asm volatile("" ::: "memory")
 #define mb() __sync_synchronize()
+#if defined(__i386__)
+#define wmb() mb()
+#define rmb() mb()
+#else
+#define wmb() asm volatile("sfence" ::: "memory")
+#define rmb() asm volatile("lfence" ::: "memory")
+#endif
 
 #define smp_mb()	mb()
 # define smp_rmb()	barrier()

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> > 
> > So let's forget the virtio strangeness for a minute,
> 
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device.  But now with virtio-mmio, turns out we
> are :)

You think virtio-mmio this issue too?  It's reported on remoteproc...

> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.
> 
> Thoughts?
> Rusty. 

Generally it does seem the best we can do for 3.2.

Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
to find some real setups (as in - not my laptop) to run some benchmarks
on, to verify there's no major problem.
I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Further, if we do revert, need to remember to apply the following
beforehand, to avoid breaking virtio tool:

tools/virtio: implement mandatory barriers for x86

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 68b8b8d..1bf0e80 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -172,11 +172,18 @@ struct virtqueue {
 #define MODULE_LICENSE(__MODULE_LICENSE_value) \
 	const char *__MODULE_LICENSE_name = __MODULE_LICENSE_value
 
 #define CONFIG_SMP
 
 #if defined(__i386__) || defined(__x86_64__)
 #define barrier() asm volatile("" ::: "memory")
 #define mb() __sync_synchronize()
+#if defined(__i386__)
+#define wmb() mb()
+#define rmb() mb()
+#else
+#define wmb() asm volatile("sfence" ::: "memory")
+#define rmb() asm volatile("lfence" ::: "memory")
+#endif
 
 #define smp_mb()	mb()
 # define smp_rmb()	barrier()

-- 
MST

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> > 
> > So let's forget the virtio strangeness for a minute,
> 
> Hmm, we got away with light barriers because we knew we were not
> *really* talking to a device.  But now with virtio-mmio, turns out we
> are :)

You think virtio-mmio this issue too?  It's reported on remoteproc...

> I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> optimization later if it proves worthwhile.
> 
> Thoughts?
> Rusty. 

Generally it does seem the best we can do for 3.2.

Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
to find some real setups (as in - not my laptop) to run some benchmarks
on, to verify there's no major problem.
I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Further, if we do revert, need to remember to apply the following
beforehand, to avoid breaking virtio tool:

tools/virtio: implement mandatory barriers for x86

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 68b8b8d..1bf0e80 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -172,11 +172,18 @@ struct virtqueue {
 #define MODULE_LICENSE(__MODULE_LICENSE_value) \
 	const char *__MODULE_LICENSE_name = __MODULE_LICENSE_value
 
 #define CONFIG_SMP
 
 #if defined(__i386__) || defined(__x86_64__)
 #define barrier() asm volatile("" ::: "memory")
 #define mb() __sync_synchronize()
+#if defined(__i386__)
+#define wmb() mb()
+#define rmb() mb()
+#else
+#define wmb() asm volatile("sfence" ::: "memory")
+#define rmb() asm volatile("lfence" ::: "memory")
+#endif
 
 #define smp_mb()	mb()
 # define smp_rmb()	barrier()

-- 
MST

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 10:12:37 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> > On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > > can be selected at compile time.
> > > 
> > > So let's forget the virtio strangeness for a minute,
> > 
> > Hmm, we got away with light barriers because we knew we were not
> > *really* talking to a device.  But now with virtio-mmio, turns out we
> > are :)
> 
> You think virtio-mmio this issue too?  It's reported on remoteproc...

I think any non-virtual, non-PCI device has to worry about it.  Perhaps
all virtio-mmio are virtual (at this point).

I'm tempted to say we want permission from the device to do relaxed
barriers (so I don't have to worry about it!)

> > I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> > optimization later if it proves worthwhile.
> 
> Generally it does seem the best we can do for 3.2.
> 
> Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
> to find some real setups (as in - not my laptop) to run some benchmarks
> on, to verify there's no major problem.
> I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Yep, no huge hurry.  Thanks!

Cheers,
Rusty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 10:12:37 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> > On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > > can be selected at compile time.
> > > 
> > > So let's forget the virtio strangeness for a minute,
> > 
> > Hmm, we got away with light barriers because we knew we were not
> > *really* talking to a device.  But now with virtio-mmio, turns out we
> > are :)
> 
> You think virtio-mmio this issue too?  It's reported on remoteproc...

I think any non-virtual, non-PCI device has to worry about it.  Perhaps
all virtio-mmio are virtual (at this point).

I'm tempted to say we want permission from the device to do relaxed
barriers (so I don't have to worry about it!)

> > I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> > optimization later if it proves worthwhile.
> 
> Generally it does seem the best we can do for 3.2.
> 
> Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
> to find some real setups (as in - not my laptop) to run some benchmarks
> on, to verify there's no major problem.
> I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Yep, no huge hurry.  Thanks!

Cheers,
Rusty.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Thu, 1 Dec 2011 10:12:37 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Dec 01, 2011 at 12:58:59PM +1030, Rusty Russell wrote:
> > On Thu, 1 Dec 2011 01:13:07 +0200, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > > can be selected at compile time.
> > > 
> > > So let's forget the virtio strangeness for a minute,
> > 
> > Hmm, we got away with light barriers because we knew we were not
> > *really* talking to a device.  But now with virtio-mmio, turns out we
> > are :)
> 
> You think virtio-mmio this issue too?  It's reported on remoteproc...

I think any non-virtual, non-PCI device has to worry about it.  Perhaps
all virtio-mmio are virtual (at this point).

I'm tempted to say we want permission from the device to do relaxed
barriers (so I don't have to worry about it!)

> > I'm really tempted to revert d57ed95 for 3.2, and we can revisit this
> > optimization later if it proves worthwhile.
> 
> Generally it does seem the best we can do for 3.2.
> 
> Given it's rc3, I'd be a bit wary of introducing regressions - I'll try
> to find some real setups (as in - not my laptop) to run some benchmarks
> on, to verify there's no major problem.
> I hope I can report on this in about a week from now - want to hold onto this meanwhile?

Yep, no huge hurry.  Thanks!

Cheers,
Rusty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.

But then you can't use the same kernel image for both scenarios.

It won't take long until people will use virtio on ARM for both
virtualization and for talking to devices, and having to rebuild the
kernel for different use cases is nasty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.

But then you can't use the same kernel image for both scenarios.

It won't take long until people will use virtio on ARM for both
virtualization and for talking to devices, and having to rebuild the
kernel for different use cases is nasty.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> For x86, stores into memory are ordered. So I think that yes, smp_XXX
> can be selected at compile time.

But then you can't use the same kernel image for both scenarios.

It won't take long until people will use virtio on ARM for both
virtualization and for talking to devices, and having to rebuild the
kernel for different use cases is nasty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 08:14:26AM +0200, Ohad Ben-Cohen wrote:
> On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> 
> But then you can't use the same kernel image for both scenarios.

I was talking about virtio-pci. That always allocates the ring
in the normal memory.

> It won't take long until people will use virtio on ARM for both
> virtualization and for talking to devices, and having to rebuild the
> kernel for different use cases is nasty.

Yes, I understand that it's nasty.

> _______________________________________________
> Virtualization mailing list
> Virtualization@lists.linux-foundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 08:14:26AM +0200, Ohad Ben-Cohen wrote:
> On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> 
> But then you can't use the same kernel image for both scenarios.

I was talking about virtio-pci. That always allocates the ring
in the normal memory.

> It won't take long until people will use virtio on ARM for both
> virtualization and for talking to devices, and having to rebuild the
> kernel for different use cases is nasty.

Yes, I understand that it's nasty.

> _______________________________________________
> Virtualization mailing list
> Virtualization@lists.linux-foundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/virtualization

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Thu, Dec 01, 2011 at 08:14:26AM +0200, Ohad Ben-Cohen wrote:
> On Thu, Dec 1, 2011 at 1:13 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > For x86, stores into memory are ordered. So I think that yes, smp_XXX
> > can be selected at compile time.
> 
> But then you can't use the same kernel image for both scenarios.

I was talking about virtio-pci. That always allocates the ring
in the normal memory.

> It won't take long until people will use virtio on ARM for both
> virtualization and for talking to devices, and having to rebuild the
> kernel for different use cases is nasty.

Yes, I understand that it's nasty.

> _______________________________________________
> Virtualization mailing list
> Virtualization at lists.linux-foundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.
> 
> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.
> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.

Have you measured the impact of using normal barriers (non-SMP ones)
like we use on normal HW drivers unconditionally ?

IE. If the difference is small enough I'd say just go for it and avoid
the bloat.

Ben.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.
> 
> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.
> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.

Have you measured the impact of using normal barriers (non-SMP ones)
like we use on normal HW drivers unconditionally ?

IE. If the difference is small enough I'd say just go for it and avoid
the bloat.

Ben.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> Virtio is using memory barriers to control the ordering of
> references to the vrings on SMP systems. When the guest is compiled
> with SMP support, virtio is only using SMP barriers in order to
> avoid incurring the overhead involved with mandatory barriers.
> 
> Lately, though, virtio is being increasingly used with inter-processor
> communication scenarios too, which involve running two (separate)
> instances of operating systems on two (separate) processors, each of
> which might either be UP or SMP.
> 
> To control the ordering of memory references when the vrings are shared
> between two external processors, we must always use mandatory barriers.
> 
> A trivial, albeit sub-optimal, solution would be to simply revert
> commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> that's going to have a negative impact on performance of SMP-based
> virtualization use cases.

Have you measured the impact of using normal barriers (non-SMP ones)
like we use on normal HW drivers unconditionally ?

IE. If the difference is small enough I'd say just go for it and avoid
the bloat.

Ben.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > A trivial, albeit sub-optimal, solution would be to simply revert
> > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > that's going to have a negative impact on performance of SMP-based
> > virtualization use cases.
> 
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
> 
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

Yep.  Plan is:
1) Measure the difference.
2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
   revisit for 3.3.

Cheers,
Rusty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > A trivial, albeit sub-optimal, solution would be to simply revert
> > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > that's going to have a negative impact on performance of SMP-based
> > virtualization use cases.
> 
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
> 
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

Yep.  Plan is:
1) Measure the difference.
2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
   revisit for 3.3.

Cheers,
Rusty.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > A trivial, albeit sub-optimal, solution would be to simply revert
> > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > that's going to have a negative impact on performance of SMP-based
> > virtualization use cases.
> 
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
> 
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

Yep.  Plan is:
1) Measure the difference.
2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
   revisit for 3.3.

Cheers,
Rusty.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

[-- Attachment #1: Type: text/plain, Size: 2147 bytes --]

On Sat, Dec 03, 2011 at 03:44:36PM +1030, Rusty Russell wrote:
> On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> > On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > > A trivial, albeit sub-optimal, solution would be to simply revert
> > > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > > that's going to have a negative impact on performance of SMP-based
> > > virtualization use cases.
> > 
> > Have you measured the impact of using normal barriers (non-SMP ones)
> > like we use on normal HW drivers unconditionally ?
> > 
> > IE. If the difference is small enough I'd say just go for it and avoid
> > the bloat.
> 
> Yep.  Plan is:
> 1) Measure the difference.
> 2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
> 3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
> 4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
>    revisit for 3.3.
> 
> Cheers,
> Rusty.

Forwarding some results by Amos, who run multiple netperf streams in
parallel, from an external box to the guest.  TCP_STREAM results were
noisy.  This could be due to buffering done by TCP, where packet size
varies even as message size is constant.

TCP_RR results were consistent. In this benchmark, after switching
to mandatory barriers, CPU utilization increased by up to 35% while
throughput went down by up to 14%. the normalized throughput/cpu
regressed consistently, between 7 and 35%

The "fix" applied was simply this:

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 3198f2e..fdccb77 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,7 +23,7 @@
 
 /* virtio guest is communicating with a virtual "device" that actually runs on
  * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
+#if 0
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
  * through relaxed memory I/O windows (which virtio does not use). */



-- 
MST

[-- Attachment #2: compare-external_host-guest.txt --]
[-- Type: text/plain, Size: 18832 bytes --]

Fri Dec  9 23:57:33 2011

1 - old-exhost_guest.txt
2 - fixed-exhost_guest.txt

======================================================================================================================================================
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    949.64| 10.64|        89|   1170134|   1368739|         0|        17|    487392|    488820|    504716|       2.39|       2.71
2            1|   64|    946.03| 10.87|        87|   1119582|   1325851|         0|        17|    493763|    485865|    516161|       2.30|       2.57
%             |  0.0|      -0.4|  +2.2|      -2.2|      -4.3|      -3.1|         0|       0.0|      +1.3|      -0.6|      +2.3|       -3.8|       -5.2
1            2|   64|   1877.15| 15.45|       121|   2151267|   2561929|         0|        33|    923916|    971093|    969360|       2.22|       2.64
2            2|   64|   1867.63| 15.06|       124|   2212457|   2607606|         0|        33|    836160|    927721|    883964|       2.38|       2.95
%             |  0.0|      -0.5|  -2.5|      +2.5|      +2.8|      +1.8|         0|       0.0|      -9.5|      -4.5|      -8.8|       +7.2|      +11.7
1            4|   64|   3577.38| 19.62|       182|   4176151|   5036661|         0|        64|   1677417|   1412979|   1859101|       2.96|       2.71
2            4|   64|   3583.17| 20.05|       178|   4215327|   5063534|         0|        65|   1682582|   1549394|   1759033|       2.72|       2.88
%             |  0.0|      +0.2|  +2.2|      -2.2|      +0.9|      +0.5|         0|      +1.6|      +0.3|      +9.7|      -5.4|       -8.1|       +6.3
1            1|  256|   2654.52| 11.41|       232|    925787|   1029214|         0|        14|    597763|    670927|    619414|       1.38|       1.66
2            1|  256|   2632.22| 20.32|       129|    977446|   1036094|         0|        15|    742699|    715460|    764512|       1.37|       1.36
%             |  0.0|      -0.8| +78.1|     -44.4|      +5.6|      +0.7|         0|      +7.1|     +24.2|      +6.6|     +23.4|       -0.7|      -18.1
1            2|  256|   5228.76| 16.94|       308|   1949442|   2082492|         0|        30|   1230329|   1323945|   1274262|       1.47|       1.63
2            2|  256|   5140.98| 19.58|       262|   1991090|   2093206|         0|        30|   1400232|   1271363|   1441564|       1.57|       1.45
%             |  0.0|      -1.7| +15.6|     -14.9|      +2.1|      +0.5|         0|       0.0|     +13.8|      -4.0|     +13.1|       +6.8|      -11.0
1            4|  256|   9412.61| 24.04|       391|   2292404|   2351356|         0|        35|   1669864|    555786|   1741742|       4.12|       1.35
2            4|  256|   9408.92| 22.80|       412|   2303267|   2350153|         0|        35|   1665622|    889883|   1760349|       2.59|       1.34
%             |  0.0|      -0.0|  -5.2|      +5.4|      +0.5|      -0.1|         0|       0.0|      -0.3|     +60.1|      +1.1|      -37.1|       -0.7
1            1|  512|   3683.30| 13.18|       279|    914301|   1024803|         0|        14|    633219|    676673|    661047|       1.35|       1.55
2            1|  512|   3639.29| 14.48|       251|    997516|   1039428|         0|        15|    682321|    754864|    708906|       1.32|       1.47
%             |  0.0|      -1.2|  +9.9|     -10.0|      +9.1|      +1.4|         0|      +7.1|      +7.8|     +11.6|      +7.2|       -2.2|       -5.2
1            2|  512|   7476.00| 19.81|       377|   1938184|   2018640|         0|        29|   1473562|    864841|   1513069|       2.24|       1.33
2            2|  512|   7220.90| 19.18|       376|   1785571|   2017992|         0|        28|   1177162|   1113322|   1232894|       1.60|       1.64
%             |  0.0|      -3.4|  -3.2|      -0.3|      -7.9|      -0.0|         0|      -3.4|     -20.1|     +28.7|     -18.5|      -28.6|      +23.3
1            4|  512|   8858.46| 23.02|       384|   2312952|   2376210|         0|        38|   1757540|     92819|   1803434|      24.92|       1.32
2            4|  512|   9412.01| 24.78|       379|   1984425|   2308926|         0|        30|   1597209|    682183|   1808295|       2.91|       1.28
%             |  0.0|      +6.2|  +7.6|      -1.3|     -14.2|      -2.8|         0|     -21.1|      -9.1|    +635.0|      +0.3|      -88.3|       -3.0
1            1| 1024|   4208.94| 14.76|       285|    849941|    921136|         0|        13|    674738|    613922|    700105|       1.38|       1.32
2            1| 1024|   4192.50| 14.03|       298|    817761|    922081|         0|        12|    583808|    566704|    609916|       1.44|       1.51
%             |  0.0|      -0.4|  -4.9|      +4.6|      -3.8|      +0.1|         0|      -7.7|     -13.5|      -7.7|     -12.9|       +4.3|      +14.4
1            2| 1024|   9408.36| 21.04|       447|   1767352|   1918373|         0|        27|   1124248|    860371|   1179068|       2.05|       1.63
2            2| 1024|   8694.83| 18.20|       477|   1784228|   1872733|         0|        28|   1116901|   1220318|   1161491|       1.46|       1.61
%             |  0.0|      -7.6| -13.5|      +6.7|      +1.0|      -2.4|         0|      +3.7|      -0.7|     +41.8|      -1.5|      -28.8|       -1.2
1            4| 1024|   9411.98| 24.04|       391|   1961925|   2292825|         0|        30|   1398947|    965506|   1835604|       2.03|       1.25
2            4| 1024|   9409.97| 26.15|       359|   2072379|   2317492|         0|        34|   1494789|    855245|   1874374|       2.42|       1.24
%             |  0.0|      -0.0|  +8.8|      -8.2|      +5.6|      +1.1|         0|     +13.3|      +6.9|     -11.4|      +2.1|      +19.2|       -0.8
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1092.21| 17.68|        61|   5855912|   1155517|         4|        89|    476428|   4912055|    538322|       1.19|       2.15
2            1|   64|   1031.43| 17.42|        59|   5533459|   1114148|         1|        85|    476884|   4701241|    538789|       1.18|       2.07
%             |  0.0|      -5.6|  -1.5|      -3.3|      -5.5|      -3.6|     -75.0|      -4.5|      +0.1|      -4.3|      +0.1|       -0.8|       -3.7
1            2|   64|   2540.09| 32.76|        77|  13466718|   2509747|         1|       206|    473959|   2026564|    594762|       6.65|       4.22
2            2|   64|   2523.91| 32.72|        77|  13371187|   2486851|         1|       204|    477741|   1909387|    598663|       7.00|       4.15
%             |  0.0|      -0.6|  -0.1|       0.0|      -0.7|      -0.9|       0.0|      -1.0|      +0.8|      -5.8|      +0.7|       +5.3|       -1.7
1            4|   64|   2532.23| 32.96|        76|  13444352|   2555416|         1|       205|    477843|   2048074|    598584|       6.56|       4.27
2            4|   64|   2519.36| 33.10|        76|  13371867|   2530066|         3|       207|    476255|   1917578|    596967|       6.97|       4.24
%             |  0.0|      -0.5|  +0.4|       0.0|      -0.5|      -1.0|    +200.0|      +1.0|      -0.3|      -6.4|      -0.3|       +6.3|       -0.7
1            1|  256|   3481.21| 24.08|       144|  18046440|   1051754|         0|       277|     95147|    115092|    157259|     156.80|       6.69
2            1|  256|   3582.56| 25.00|       143|  18555128|   1079983|         2|       285|     82839|     69704|    144987|     266.20|       7.45
%             |  0.0|      +2.9|  +3.8|      -0.7|      +2.8|      +2.7|         0|      +2.9|     -12.9|     -39.4|      -7.8|      +69.8|      +11.4
1            2|  256|   7835.95| 29.20|       268|   9215513|    988831|         2|     13609|    242276|    235627|    359178|      39.11|       2.75
2            2|  256|   7227.12| 26.14|       276|   8224494|    974662|         0|      3787|    267604|    257681|    362507|      31.92|       2.69
%             |  0.0|      -7.8| -10.5|      +3.0|     -10.8|      -1.4|    -100.0|     -72.2|     +10.5|      +9.4|      +0.9|      -18.4|       -2.2
1            4|  256|   9286.71| 29.59|       313|   3119380|   1125149|      1585|      2794|    365162|    334103|    483517|       9.34|       2.33
2            4|  256|   9365.28| 28.90|       324|   2609986|   1116090|       971|      2064|    373494|    342389|    488519|       7.62|       2.28
%             |  0.0|      +0.8|  -2.3|      +3.5|     -16.3|      -0.8|     -38.7|     -26.1|      +2.3|      +2.5|      +1.0|      -18.4|       -2.1
1            1|  512|   9067.78| 17.76|       510|   2489492|    926754|         0|      3072|    350716|    318289|    415029|       7.82|       2.23
2            1|  512|   9363.32| 17.00|       550|   1239455|    934632|         0|       360|    444912|    419859|    506000|       2.95|       1.85
%             |  0.0|      +3.3|  -4.3|      +7.8|     -50.2|      +0.9|         0|     -88.3|     +26.9|     +31.9|     +21.9|      -62.3|      -17.0
1            2|  512|   8897.24| 20.18|       440|   4677263|    765728|         0|      8964|    312268|    266093|    384872|      17.58|       1.99
2            2|  512|   8994.28| 19.60|       458|   4068494|    777077|         0|      8040|    324208|    279281|    394930|      14.57|       1.97
%             |  0.0|      +1.1|  -2.9|      +4.1|     -13.0|      +1.5|         0|     -10.3|      +3.8|      +5.0|      +2.6|      -17.1|       -1.0
1            4|  512|   8660.59| 22.89|       378|   6274162|   1004294|         0|      9581|    300087|    248234|    381499|      25.28|       2.63
2            4|  512|   8720.95| 22.42|       388|   6321071|    996665|         2|      9307|    293339|    239522|    372874|      26.39|       2.67
%             |  0.0|      +0.7|  -2.1|      +2.6|      +0.7|      -0.8|         0|      -2.9|      -2.2|      -3.5|      -2.3|       +4.4|       +1.5
1            1| 1024|   9382.57| 16.55|       566|   1098014|    943968|         0|        63|    411612|    385473|    463009|       2.85|       2.04
2            1| 1024|   9375.36| 16.46|       569|   1113185|    962141|         0|       109|    434974|    419286|    486685|       2.65|       1.98
%             |  0.0|      -0.1|  -0.5|      +0.5|      +1.4|      +1.9|         0|     +73.0|      +5.7|      +8.8|      +5.1|       -7.0|       -2.9
1            2| 1024|   9341.87| 17.32|       539|   1449472|    807439|         0|       897|    380053|    354028|    433351|       4.09|       1.86
2            2| 1024|   9364.12| 17.13|       546|   1344190|    794110|         0|       581|    379684|    352227|    431694|       3.82|       1.84
%             |  0.0|      +0.2|  -1.1|      +1.3|      -7.3|      -1.7|         0|     -35.2|      -0.1|      -0.5|      -0.4|       -6.6|       -1.1
1            4| 1024|   9391.41| 18.70|       502|   1509582|   1104681|      2166|       658|    399083|    369721|    453543|       4.08|       2.44
2            4| 1024|   9404.67| 18.45|       509|   1361411|   1083647|       386|       398|    400368|    374415|    453718|       3.64|       2.39
%             |  0.0|      +0.1|  -1.3|      +1.4|      -9.8|      -1.9|     -82.2|     -39.5|      +0.3|      +1.3|      +0.0|      -10.8|       -2.0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10874.30|  6.75|      1611|    652525|    652529|         0|        10|    652518|    652582|    663468|       1.00|       0.98
2           50|   64|   9568.49|  9.13|      1048|    574176|    574180|         0|         8|    574166|    574255|    593988|       1.00|       0.97
%             |  0.0|     -12.0| +35.3|     -34.9|     -12.0|     -12.0|         0|     -20.0|     -12.0|     -12.0|     -10.5|        0.0|       -1.0
1          100|   64|  10880.83|  6.77|      1607|    652919|    652926|         0|        10|    652912|    652981|    664406|       1.00|       0.98
2          100|   64|   9586.65|  7.67|      1249|    575267|    575270|         0|         9|    575257|    575326|    593343|       1.00|       0.97
%             |  0.0|     -11.9| +13.3|     -22.3|     -11.9|     -11.9|         0|     -10.0|     -11.9|     -11.9|     -10.7|        0.0|       -1.0
1          250|   64|  10930.11|  6.81|      1605|    655873|    655884|         0|        10|    655869|    655951|    664979|       1.00|       0.99
2          250|   64|   9556.15|  7.65|      1249|    573436|    573440|         0|         9|    573427|    573488|    591151|       1.00|       0.97
%             |  0.0|     -12.6| +12.3|     -22.2|     -12.6|     -12.6|         0|     -10.0|     -12.6|     -12.6|     -11.1|        0.0|       -2.0
1          500|   64|  10930.33|  6.77|      1614|    655886|    655890|         0|        10|    655877|    655949|    664983|       1.00|       0.99
2          500|   64|   9598.61|  8.46|      1134|    575983|    575987|         0|         9|    575975|    576057|    595875|       1.00|       0.97
%             |  0.0|     -12.2| +25.0|     -29.7|     -12.2|     -12.2|         0|     -10.0|     -12.2|     -12.2|     -10.4|        0.0|       -2.0
1           50|  256|  10435.24|  8.87|      1176|    626181|    626184|         0|         9|    626173|    626240|    641870|       1.00|       0.98
2           50|  256|   8948.58| 10.06|       889|    536981|    536985|         0|         8|    536971|    537025|    552906|       1.00|       0.97
%             |  0.0|     -14.2| +13.4|     -24.4|     -14.2|     -14.2|         0|     -11.1|     -14.2|     -14.2|     -13.9|        0.0|       -1.0
1          100|  256|  10401.87|  8.73|      1191|    624179|    624184|         0|        10|    624168|    624248|    640003|       1.00|       0.98
2          100|  256|   9060.39| 10.46|       866|    543692|    543696|         0|         8|    543683|    543752|    559009|       1.00|       0.97
%             |  0.0|     -12.9| +19.8|     -27.3|     -12.9|     -12.9|         0|     -20.0|     -12.9|     -12.9|     -12.7|        0.0|       -1.0
1          250|  256|  10369.91|  8.79|      1179|    622261|    622267|         0|         9|    622252|    622325|    638523|       1.00|       0.97
2          250|  256|   9011.48| 10.88|       828|    540755|    540758|         0|         8|    540743|    540830|    555336|       1.00|       0.97
%             |  0.0|     -13.1| +23.8|     -29.8|     -13.1|     -13.1|         0|     -11.1|     -13.1|     -13.1|     -13.0|        0.0|        0.0
1          500|  256|  10400.42|  8.70|      1195|    624094|    624099|         0|        10|    624085|    624155|    639254|       1.00|       0.98
2          500|  256|   9178.08| 10.83|       847|    550753|    550757|         0|         9|    550744|    550811|    566165|       1.00|       0.97
%             |  0.0|     -11.8| +24.5|     -29.1|     -11.8|     -11.8|         0|     -10.0|     -11.8|     -11.8|     -11.4|        0.0|       -1.0
1           50|  512|   8231.77|  6.22|      1323|    493974|    493978|         0|         7|    493966|    494049|    507228|       1.00|       0.97
2           50|  512|   7718.30|  5.81|      1328|    463164|    463168|         0|         7|    463155|    463218|    478833|       1.00|       0.97
%             |  0.0|      -6.2|  -6.6|      +0.4|      -6.2|      -6.2|         0|       0.0|      -6.2|      -6.2|      -5.6|        0.0|        0.0
1          100|  512|   8267.63|  6.16|      1342|    496125|    496130|         0|         8|    496115|    496186|    509422|       1.00|       0.97
2          100|  512|   7713.78|  8.29|       930|    462894|    462900|         0|         7|    462885|    462977|    488732|       1.00|       0.95
%             |  0.0|      -6.7| +34.6|     -30.7|      -6.7|      -6.7|         0|     -12.5|      -6.7|      -6.7|      -4.1|        0.0|       -2.1
1          250|  512|   8227.23|  6.30|      1305|    493701|    493704|         0|         7|    493691|    493751|    507689|       1.00|       0.97
2          250|  512|   7737.14|  8.35|       926|    464296|    464303|         0|         7|    464291|    464358|    486086|       1.00|       0.96
%             |  0.0|      -6.0| +32.5|     -29.0|      -6.0|      -6.0|         0|       0.0|      -6.0|      -6.0|      -4.3|        0.0|       -1.0
1          500|  512|   8259.80|  6.20|      1332|    495656|    495661|         0|         8|    495649|    495711|    509070|       1.00|       0.97
2          500|  512|   7614.72|  7.47|      1019|    456950|    456959|         0|         7|    456941|    457005|    476914|       1.00|       0.96
%             |  0.0|      -7.8| +20.5|     -23.5|      -7.8|      -7.8|         0|     -12.5|      -7.8|      -7.8|      -6.3|        0.0|       -1.0
1           50| 1024|   7879.43|  6.80|      1158|    472832|    472835|         0|         7|    472822|    472906|    488163|       1.00|       0.97
2           50| 1024|   7302.28|  7.35|       993|    438203|    438207|         0|         7|    438193|    438261|    455627|       1.00|       0.96
%             |  0.0|      -7.3|  +8.1|     -14.2|      -7.3|      -7.3|         0|       0.0|      -7.3|      -7.3|      -6.7|        0.0|       -1.0
1          100| 1024|   7916.07|  6.84|      1157|    475031|    475038|         0|         8|    475024|    475097|    490797|       1.00|       0.97
2          100| 1024|   7491.90|  6.96|      1076|    449582|    449587|         0|         7|    449573|    449627|    462560|       1.00|       0.97
%             |  0.0|      -5.4|  +1.8|      -7.0|      -5.4|      -5.4|         0|     -12.5|      -5.4|      -5.4|      -5.8|        0.0|        0.0
1          250| 1024|   7865.20|  6.82|      1153|    471978|    471984|         0|         7|    471969|    472035|    486978|       1.00|       0.97
2          250| 1024|   7472.83|  7.07|      1056|    448436|    448442|         0|         6|    448431|    448496|    463680|       1.00|       0.97
%             |  0.0|      -5.0|  +3.7|      -8.4|      -5.0|      -5.0|         0|     -14.3|      -5.0|      -5.0|      -4.8|        0.0|        0.0
1          500| 1024|   7931.30|  6.99|      1134|    475944|    475954|         0|         7|    475939|    476016|    491021|       1.00|       0.97
2          500| 1024|   7407.54|  7.57|       978|    444519|    444524|         0|         7|    444509|    444567|    462360|       1.00|       0.96
%             |  0.0|      -6.6|  +8.3|     -13.8|      -6.6|      -6.6|         0|       0.0|      -6.6|      -6.6|      -5.8|        0.0|       -1.0

[-- Attachment #3: Type: text/plain, Size: 183 bytes --]

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

[-- Attachment #1: Type: text/plain, Size: 2147 bytes --]

On Sat, Dec 03, 2011 at 03:44:36PM +1030, Rusty Russell wrote:
> On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> > On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > > A trivial, albeit sub-optimal, solution would be to simply revert
> > > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > > that's going to have a negative impact on performance of SMP-based
> > > virtualization use cases.
> > 
> > Have you measured the impact of using normal barriers (non-SMP ones)
> > like we use on normal HW drivers unconditionally ?
> > 
> > IE. If the difference is small enough I'd say just go for it and avoid
> > the bloat.
> 
> Yep.  Plan is:
> 1) Measure the difference.
> 2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
> 3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
> 4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
>    revisit for 3.3.
> 
> Cheers,
> Rusty.

Forwarding some results by Amos, who run multiple netperf streams in
parallel, from an external box to the guest.  TCP_STREAM results were
noisy.  This could be due to buffering done by TCP, where packet size
varies even as message size is constant.

TCP_RR results were consistent. In this benchmark, after switching
to mandatory barriers, CPU utilization increased by up to 35% while
throughput went down by up to 14%. the normalized throughput/cpu
regressed consistently, between 7 and 35%

The "fix" applied was simply this:

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 3198f2e..fdccb77 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,7 +23,7 @@
 
 /* virtio guest is communicating with a virtual "device" that actually runs on
  * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
+#if 0
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
  * through relaxed memory I/O windows (which virtio does not use). */



-- 
MST

[-- Attachment #2: compare-external_host-guest.txt --]
[-- Type: text/plain, Size: 18832 bytes --]

Fri Dec  9 23:57:33 2011

1 - old-exhost_guest.txt
2 - fixed-exhost_guest.txt

======================================================================================================================================================
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    949.64| 10.64|        89|   1170134|   1368739|         0|        17|    487392|    488820|    504716|       2.39|       2.71
2            1|   64|    946.03| 10.87|        87|   1119582|   1325851|         0|        17|    493763|    485865|    516161|       2.30|       2.57
%             |  0.0|      -0.4|  +2.2|      -2.2|      -4.3|      -3.1|         0|       0.0|      +1.3|      -0.6|      +2.3|       -3.8|       -5.2
1            2|   64|   1877.15| 15.45|       121|   2151267|   2561929|         0|        33|    923916|    971093|    969360|       2.22|       2.64
2            2|   64|   1867.63| 15.06|       124|   2212457|   2607606|         0|        33|    836160|    927721|    883964|       2.38|       2.95
%             |  0.0|      -0.5|  -2.5|      +2.5|      +2.8|      +1.8|         0|       0.0|      -9.5|      -4.5|      -8.8|       +7.2|      +11.7
1            4|   64|   3577.38| 19.62|       182|   4176151|   5036661|         0|        64|   1677417|   1412979|   1859101|       2.96|       2.71
2            4|   64|   3583.17| 20.05|       178|   4215327|   5063534|         0|        65|   1682582|   1549394|   1759033|       2.72|       2.88
%             |  0.0|      +0.2|  +2.2|      -2.2|      +0.9|      +0.5|         0|      +1.6|      +0.3|      +9.7|      -5.4|       -8.1|       +6.3
1            1|  256|   2654.52| 11.41|       232|    925787|   1029214|         0|        14|    597763|    670927|    619414|       1.38|       1.66
2            1|  256|   2632.22| 20.32|       129|    977446|   1036094|         0|        15|    742699|    715460|    764512|       1.37|       1.36
%             |  0.0|      -0.8| +78.1|     -44.4|      +5.6|      +0.7|         0|      +7.1|     +24.2|      +6.6|     +23.4|       -0.7|      -18.1
1            2|  256|   5228.76| 16.94|       308|   1949442|   2082492|         0|        30|   1230329|   1323945|   1274262|       1.47|       1.63
2            2|  256|   5140.98| 19.58|       262|   1991090|   2093206|         0|        30|   1400232|   1271363|   1441564|       1.57|       1.45
%             |  0.0|      -1.7| +15.6|     -14.9|      +2.1|      +0.5|         0|       0.0|     +13.8|      -4.0|     +13.1|       +6.8|      -11.0
1            4|  256|   9412.61| 24.04|       391|   2292404|   2351356|         0|        35|   1669864|    555786|   1741742|       4.12|       1.35
2            4|  256|   9408.92| 22.80|       412|   2303267|   2350153|         0|        35|   1665622|    889883|   1760349|       2.59|       1.34
%             |  0.0|      -0.0|  -5.2|      +5.4|      +0.5|      -0.1|         0|       0.0|      -0.3|     +60.1|      +1.1|      -37.1|       -0.7
1            1|  512|   3683.30| 13.18|       279|    914301|   1024803|         0|        14|    633219|    676673|    661047|       1.35|       1.55
2            1|  512|   3639.29| 14.48|       251|    997516|   1039428|         0|        15|    682321|    754864|    708906|       1.32|       1.47
%             |  0.0|      -1.2|  +9.9|     -10.0|      +9.1|      +1.4|         0|      +7.1|      +7.8|     +11.6|      +7.2|       -2.2|       -5.2
1            2|  512|   7476.00| 19.81|       377|   1938184|   2018640|         0|        29|   1473562|    864841|   1513069|       2.24|       1.33
2            2|  512|   7220.90| 19.18|       376|   1785571|   2017992|         0|        28|   1177162|   1113322|   1232894|       1.60|       1.64
%             |  0.0|      -3.4|  -3.2|      -0.3|      -7.9|      -0.0|         0|      -3.4|     -20.1|     +28.7|     -18.5|      -28.6|      +23.3
1            4|  512|   8858.46| 23.02|       384|   2312952|   2376210|         0|        38|   1757540|     92819|   1803434|      24.92|       1.32
2            4|  512|   9412.01| 24.78|       379|   1984425|   2308926|         0|        30|   1597209|    682183|   1808295|       2.91|       1.28
%             |  0.0|      +6.2|  +7.6|      -1.3|     -14.2|      -2.8|         0|     -21.1|      -9.1|    +635.0|      +0.3|      -88.3|       -3.0
1            1| 1024|   4208.94| 14.76|       285|    849941|    921136|         0|        13|    674738|    613922|    700105|       1.38|       1.32
2            1| 1024|   4192.50| 14.03|       298|    817761|    922081|         0|        12|    583808|    566704|    609916|       1.44|       1.51
%             |  0.0|      -0.4|  -4.9|      +4.6|      -3.8|      +0.1|         0|      -7.7|     -13.5|      -7.7|     -12.9|       +4.3|      +14.4
1            2| 1024|   9408.36| 21.04|       447|   1767352|   1918373|         0|        27|   1124248|    860371|   1179068|       2.05|       1.63
2            2| 1024|   8694.83| 18.20|       477|   1784228|   1872733|         0|        28|   1116901|   1220318|   1161491|       1.46|       1.61
%             |  0.0|      -7.6| -13.5|      +6.7|      +1.0|      -2.4|         0|      +3.7|      -0.7|     +41.8|      -1.5|      -28.8|       -1.2
1            4| 1024|   9411.98| 24.04|       391|   1961925|   2292825|         0|        30|   1398947|    965506|   1835604|       2.03|       1.25
2            4| 1024|   9409.97| 26.15|       359|   2072379|   2317492|         0|        34|   1494789|    855245|   1874374|       2.42|       1.24
%             |  0.0|      -0.0|  +8.8|      -8.2|      +5.6|      +1.1|         0|     +13.3|      +6.9|     -11.4|      +2.1|      +19.2|       -0.8
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1092.21| 17.68|        61|   5855912|   1155517|         4|        89|    476428|   4912055|    538322|       1.19|       2.15
2            1|   64|   1031.43| 17.42|        59|   5533459|   1114148|         1|        85|    476884|   4701241|    538789|       1.18|       2.07
%             |  0.0|      -5.6|  -1.5|      -3.3|      -5.5|      -3.6|     -75.0|      -4.5|      +0.1|      -4.3|      +0.1|       -0.8|       -3.7
1            2|   64|   2540.09| 32.76|        77|  13466718|   2509747|         1|       206|    473959|   2026564|    594762|       6.65|       4.22
2            2|   64|   2523.91| 32.72|        77|  13371187|   2486851|         1|       204|    477741|   1909387|    598663|       7.00|       4.15
%             |  0.0|      -0.6|  -0.1|       0.0|      -0.7|      -0.9|       0.0|      -1.0|      +0.8|      -5.8|      +0.7|       +5.3|       -1.7
1            4|   64|   2532.23| 32.96|        76|  13444352|   2555416|         1|       205|    477843|   2048074|    598584|       6.56|       4.27
2            4|   64|   2519.36| 33.10|        76|  13371867|   2530066|         3|       207|    476255|   1917578|    596967|       6.97|       4.24
%             |  0.0|      -0.5|  +0.4|       0.0|      -0.5|      -1.0|    +200.0|      +1.0|      -0.3|      -6.4|      -0.3|       +6.3|       -0.7
1            1|  256|   3481.21| 24.08|       144|  18046440|   1051754|         0|       277|     95147|    115092|    157259|     156.80|       6.69
2            1|  256|   3582.56| 25.00|       143|  18555128|   1079983|         2|       285|     82839|     69704|    144987|     266.20|       7.45
%             |  0.0|      +2.9|  +3.8|      -0.7|      +2.8|      +2.7|         0|      +2.9|     -12.9|     -39.4|      -7.8|      +69.8|      +11.4
1            2|  256|   7835.95| 29.20|       268|   9215513|    988831|         2|     13609|    242276|    235627|    359178|      39.11|       2.75
2            2|  256|   7227.12| 26.14|       276|   8224494|    974662|         0|      3787|    267604|    257681|    362507|      31.92|       2.69
%             |  0.0|      -7.8| -10.5|      +3.0|     -10.8|      -1.4|    -100.0|     -72.2|     +10.5|      +9.4|      +0.9|      -18.4|       -2.2
1            4|  256|   9286.71| 29.59|       313|   3119380|   1125149|      1585|      2794|    365162|    334103|    483517|       9.34|       2.33
2            4|  256|   9365.28| 28.90|       324|   2609986|   1116090|       971|      2064|    373494|    342389|    488519|       7.62|       2.28
%             |  0.0|      +0.8|  -2.3|      +3.5|     -16.3|      -0.8|     -38.7|     -26.1|      +2.3|      +2.5|      +1.0|      -18.4|       -2.1
1            1|  512|   9067.78| 17.76|       510|   2489492|    926754|         0|      3072|    350716|    318289|    415029|       7.82|       2.23
2            1|  512|   9363.32| 17.00|       550|   1239455|    934632|         0|       360|    444912|    419859|    506000|       2.95|       1.85
%             |  0.0|      +3.3|  -4.3|      +7.8|     -50.2|      +0.9|         0|     -88.3|     +26.9|     +31.9|     +21.9|      -62.3|      -17.0
1            2|  512|   8897.24| 20.18|       440|   4677263|    765728|         0|      8964|    312268|    266093|    384872|      17.58|       1.99
2            2|  512|   8994.28| 19.60|       458|   4068494|    777077|         0|      8040|    324208|    279281|    394930|      14.57|       1.97
%             |  0.0|      +1.1|  -2.9|      +4.1|     -13.0|      +1.5|         0|     -10.3|      +3.8|      +5.0|      +2.6|      -17.1|       -1.0
1            4|  512|   8660.59| 22.89|       378|   6274162|   1004294|         0|      9581|    300087|    248234|    381499|      25.28|       2.63
2            4|  512|   8720.95| 22.42|       388|   6321071|    996665|         2|      9307|    293339|    239522|    372874|      26.39|       2.67
%             |  0.0|      +0.7|  -2.1|      +2.6|      +0.7|      -0.8|         0|      -2.9|      -2.2|      -3.5|      -2.3|       +4.4|       +1.5
1            1| 1024|   9382.57| 16.55|       566|   1098014|    943968|         0|        63|    411612|    385473|    463009|       2.85|       2.04
2            1| 1024|   9375.36| 16.46|       569|   1113185|    962141|         0|       109|    434974|    419286|    486685|       2.65|       1.98
%             |  0.0|      -0.1|  -0.5|      +0.5|      +1.4|      +1.9|         0|     +73.0|      +5.7|      +8.8|      +5.1|       -7.0|       -2.9
1            2| 1024|   9341.87| 17.32|       539|   1449472|    807439|         0|       897|    380053|    354028|    433351|       4.09|       1.86
2            2| 1024|   9364.12| 17.13|       546|   1344190|    794110|         0|       581|    379684|    352227|    431694|       3.82|       1.84
%             |  0.0|      +0.2|  -1.1|      +1.3|      -7.3|      -1.7|         0|     -35.2|      -0.1|      -0.5|      -0.4|       -6.6|       -1.1
1            4| 1024|   9391.41| 18.70|       502|   1509582|   1104681|      2166|       658|    399083|    369721|    453543|       4.08|       2.44
2            4| 1024|   9404.67| 18.45|       509|   1361411|   1083647|       386|       398|    400368|    374415|    453718|       3.64|       2.39
%             |  0.0|      +0.1|  -1.3|      +1.4|      -9.8|      -1.9|     -82.2|     -39.5|      +0.3|      +1.3|      +0.0|      -10.8|       -2.0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10874.30|  6.75|      1611|    652525|    652529|         0|        10|    652518|    652582|    663468|       1.00|       0.98
2           50|   64|   9568.49|  9.13|      1048|    574176|    574180|         0|         8|    574166|    574255|    593988|       1.00|       0.97
%             |  0.0|     -12.0| +35.3|     -34.9|     -12.0|     -12.0|         0|     -20.0|     -12.0|     -12.0|     -10.5|        0.0|       -1.0
1          100|   64|  10880.83|  6.77|      1607|    652919|    652926|         0|        10|    652912|    652981|    664406|       1.00|       0.98
2          100|   64|   9586.65|  7.67|      1249|    575267|    575270|         0|         9|    575257|    575326|    593343|       1.00|       0.97
%             |  0.0|     -11.9| +13.3|     -22.3|     -11.9|     -11.9|         0|     -10.0|     -11.9|     -11.9|     -10.7|        0.0|       -1.0
1          250|   64|  10930.11|  6.81|      1605|    655873|    655884|         0|        10|    655869|    655951|    664979|       1.00|       0.99
2          250|   64|   9556.15|  7.65|      1249|    573436|    573440|         0|         9|    573427|    573488|    591151|       1.00|       0.97
%             |  0.0|     -12.6| +12.3|     -22.2|     -12.6|     -12.6|         0|     -10.0|     -12.6|     -12.6|     -11.1|        0.0|       -2.0
1          500|   64|  10930.33|  6.77|      1614|    655886|    655890|         0|        10|    655877|    655949|    664983|       1.00|       0.99
2          500|   64|   9598.61|  8.46|      1134|    575983|    575987|         0|         9|    575975|    576057|    595875|       1.00|       0.97
%             |  0.0|     -12.2| +25.0|     -29.7|     -12.2|     -12.2|         0|     -10.0|     -12.2|     -12.2|     -10.4|        0.0|       -2.0
1           50|  256|  10435.24|  8.87|      1176|    626181|    626184|         0|         9|    626173|    626240|    641870|       1.00|       0.98
2           50|  256|   8948.58| 10.06|       889|    536981|    536985|         0|         8|    536971|    537025|    552906|       1.00|       0.97
%             |  0.0|     -14.2| +13.4|     -24.4|     -14.2|     -14.2|         0|     -11.1|     -14.2|     -14.2|     -13.9|        0.0|       -1.0
1          100|  256|  10401.87|  8.73|      1191|    624179|    624184|         0|        10|    624168|    624248|    640003|       1.00|       0.98
2          100|  256|   9060.39| 10.46|       866|    543692|    543696|         0|         8|    543683|    543752|    559009|       1.00|       0.97
%             |  0.0|     -12.9| +19.8|     -27.3|     -12.9|     -12.9|         0|     -20.0|     -12.9|     -12.9|     -12.7|        0.0|       -1.0
1          250|  256|  10369.91|  8.79|      1179|    622261|    622267|         0|         9|    622252|    622325|    638523|       1.00|       0.97
2          250|  256|   9011.48| 10.88|       828|    540755|    540758|         0|         8|    540743|    540830|    555336|       1.00|       0.97
%             |  0.0|     -13.1| +23.8|     -29.8|     -13.1|     -13.1|         0|     -11.1|     -13.1|     -13.1|     -13.0|        0.0|        0.0
1          500|  256|  10400.42|  8.70|      1195|    624094|    624099|         0|        10|    624085|    624155|    639254|       1.00|       0.98
2          500|  256|   9178.08| 10.83|       847|    550753|    550757|         0|         9|    550744|    550811|    566165|       1.00|       0.97
%             |  0.0|     -11.8| +24.5|     -29.1|     -11.8|     -11.8|         0|     -10.0|     -11.8|     -11.8|     -11.4|        0.0|       -1.0
1           50|  512|   8231.77|  6.22|      1323|    493974|    493978|         0|         7|    493966|    494049|    507228|       1.00|       0.97
2           50|  512|   7718.30|  5.81|      1328|    463164|    463168|         0|         7|    463155|    463218|    478833|       1.00|       0.97
%             |  0.0|      -6.2|  -6.6|      +0.4|      -6.2|      -6.2|         0|       0.0|      -6.2|      -6.2|      -5.6|        0.0|        0.0
1          100|  512|   8267.63|  6.16|      1342|    496125|    496130|         0|         8|    496115|    496186|    509422|       1.00|       0.97
2          100|  512|   7713.78|  8.29|       930|    462894|    462900|         0|         7|    462885|    462977|    488732|       1.00|       0.95
%             |  0.0|      -6.7| +34.6|     -30.7|      -6.7|      -6.7|         0|     -12.5|      -6.7|      -6.7|      -4.1|        0.0|       -2.1
1          250|  512|   8227.23|  6.30|      1305|    493701|    493704|         0|         7|    493691|    493751|    507689|       1.00|       0.97
2          250|  512|   7737.14|  8.35|       926|    464296|    464303|         0|         7|    464291|    464358|    486086|       1.00|       0.96
%             |  0.0|      -6.0| +32.5|     -29.0|      -6.0|      -6.0|         0|       0.0|      -6.0|      -6.0|      -4.3|        0.0|       -1.0
1          500|  512|   8259.80|  6.20|      1332|    495656|    495661|         0|         8|    495649|    495711|    509070|       1.00|       0.97
2          500|  512|   7614.72|  7.47|      1019|    456950|    456959|         0|         7|    456941|    457005|    476914|       1.00|       0.96
%             |  0.0|      -7.8| +20.5|     -23.5|      -7.8|      -7.8|         0|     -12.5|      -7.8|      -7.8|      -6.3|        0.0|       -1.0
1           50| 1024|   7879.43|  6.80|      1158|    472832|    472835|         0|         7|    472822|    472906|    488163|       1.00|       0.97
2           50| 1024|   7302.28|  7.35|       993|    438203|    438207|         0|         7|    438193|    438261|    455627|       1.00|       0.96
%             |  0.0|      -7.3|  +8.1|     -14.2|      -7.3|      -7.3|         0|       0.0|      -7.3|      -7.3|      -6.7|        0.0|       -1.0
1          100| 1024|   7916.07|  6.84|      1157|    475031|    475038|         0|         8|    475024|    475097|    490797|       1.00|       0.97
2          100| 1024|   7491.90|  6.96|      1076|    449582|    449587|         0|         7|    449573|    449627|    462560|       1.00|       0.97
%             |  0.0|      -5.4|  +1.8|      -7.0|      -5.4|      -5.4|         0|     -12.5|      -5.4|      -5.4|      -5.8|        0.0|        0.0
1          250| 1024|   7865.20|  6.82|      1153|    471978|    471984|         0|         7|    471969|    472035|    486978|       1.00|       0.97
2          250| 1024|   7472.83|  7.07|      1056|    448436|    448442|         0|         6|    448431|    448496|    463680|       1.00|       0.97
%             |  0.0|      -5.0|  +3.7|      -8.4|      -5.0|      -5.0|         0|     -14.3|      -5.0|      -5.0|      -4.8|        0.0|        0.0
1          500| 1024|   7931.30|  6.99|      1134|    475944|    475954|         0|         7|    475939|    476016|    491021|       1.00|       0.97
2          500| 1024|   7407.54|  7.57|       978|    444519|    444524|         0|         7|    444509|    444567|    462360|       1.00|       0.96
%             |  0.0|      -6.6|  +8.3|     -13.8|      -6.6|      -6.6|         0|       0.0|      -6.6|      -6.6|      -5.8|        0.0|       -1.0

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Michael S. Tsirkin]

On Sat, Dec 03, 2011 at 03:44:36PM +1030, Rusty Russell wrote:
> On Sat, 03 Dec 2011 10:09:44 +1100, Benjamin Herrenschmidt <benh@kernel.crashing.org> wrote:
> > On Tue, 2011-11-29 at 14:31 +0200, Ohad Ben-Cohen wrote:
> > > A trivial, albeit sub-optimal, solution would be to simply revert
> > > commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
> > > that's going to have a negative impact on performance of SMP-based
> > > virtualization use cases.
> > 
> > Have you measured the impact of using normal barriers (non-SMP ones)
> > like we use on normal HW drivers unconditionally ?
> > 
> > IE. If the difference is small enough I'd say just go for it and avoid
> > the bloat.
> 
> Yep.  Plan is:
> 1) Measure the difference.
> 2) Difference unmeassurable?  Use normal barriers (ie. revert d57ed95).
> 3) Difference small?  Revert d57ed95 for 3.2, revisit for 3.3.
> 4) Difference large?  Runtime switch based on "if you're PCI" for 3.2,
>    revisit for 3.3.
> 
> Cheers,
> Rusty.

Forwarding some results by Amos, who run multiple netperf streams in
parallel, from an external box to the guest.  TCP_STREAM results were
noisy.  This could be due to buffering done by TCP, where packet size
varies even as message size is constant.

TCP_RR results were consistent. In this benchmark, after switching
to mandatory barriers, CPU utilization increased by up to 35% while
throughput went down by up to 14%. the normalized throughput/cpu
regressed consistently, between 7 and 35%

The "fix" applied was simply this:

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 3198f2e..fdccb77 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,7 +23,7 @@
 
 /* virtio guest is communicating with a virtual "device" that actually runs on
  * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
+#if 0
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
  * through relaxed memory I/O windows (which virtio does not use). */



-- 
MST
-------------- next part --------------
Fri Dec  9 23:57:33 2011

1 - old-exhost_guest.txt
2 - fixed-exhost_guest.txt

======================================================================================================================================================
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    949.64| 10.64|        89|   1170134|   1368739|         0|        17|    487392|    488820|    504716|       2.39|       2.71
2            1|   64|    946.03| 10.87|        87|   1119582|   1325851|         0|        17|    493763|    485865|    516161|       2.30|       2.57
%             |  0.0|      -0.4|  +2.2|      -2.2|      -4.3|      -3.1|         0|       0.0|      +1.3|      -0.6|      +2.3|       -3.8|       -5.2
1            2|   64|   1877.15| 15.45|       121|   2151267|   2561929|         0|        33|    923916|    971093|    969360|       2.22|       2.64
2            2|   64|   1867.63| 15.06|       124|   2212457|   2607606|         0|        33|    836160|    927721|    883964|       2.38|       2.95
%             |  0.0|      -0.5|  -2.5|      +2.5|      +2.8|      +1.8|         0|       0.0|      -9.5|      -4.5|      -8.8|       +7.2|      +11.7
1            4|   64|   3577.38| 19.62|       182|   4176151|   5036661|         0|        64|   1677417|   1412979|   1859101|       2.96|       2.71
2            4|   64|   3583.17| 20.05|       178|   4215327|   5063534|         0|        65|   1682582|   1549394|   1759033|       2.72|       2.88
%             |  0.0|      +0.2|  +2.2|      -2.2|      +0.9|      +0.5|         0|      +1.6|      +0.3|      +9.7|      -5.4|       -8.1|       +6.3
1            1|  256|   2654.52| 11.41|       232|    925787|   1029214|         0|        14|    597763|    670927|    619414|       1.38|       1.66
2            1|  256|   2632.22| 20.32|       129|    977446|   1036094|         0|        15|    742699|    715460|    764512|       1.37|       1.36
%             |  0.0|      -0.8| +78.1|     -44.4|      +5.6|      +0.7|         0|      +7.1|     +24.2|      +6.6|     +23.4|       -0.7|      -18.1
1            2|  256|   5228.76| 16.94|       308|   1949442|   2082492|         0|        30|   1230329|   1323945|   1274262|       1.47|       1.63
2            2|  256|   5140.98| 19.58|       262|   1991090|   2093206|         0|        30|   1400232|   1271363|   1441564|       1.57|       1.45
%             |  0.0|      -1.7| +15.6|     -14.9|      +2.1|      +0.5|         0|       0.0|     +13.8|      -4.0|     +13.1|       +6.8|      -11.0
1            4|  256|   9412.61| 24.04|       391|   2292404|   2351356|         0|        35|   1669864|    555786|   1741742|       4.12|       1.35
2            4|  256|   9408.92| 22.80|       412|   2303267|   2350153|         0|        35|   1665622|    889883|   1760349|       2.59|       1.34
%             |  0.0|      -0.0|  -5.2|      +5.4|      +0.5|      -0.1|         0|       0.0|      -0.3|     +60.1|      +1.1|      -37.1|       -0.7
1            1|  512|   3683.30| 13.18|       279|    914301|   1024803|         0|        14|    633219|    676673|    661047|       1.35|       1.55
2            1|  512|   3639.29| 14.48|       251|    997516|   1039428|         0|        15|    682321|    754864|    708906|       1.32|       1.47
%             |  0.0|      -1.2|  +9.9|     -10.0|      +9.1|      +1.4|         0|      +7.1|      +7.8|     +11.6|      +7.2|       -2.2|       -5.2
1            2|  512|   7476.00| 19.81|       377|   1938184|   2018640|         0|        29|   1473562|    864841|   1513069|       2.24|       1.33
2            2|  512|   7220.90| 19.18|       376|   1785571|   2017992|         0|        28|   1177162|   1113322|   1232894|       1.60|       1.64
%             |  0.0|      -3.4|  -3.2|      -0.3|      -7.9|      -0.0|         0|      -3.4|     -20.1|     +28.7|     -18.5|      -28.6|      +23.3
1            4|  512|   8858.46| 23.02|       384|   2312952|   2376210|         0|        38|   1757540|     92819|   1803434|      24.92|       1.32
2            4|  512|   9412.01| 24.78|       379|   1984425|   2308926|         0|        30|   1597209|    682183|   1808295|       2.91|       1.28
%             |  0.0|      +6.2|  +7.6|      -1.3|     -14.2|      -2.8|         0|     -21.1|      -9.1|    +635.0|      +0.3|      -88.3|       -3.0
1            1| 1024|   4208.94| 14.76|       285|    849941|    921136|         0|        13|    674738|    613922|    700105|       1.38|       1.32
2            1| 1024|   4192.50| 14.03|       298|    817761|    922081|         0|        12|    583808|    566704|    609916|       1.44|       1.51
%             |  0.0|      -0.4|  -4.9|      +4.6|      -3.8|      +0.1|         0|      -7.7|     -13.5|      -7.7|     -12.9|       +4.3|      +14.4
1            2| 1024|   9408.36| 21.04|       447|   1767352|   1918373|         0|        27|   1124248|    860371|   1179068|       2.05|       1.63
2            2| 1024|   8694.83| 18.20|       477|   1784228|   1872733|         0|        28|   1116901|   1220318|   1161491|       1.46|       1.61
%             |  0.0|      -7.6| -13.5|      +6.7|      +1.0|      -2.4|         0|      +3.7|      -0.7|     +41.8|      -1.5|      -28.8|       -1.2
1            4| 1024|   9411.98| 24.04|       391|   1961925|   2292825|         0|        30|   1398947|    965506|   1835604|       2.03|       1.25
2            4| 1024|   9409.97| 26.15|       359|   2072379|   2317492|         0|        34|   1494789|    855245|   1874374|       2.42|       1.24
%             |  0.0|      -0.0|  +8.8|      -8.2|      +5.6|      +1.1|         0|     +13.3|      +6.9|     -11.4|      +2.1|      +19.2|       -0.8
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1092.21| 17.68|        61|   5855912|   1155517|         4|        89|    476428|   4912055|    538322|       1.19|       2.15
2            1|   64|   1031.43| 17.42|        59|   5533459|   1114148|         1|        85|    476884|   4701241|    538789|       1.18|       2.07
%             |  0.0|      -5.6|  -1.5|      -3.3|      -5.5|      -3.6|     -75.0|      -4.5|      +0.1|      -4.3|      +0.1|       -0.8|       -3.7
1            2|   64|   2540.09| 32.76|        77|  13466718|   2509747|         1|       206|    473959|   2026564|    594762|       6.65|       4.22
2            2|   64|   2523.91| 32.72|        77|  13371187|   2486851|         1|       204|    477741|   1909387|    598663|       7.00|       4.15
%             |  0.0|      -0.6|  -0.1|       0.0|      -0.7|      -0.9|       0.0|      -1.0|      +0.8|      -5.8|      +0.7|       +5.3|       -1.7
1            4|   64|   2532.23| 32.96|        76|  13444352|   2555416|         1|       205|    477843|   2048074|    598584|       6.56|       4.27
2            4|   64|   2519.36| 33.10|        76|  13371867|   2530066|         3|       207|    476255|   1917578|    596967|       6.97|       4.24
%             |  0.0|      -0.5|  +0.4|       0.0|      -0.5|      -1.0|    +200.0|      +1.0|      -0.3|      -6.4|      -0.3|       +6.3|       -0.7
1            1|  256|   3481.21| 24.08|       144|  18046440|   1051754|         0|       277|     95147|    115092|    157259|     156.80|       6.69
2            1|  256|   3582.56| 25.00|       143|  18555128|   1079983|         2|       285|     82839|     69704|    144987|     266.20|       7.45
%             |  0.0|      +2.9|  +3.8|      -0.7|      +2.8|      +2.7|         0|      +2.9|     -12.9|     -39.4|      -7.8|      +69.8|      +11.4
1            2|  256|   7835.95| 29.20|       268|   9215513|    988831|         2|     13609|    242276|    235627|    359178|      39.11|       2.75
2            2|  256|   7227.12| 26.14|       276|   8224494|    974662|         0|      3787|    267604|    257681|    362507|      31.92|       2.69
%             |  0.0|      -7.8| -10.5|      +3.0|     -10.8|      -1.4|    -100.0|     -72.2|     +10.5|      +9.4|      +0.9|      -18.4|       -2.2
1            4|  256|   9286.71| 29.59|       313|   3119380|   1125149|      1585|      2794|    365162|    334103|    483517|       9.34|       2.33
2            4|  256|   9365.28| 28.90|       324|   2609986|   1116090|       971|      2064|    373494|    342389|    488519|       7.62|       2.28
%             |  0.0|      +0.8|  -2.3|      +3.5|     -16.3|      -0.8|     -38.7|     -26.1|      +2.3|      +2.5|      +1.0|      -18.4|       -2.1
1            1|  512|   9067.78| 17.76|       510|   2489492|    926754|         0|      3072|    350716|    318289|    415029|       7.82|       2.23
2            1|  512|   9363.32| 17.00|       550|   1239455|    934632|         0|       360|    444912|    419859|    506000|       2.95|       1.85
%             |  0.0|      +3.3|  -4.3|      +7.8|     -50.2|      +0.9|         0|     -88.3|     +26.9|     +31.9|     +21.9|      -62.3|      -17.0
1            2|  512|   8897.24| 20.18|       440|   4677263|    765728|         0|      8964|    312268|    266093|    384872|      17.58|       1.99
2            2|  512|   8994.28| 19.60|       458|   4068494|    777077|         0|      8040|    324208|    279281|    394930|      14.57|       1.97
%             |  0.0|      +1.1|  -2.9|      +4.1|     -13.0|      +1.5|         0|     -10.3|      +3.8|      +5.0|      +2.6|      -17.1|       -1.0
1            4|  512|   8660.59| 22.89|       378|   6274162|   1004294|         0|      9581|    300087|    248234|    381499|      25.28|       2.63
2            4|  512|   8720.95| 22.42|       388|   6321071|    996665|         2|      9307|    293339|    239522|    372874|      26.39|       2.67
%             |  0.0|      +0.7|  -2.1|      +2.6|      +0.7|      -0.8|         0|      -2.9|      -2.2|      -3.5|      -2.3|       +4.4|       +1.5
1            1| 1024|   9382.57| 16.55|       566|   1098014|    943968|         0|        63|    411612|    385473|    463009|       2.85|       2.04
2            1| 1024|   9375.36| 16.46|       569|   1113185|    962141|         0|       109|    434974|    419286|    486685|       2.65|       1.98
%             |  0.0|      -0.1|  -0.5|      +0.5|      +1.4|      +1.9|         0|     +73.0|      +5.7|      +8.8|      +5.1|       -7.0|       -2.9
1            2| 1024|   9341.87| 17.32|       539|   1449472|    807439|         0|       897|    380053|    354028|    433351|       4.09|       1.86
2            2| 1024|   9364.12| 17.13|       546|   1344190|    794110|         0|       581|    379684|    352227|    431694|       3.82|       1.84
%             |  0.0|      +0.2|  -1.1|      +1.3|      -7.3|      -1.7|         0|     -35.2|      -0.1|      -0.5|      -0.4|       -6.6|       -1.1
1            4| 1024|   9391.41| 18.70|       502|   1509582|   1104681|      2166|       658|    399083|    369721|    453543|       4.08|       2.44
2            4| 1024|   9404.67| 18.45|       509|   1361411|   1083647|       386|       398|    400368|    374415|    453718|       3.64|       2.39
%             |  0.0|      +0.1|  -1.3|      +1.4|      -9.8|      -1.9|     -82.2|     -39.5|      +0.3|      +1.3|      +0.0|      -10.8|       -2.0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10874.30|  6.75|      1611|    652525|    652529|         0|        10|    652518|    652582|    663468|       1.00|       0.98
2           50|   64|   9568.49|  9.13|      1048|    574176|    574180|         0|         8|    574166|    574255|    593988|       1.00|       0.97
%             |  0.0|     -12.0| +35.3|     -34.9|     -12.0|     -12.0|         0|     -20.0|     -12.0|     -12.0|     -10.5|        0.0|       -1.0
1          100|   64|  10880.83|  6.77|      1607|    652919|    652926|         0|        10|    652912|    652981|    664406|       1.00|       0.98
2          100|   64|   9586.65|  7.67|      1249|    575267|    575270|         0|         9|    575257|    575326|    593343|       1.00|       0.97
%             |  0.0|     -11.9| +13.3|     -22.3|     -11.9|     -11.9|         0|     -10.0|     -11.9|     -11.9|     -10.7|        0.0|       -1.0
1          250|   64|  10930.11|  6.81|      1605|    655873|    655884|         0|        10|    655869|    655951|    664979|       1.00|       0.99
2          250|   64|   9556.15|  7.65|      1249|    573436|    573440|         0|         9|    573427|    573488|    591151|       1.00|       0.97
%             |  0.0|     -12.6| +12.3|     -22.2|     -12.6|     -12.6|         0|     -10.0|     -12.6|     -12.6|     -11.1|        0.0|       -2.0
1          500|   64|  10930.33|  6.77|      1614|    655886|    655890|         0|        10|    655877|    655949|    664983|       1.00|       0.99
2          500|   64|   9598.61|  8.46|      1134|    575983|    575987|         0|         9|    575975|    576057|    595875|       1.00|       0.97
%             |  0.0|     -12.2| +25.0|     -29.7|     -12.2|     -12.2|         0|     -10.0|     -12.2|     -12.2|     -10.4|        0.0|       -2.0
1           50|  256|  10435.24|  8.87|      1176|    626181|    626184|         0|         9|    626173|    626240|    641870|       1.00|       0.98
2           50|  256|   8948.58| 10.06|       889|    536981|    536985|         0|         8|    536971|    537025|    552906|       1.00|       0.97
%             |  0.0|     -14.2| +13.4|     -24.4|     -14.2|     -14.2|         0|     -11.1|     -14.2|     -14.2|     -13.9|        0.0|       -1.0
1          100|  256|  10401.87|  8.73|      1191|    624179|    624184|         0|        10|    624168|    624248|    640003|       1.00|       0.98
2          100|  256|   9060.39| 10.46|       866|    543692|    543696|         0|         8|    543683|    543752|    559009|       1.00|       0.97
%             |  0.0|     -12.9| +19.8|     -27.3|     -12.9|     -12.9|         0|     -20.0|     -12.9|     -12.9|     -12.7|        0.0|       -1.0
1          250|  256|  10369.91|  8.79|      1179|    622261|    622267|         0|         9|    622252|    622325|    638523|       1.00|       0.97
2          250|  256|   9011.48| 10.88|       828|    540755|    540758|         0|         8|    540743|    540830|    555336|       1.00|       0.97
%             |  0.0|     -13.1| +23.8|     -29.8|     -13.1|     -13.1|         0|     -11.1|     -13.1|     -13.1|     -13.0|        0.0|        0.0
1          500|  256|  10400.42|  8.70|      1195|    624094|    624099|         0|        10|    624085|    624155|    639254|       1.00|       0.98
2          500|  256|   9178.08| 10.83|       847|    550753|    550757|         0|         9|    550744|    550811|    566165|       1.00|       0.97
%             |  0.0|     -11.8| +24.5|     -29.1|     -11.8|     -11.8|         0|     -10.0|     -11.8|     -11.8|     -11.4|        0.0|       -1.0
1           50|  512|   8231.77|  6.22|      1323|    493974|    493978|         0|         7|    493966|    494049|    507228|       1.00|       0.97
2           50|  512|   7718.30|  5.81|      1328|    463164|    463168|         0|         7|    463155|    463218|    478833|       1.00|       0.97
%             |  0.0|      -6.2|  -6.6|      +0.4|      -6.2|      -6.2|         0|       0.0|      -6.2|      -6.2|      -5.6|        0.0|        0.0
1          100|  512|   8267.63|  6.16|      1342|    496125|    496130|         0|         8|    496115|    496186|    509422|       1.00|       0.97
2          100|  512|   7713.78|  8.29|       930|    462894|    462900|         0|         7|    462885|    462977|    488732|       1.00|       0.95
%             |  0.0|      -6.7| +34.6|     -30.7|      -6.7|      -6.7|         0|     -12.5|      -6.7|      -6.7|      -4.1|        0.0|       -2.1
1          250|  512|   8227.23|  6.30|      1305|    493701|    493704|         0|         7|    493691|    493751|    507689|       1.00|       0.97
2          250|  512|   7737.14|  8.35|       926|    464296|    464303|         0|         7|    464291|    464358|    486086|       1.00|       0.96
%             |  0.0|      -6.0| +32.5|     -29.0|      -6.0|      -6.0|         0|       0.0|      -6.0|      -6.0|      -4.3|        0.0|       -1.0
1          500|  512|   8259.80|  6.20|      1332|    495656|    495661|         0|         8|    495649|    495711|    509070|       1.00|       0.97
2          500|  512|   7614.72|  7.47|      1019|    456950|    456959|         0|         7|    456941|    457005|    476914|       1.00|       0.96
%             |  0.0|      -7.8| +20.5|     -23.5|      -7.8|      -7.8|         0|     -12.5|      -7.8|      -7.8|      -6.3|        0.0|       -1.0
1           50| 1024|   7879.43|  6.80|      1158|    472832|    472835|         0|         7|    472822|    472906|    488163|       1.00|       0.97
2           50| 1024|   7302.28|  7.35|       993|    438203|    438207|         0|         7|    438193|    438261|    455627|       1.00|       0.96
%             |  0.0|      -7.3|  +8.1|     -14.2|      -7.3|      -7.3|         0|       0.0|      -7.3|      -7.3|      -6.7|        0.0|       -1.0
1          100| 1024|   7916.07|  6.84|      1157|    475031|    475038|         0|         8|    475024|    475097|    490797|       1.00|       0.97
2          100| 1024|   7491.90|  6.96|      1076|    449582|    449587|         0|         7|    449573|    449627|    462560|       1.00|       0.97
%             |  0.0|      -5.4|  +1.8|      -7.0|      -5.4|      -5.4|         0|     -12.5|      -5.4|      -5.4|      -5.8|        0.0|        0.0
1          250| 1024|   7865.20|  6.82|      1153|    471978|    471984|         0|         7|    471969|    472035|    486978|       1.00|       0.97
2          250| 1024|   7472.83|  7.07|      1056|    448436|    448442|         0|         6|    448431|    448496|    463680|       1.00|       0.97
%             |  0.0|      -5.0|  +3.7|      -8.4|      -5.0|      -5.0|         0|     -14.3|      -5.0|      -5.0|      -4.8|        0.0|        0.0
1          500| 1024|   7931.30|  6.99|      1134|    475944|    475954|         0|         7|    475939|    476016|    491021|       1.00|       0.97
2          500| 1024|   7407.54|  7.57|       978|    444519|    444524|         0|         7|    444509|    444567|    462360|       1.00|       0.96
%             |  0.0|      -6.6|  +8.3|     -13.8|      -6.6|      -6.6|         0|       0.0|      -6.6|      -6.6|      -5.8|        0.0|       -1.0

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:

> Forwarding some results by Amos, who run multiple netperf streams in
> parallel, from an external box to the guest.  TCP_STREAM results were
> noisy.  This could be due to buffering done by TCP, where packet size
> varies even as message size is constant.
> 
> TCP_RR results were consistent. In this benchmark, after switching
> to mandatory barriers, CPU utilization increased by up to 35% while
> throughput went down by up to 14%. the normalized throughput/cpu
> regressed consistently, between 7 and 35%
> 
> The "fix" applied was simply this:

What machine & processor was this  ?

Cheers,
Ben.

> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 3198f2e..fdccb77 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,7 +23,7 @@
>  
>  /* virtio guest is communicating with a virtual "device" that actually runs on
>   * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> +#if 0
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
>   * through relaxed memory I/O windows (which virtio does not use). */
> 
> 
> 

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:

> Forwarding some results by Amos, who run multiple netperf streams in
> parallel, from an external box to the guest.  TCP_STREAM results were
> noisy.  This could be due to buffering done by TCP, where packet size
> varies even as message size is constant.
> 
> TCP_RR results were consistent. In this benchmark, after switching
> to mandatory barriers, CPU utilization increased by up to 35% while
> throughput went down by up to 14%. the normalized throughput/cpu
> regressed consistently, between 7 and 35%
> 
> The "fix" applied was simply this:

What machine & processor was this  ?

Cheers,
Ben.

> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 3198f2e..fdccb77 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,7 +23,7 @@
>  
>  /* virtio guest is communicating with a virtual "device" that actually runs on
>   * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> +#if 0
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
>   * through relaxed memory I/O windows (which virtio does not use). */
> 
> 
> 

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:

> Forwarding some results by Amos, who run multiple netperf streams in
> parallel, from an external box to the guest.  TCP_STREAM results were
> noisy.  This could be due to buffering done by TCP, where packet size
> varies even as message size is constant.
> 
> TCP_RR results were consistent. In this benchmark, after switching
> to mandatory barriers, CPU utilization increased by up to 35% while
> throughput went down by up to 14%. the normalized throughput/cpu
> regressed consistently, between 7 and 35%
> 
> The "fix" applied was simply this:

What machine & processor was this  ?

Cheers,
Ben.

> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 3198f2e..fdccb77 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -23,7 +23,7 @@
>  
>  /* virtio guest is communicating with a virtual "device" that actually runs on
>   * a host processor.  Memory barriers are used to control SMP effects. */
> -#ifdef CONFIG_SMP
> +#if 0
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
>   * through relaxed memory I/O windows (which virtio does not use). */
> 
> 
> 

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>
>> Forwarding some results by Amos, who run multiple netperf streams in
>> parallel, from an external box to the guest.  TCP_STREAM results were
>> noisy.  This could be due to buffering done by TCP, where packet size
>> varies even as message size is constant.
>>
>> TCP_RR results were consistent. In this benchmark, after switching
>> to mandatory barriers, CPU utilization increased by up to 35% while
>> throughput went down by up to 14%. the normalized throughput/cpu
>> regressed consistently, between 7 and 35%
>>
>> The "fix" applied was simply this:
>
> What machine&  processor was this  ?

pined guest memory to numa node 1
# numactl -m 1 qemu-kvm ...
pined guest vcpu threads and vhost thread to single cpu of numa node 1
# taskset -p 0x10 8348 (vhost_net_thread)
# taskset -p 0x20 8353 (vcpu 1 thread)
# taskset -p 0x40 8357 (vcpu 2 thread)
pined cpu/memory of netperf client process to node 1
# numactl --cpunodebind=1 --membind=1 netperf ...

8 cores
-------
processor	: 7
vendor_id	: GenuineIntel
cpu family	: 6
model		: 44
model name	: Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz
stepping	: 2
microcode	: 0xc
cpu MHz		: 1596.000
cache size	: 12288 KB
physical id	: 1
siblings	: 4
core id		: 10
cpu cores	: 4
apicid		: 52
initial apicid	: 52
fpu		: yes
fpu_exception	: yes
cpuid level	: 11
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov 
pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx 
pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl 
xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx 
smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes 
lahf_lm ida arat epb dts tpr_shadow vnmi flexpriority ept vpid
bogomips	: 4787.76
clflush size	: 64
cache_alignment	: 64
address sizes	: 40 bits physical, 48 bits virtual
power management:

# cat /proc/meminfo
MemTotal:       16446616 kB
MemFree:        15874092 kB
Buffers:           30404 kB
Cached:           238640 kB
SwapCached:            0 kB
Active:           100204 kB
Inactive:         184312 kB
Active(anon):      15724 kB
Inactive(anon):        4 kB
Active(file):      84480 kB
Inactive(file):   184308 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:       8388604 kB
SwapFree:        8388604 kB
Dirty:                56 kB
Writeback:             0 kB
AnonPages:         15548 kB
Mapped:            11540 kB
Shmem:               256 kB
Slab:              82444 kB
SReclaimable:      19220 kB
SUnreclaim:        63224 kB
KernelStack:        1224 kB
PageTables:         2256 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:    16611912 kB
Committed_AS:     209068 kB
VmallocTotal:   34359738367 kB
VmallocUsed:      224244 kB
VmallocChunk:   34351073668 kB
HardwareCorrupted:     0 kB
AnonHugePages:         0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:        9876 kB
DirectMap2M:     2070528 kB
DirectMap1G:    14680064 kB

# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3
node 0 size: 8175 MB
node 0 free: 7706 MB
node 1 cpus: 4 5 6 7
node 1 size: 8192 MB
node 1 free: 7796 MB
node distances:
node   0   1
   0:  10  20
   1:  20  10
# numactl --show
policy: default
preferred node: current
physcpubind: 0 1 2 3 4 5 6 7
cpubind: 0 1
nodebind: 0 1
membind: 0 1


> Cheers,
> Ben.
>
>> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
>> index 3198f2e..fdccb77 100644
>> --- a/drivers/virtio/virtio_ring.c
>> +++ b/drivers/virtio/virtio_ring.c
>> @@ -23,7 +23,7 @@
>>
>>   /* virtio guest is communicating with a virtual "device" that actually runs on
>>    * a host processor.  Memory barriers are used to control SMP effects. */
>> -#ifdef CONFIG_SMP
>> +#if 0
>>   /* Where possible, use SMP barriers which are more lightweight than mandatory
>>    * barriers, because mandatory barriers control MMIO effects on accesses
>>    * through relaxed memory I/O windows (which virtio does not use). */
>>
>>
>>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>
>> Forwarding some results by Amos, who run multiple netperf streams in
>> parallel, from an external box to the guest.  TCP_STREAM results were
>> noisy.  This could be due to buffering done by TCP, where packet size
>> varies even as message size is constant.
>>
>> TCP_RR results were consistent. In this benchmark, after switching
>> to mandatory barriers, CPU utilization increased by up to 35% while
>> throughput went down by up to 14%. the normalized throughput/cpu
>> regressed consistently, between 7 and 35%
>>
>> The "fix" applied was simply this:
>
> What machine&  processor was this  ?

pined guest memory to numa node 1
# numactl -m 1 qemu-kvm ...
pined guest vcpu threads and vhost thread to single cpu of numa node 1
# taskset -p 0x10 8348 (vhost_net_thread)
# taskset -p 0x20 8353 (vcpu 1 thread)
# taskset -p 0x40 8357 (vcpu 2 thread)
pined cpu/memory of netperf client process to node 1
# numactl --cpunodebind=1 --membind=1 netperf ...

8 cores
-------
processor	: 7
vendor_id	: GenuineIntel
cpu family	: 6
model		: 44
model name	: Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz
stepping	: 2
microcode	: 0xc
cpu MHz		: 1596.000
cache size	: 12288 KB
physical id	: 1
siblings	: 4
core id		: 10
cpu cores	: 4
apicid		: 52
initial apicid	: 52
fpu		: yes
fpu_exception	: yes
cpuid level	: 11
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov 
pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx 
pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl 
xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx 
smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes 
lahf_lm ida arat epb dts tpr_shadow vnmi flexpriority ept vpid
bogomips	: 4787.76
clflush size	: 64
cache_alignment	: 64
address sizes	: 40 bits physical, 48 bits virtual
power management:

# cat /proc/meminfo
MemTotal:       16446616 kB
MemFree:        15874092 kB
Buffers:           30404 kB
Cached:           238640 kB
SwapCached:            0 kB
Active:           100204 kB
Inactive:         184312 kB
Active(anon):      15724 kB
Inactive(anon):        4 kB
Active(file):      84480 kB
Inactive(file):   184308 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:       8388604 kB
SwapFree:        8388604 kB
Dirty:                56 kB
Writeback:             0 kB
AnonPages:         15548 kB
Mapped:            11540 kB
Shmem:               256 kB
Slab:              82444 kB
SReclaimable:      19220 kB
SUnreclaim:        63224 kB
KernelStack:        1224 kB
PageTables:         2256 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:    16611912 kB
Committed_AS:     209068 kB
VmallocTotal:   34359738367 kB
VmallocUsed:      224244 kB
VmallocChunk:   34351073668 kB
HardwareCorrupted:     0 kB
AnonHugePages:         0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:        9876 kB
DirectMap2M:     2070528 kB
DirectMap1G:    14680064 kB

# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3
node 0 size: 8175 MB
node 0 free: 7706 MB
node 1 cpus: 4 5 6 7
node 1 size: 8192 MB
node 1 free: 7796 MB
node distances:
node   0   1
   0:  10  20
   1:  20  10
# numactl --show
policy: default
preferred node: current
physcpubind: 0 1 2 3 4 5 6 7
cpubind: 0 1
nodebind: 0 1
membind: 0 1


> Cheers,
> Ben.
>
>> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
>> index 3198f2e..fdccb77 100644
>> --- a/drivers/virtio/virtio_ring.c
>> +++ b/drivers/virtio/virtio_ring.c
>> @@ -23,7 +23,7 @@
>>
>>   /* virtio guest is communicating with a virtual "device" that actually runs on
>>    * a host processor.  Memory barriers are used to control SMP effects. */
>> -#ifdef CONFIG_SMP
>> +#if 0
>>   /* Where possible, use SMP barriers which are more lightweight than mandatory
>>    * barriers, because mandatory barriers control MMIO effects on accesses
>>    * through relaxed memory I/O windows (which virtio does not use). */
>>
>>
>>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>
>> Forwarding some results by Amos, who run multiple netperf streams in
>> parallel, from an external box to the guest.  TCP_STREAM results were
>> noisy.  This could be due to buffering done by TCP, where packet size
>> varies even as message size is constant.
>>
>> TCP_RR results were consistent. In this benchmark, after switching
>> to mandatory barriers, CPU utilization increased by up to 35% while
>> throughput went down by up to 14%. the normalized throughput/cpu
>> regressed consistently, between 7 and 35%
>>
>> The "fix" applied was simply this:
>
> What machine&  processor was this  ?

pined guest memory to numa node 1
# numactl -m 1 qemu-kvm ...
pined guest vcpu threads and vhost thread to single cpu of numa node 1
# taskset -p 0x10 8348 (vhost_net_thread)
# taskset -p 0x20 8353 (vcpu 1 thread)
# taskset -p 0x40 8357 (vcpu 2 thread)
pined cpu/memory of netperf client process to node 1
# numactl --cpunodebind=1 --membind=1 netperf ...

8 cores
-------
processor	: 7
vendor_id	: GenuineIntel
cpu family	: 6
model		: 44
model name	: Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz
stepping	: 2
microcode	: 0xc
cpu MHz		: 1596.000
cache size	: 12288 KB
physical id	: 1
siblings	: 4
core id		: 10
cpu cores	: 4
apicid		: 52
initial apicid	: 52
fpu		: yes
fpu_exception	: yes
cpuid level	: 11
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov 
pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx 
pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl 
xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx 
smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes 
lahf_lm ida arat epb dts tpr_shadow vnmi flexpriority ept vpid
bogomips	: 4787.76
clflush size	: 64
cache_alignment	: 64
address sizes	: 40 bits physical, 48 bits virtual
power management:

# cat /proc/meminfo
MemTotal:       16446616 kB
MemFree:        15874092 kB
Buffers:           30404 kB
Cached:           238640 kB
SwapCached:            0 kB
Active:           100204 kB
Inactive:         184312 kB
Active(anon):      15724 kB
Inactive(anon):        4 kB
Active(file):      84480 kB
Inactive(file):   184308 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:       8388604 kB
SwapFree:        8388604 kB
Dirty:                56 kB
Writeback:             0 kB
AnonPages:         15548 kB
Mapped:            11540 kB
Shmem:               256 kB
Slab:              82444 kB
SReclaimable:      19220 kB
SUnreclaim:        63224 kB
KernelStack:        1224 kB
PageTables:         2256 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:    16611912 kB
Committed_AS:     209068 kB
VmallocTotal:   34359738367 kB
VmallocUsed:      224244 kB
VmallocChunk:   34351073668 kB
HardwareCorrupted:     0 kB
AnonHugePages:         0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:        9876 kB
DirectMap2M:     2070528 kB
DirectMap1G:    14680064 kB

# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3
node 0 size: 8175 MB
node 0 free: 7706 MB
node 1 cpus: 4 5 6 7
node 1 size: 8192 MB
node 1 free: 7796 MB
node distances:
node   0   1
   0:  10  20
   1:  20  10
# numactl --show
policy: default
preferred node: current
physcpubind: 0 1 2 3 4 5 6 7
cpubind: 0 1
nodebind: 0 1
membind: 0 1


> Cheers,
> Ben.
>
>> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
>> index 3198f2e..fdccb77 100644
>> --- a/drivers/virtio/virtio_ring.c
>> +++ b/drivers/virtio/virtio_ring.c
>> @@ -23,7 +23,7 @@
>>
>>   /* virtio guest is communicating with a virtual "device" that actually runs on
>>    * a host processor.  Memory barriers are used to control SMP effects. */
>> -#ifdef CONFIG_SMP
>> +#if 0
>>   /* Where possible, use SMP barriers which are more lightweight than mandatory
>>    * barriers, because mandatory barriers control MMIO effects on accesses
>>    * through relaxed memory I/O windows (which virtio does not use). */
>>
>>
>>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> > On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >
> >> Forwarding some results by Amos, who run multiple netperf streams in
> >> parallel, from an external box to the guest.  TCP_STREAM results were
> >> noisy.  This could be due to buffering done by TCP, where packet size
> >> varies even as message size is constant.
> >>
> >> TCP_RR results were consistent. In this benchmark, after switching
> >> to mandatory barriers, CPU utilization increased by up to 35% while
> >> throughput went down by up to 14%. the normalized throughput/cpu
> >> regressed consistently, between 7 and 35%
> >>
> >> The "fix" applied was simply this:
> >
> > What machine&  processor was this  ?
> 
> pined guest memory to numa node 1

Please try this patch.  How much does the branch cost us?

(Compiles, untested).

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> > On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >
> >> Forwarding some results by Amos, who run multiple netperf streams in
> >> parallel, from an external box to the guest.  TCP_STREAM results were
> >> noisy.  This could be due to buffering done by TCP, where packet size
> >> varies even as message size is constant.
> >>
> >> TCP_RR results were consistent. In this benchmark, after switching
> >> to mandatory barriers, CPU utilization increased by up to 35% while
> >> throughput went down by up to 14%. the normalized throughput/cpu
> >> regressed consistently, between 7 and 35%
> >>
> >> The "fix" applied was simply this:
> >
> > What machine&  processor was this  ?
> 
> pined guest memory to numa node 1

Please try this patch.  How much does the branch cost us?

(Compiles, untested).

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> > On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >
> >> Forwarding some results by Amos, who run multiple netperf streams in
> >> parallel, from an external box to the guest.  TCP_STREAM results were
> >> noisy.  This could be due to buffering done by TCP, where packet size
> >> varies even as message size is constant.
> >>
> >> TCP_RR results were consistent. In this benchmark, after switching
> >> to mandatory barriers, CPU utilization increased by up to 35% while
> >> throughput went down by up to 14%. the normalized throughput/cpu
> >> regressed consistently, between 7 and 35%
> >>
> >> The "fix" applied was simply this:
> >
> > What machine&  processor was this  ?
> 
> pined guest memory to numa node 1

Please try this patch.  How much does the branch cost us?

(Compiles, untested).

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/2011 01:12 PM, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&  processor was this  ?
>>
>> pined guest memory to numa node 1
> 
> Please try this patch.  How much does the branch cost us?

Ok, I will provide the result later.

Thanks,
Amos

> (Compiles, untested).
> 
> Thanks,
> Rusty.
> 
> From: Rusty Russell <rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
> 
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
> 
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
> 
> By comparison, this branch costs us???
> 
> Reference: https://lkml.org/lkml/2011/12/11/22
> 
> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
> ---
>  drivers/lguest/lguest_device.c |   10 ++++++----
>  drivers/s390/kvm/kvm_virtio.c  |    2 +-
>  drivers/virtio/virtio_mmio.c   |    7 ++++---
>  drivers/virtio/virtio_pci.c    |    4 ++--
>  drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>  include/linux/virtio_ring.h    |    1 +
>  tools/virtio/linux/virtio.h    |    1 +
>  tools/virtio/virtio_test.c     |    3 ++-
>  8 files changed, 38 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
> --- a/drivers/lguest/lguest_device.c
> +++ b/drivers/lguest/lguest_device.c
> @@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
>  	}
>  
>  	/*
> -	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> -	 * and we've got a pointer to its pages.
> +	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> +	 * and we've got a pointer to its pages.  Note that we set weak_barriers
> +	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
> +	 * barriers.
>  	 */
> -	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
> -				 vdev, lvq->pages, lg_notify, callback, name);
> +	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
> +				 true, lvq->pages, lg_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto unmap;
> diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
> --- a/drivers/s390/kvm/kvm_virtio.c
> +++ b/drivers/s390/kvm/kvm_virtio.c
> @@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
>  		goto out;
>  
>  	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
> -				 vdev, (void *) config->address,
> +				 vdev, true, (void *) config->address,
>  				 kvm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
>  	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
>  			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
>  
> -	/* Create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
> -				 vdev, info->queue, vm_notify, callback, name);
> +	/* Create the vring: no weak barriers, the other side is could
> +	 * be an independent "device". */
> +	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
> +				 false, info->queue, vm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto error_new_virtqueue;
> diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
> --- a/drivers/virtio/virtio_pci.c
> +++ b/drivers/virtio/virtio_pci.c
> @@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
>  		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
>  
>  	/* create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
> -				 vdev, info->queue, vp_notify, callback, name);
> +	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
> +				 true, info->queue, vp_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto out_activate_queue;
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -28,17 +28,20 @@
>  #ifdef CONFIG_SMP
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> + * through relaxed memory I/O windows (which virtio-pci does not use). */
> +#define virtio_mb(vq) \
> +	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
> +#define virtio_rmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
> +#define virtio_wmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
>  #else
>  /* We must force memory ordering even if guest is UP since host could be
>   * running on another CPU, but SMP barriers are defined to barrier() in that
>   * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> +#define virtio_mb(vq) mb()
> +#define virtio_rmb(vq) rmb()
> +#define virtio_wmb(vq) wmb()
>  #endif
>  
>  #ifdef DEBUG
> @@ -77,6 +80,9 @@ struct vring_virtqueue
>  	/* Actual memory layout for this queue */
>  	struct vring vring;
>  
> +	/* Can we use weak barriers? */
> +	bool weak_barriers;
> +
>  	/* Other side has made a mess, don't try any more. */
>  	bool broken;
>  
> @@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *),
>  				      void (*callback)(struct virtqueue *),
> @@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
>  	vq->vq.vdev = vdev;
>  	vq->vq.name = name;
>  	vq->notify = notify;
> +	vq->weak_barriers = weak_barriers;
>  	vq->broken = false;
>  	vq->last_used_idx = 0;
>  	vq->num_added = 0;
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -168,6 +168,7 @@ struct virtqueue;
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
> --- a/tools/virtio/linux/virtio.h
> +++ b/tools/virtio/linux/virtio.h
> @@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
> --- a/tools/virtio/virtio_test.c
> +++ b/tools/virtio/virtio_test.c
> @@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
>  	assert(r >= 0);
>  	memset(info->ring, 0, vring_size(num, 4096));
>  	vring_init(&info->vring, num, info->ring, 4096);
> -	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
> +	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
> +				       true, info->ring,
>  				       vq_notify, vq_callback, "test");
>  	assert(info->vq);
>  	info->vq->priv = info;
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/2011 01:12 PM, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&  processor was this  ?
>>
>> pined guest memory to numa node 1
> 
> Please try this patch.  How much does the branch cost us?

Ok, I will provide the result later.

Thanks,
Amos

> (Compiles, untested).
> 
> Thanks,
> Rusty.
> 
> From: Rusty Russell <rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
> 
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
> 
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
> 
> By comparison, this branch costs us???
> 
> Reference: https://lkml.org/lkml/2011/12/11/22
> 
> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
> ---
>  drivers/lguest/lguest_device.c |   10 ++++++----
>  drivers/s390/kvm/kvm_virtio.c  |    2 +-
>  drivers/virtio/virtio_mmio.c   |    7 ++++---
>  drivers/virtio/virtio_pci.c    |    4 ++--
>  drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>  include/linux/virtio_ring.h    |    1 +
>  tools/virtio/linux/virtio.h    |    1 +
>  tools/virtio/virtio_test.c     |    3 ++-
>  8 files changed, 38 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
> --- a/drivers/lguest/lguest_device.c
> +++ b/drivers/lguest/lguest_device.c
> @@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
>  	}
>  
>  	/*
> -	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> -	 * and we've got a pointer to its pages.
> +	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> +	 * and we've got a pointer to its pages.  Note that we set weak_barriers
> +	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
> +	 * barriers.
>  	 */
> -	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
> -				 vdev, lvq->pages, lg_notify, callback, name);
> +	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
> +				 true, lvq->pages, lg_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto unmap;
> diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
> --- a/drivers/s390/kvm/kvm_virtio.c
> +++ b/drivers/s390/kvm/kvm_virtio.c
> @@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
>  		goto out;
>  
>  	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
> -				 vdev, (void *) config->address,
> +				 vdev, true, (void *) config->address,
>  				 kvm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
>  	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
>  			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
>  
> -	/* Create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
> -				 vdev, info->queue, vm_notify, callback, name);
> +	/* Create the vring: no weak barriers, the other side is could
> +	 * be an independent "device". */
> +	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
> +				 false, info->queue, vm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto error_new_virtqueue;
> diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
> --- a/drivers/virtio/virtio_pci.c
> +++ b/drivers/virtio/virtio_pci.c
> @@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
>  		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
>  
>  	/* create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
> -				 vdev, info->queue, vp_notify, callback, name);
> +	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
> +				 true, info->queue, vp_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto out_activate_queue;
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -28,17 +28,20 @@
>  #ifdef CONFIG_SMP
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> + * through relaxed memory I/O windows (which virtio-pci does not use). */
> +#define virtio_mb(vq) \
> +	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
> +#define virtio_rmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
> +#define virtio_wmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
>  #else
>  /* We must force memory ordering even if guest is UP since host could be
>   * running on another CPU, but SMP barriers are defined to barrier() in that
>   * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> +#define virtio_mb(vq) mb()
> +#define virtio_rmb(vq) rmb()
> +#define virtio_wmb(vq) wmb()
>  #endif
>  
>  #ifdef DEBUG
> @@ -77,6 +80,9 @@ struct vring_virtqueue
>  	/* Actual memory layout for this queue */
>  	struct vring vring;
>  
> +	/* Can we use weak barriers? */
> +	bool weak_barriers;
> +
>  	/* Other side has made a mess, don't try any more. */
>  	bool broken;
>  
> @@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *),
>  				      void (*callback)(struct virtqueue *),
> @@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
>  	vq->vq.vdev = vdev;
>  	vq->vq.name = name;
>  	vq->notify = notify;
> +	vq->weak_barriers = weak_barriers;
>  	vq->broken = false;
>  	vq->last_used_idx = 0;
>  	vq->num_added = 0;
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -168,6 +168,7 @@ struct virtqueue;
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
> --- a/tools/virtio/linux/virtio.h
> +++ b/tools/virtio/linux/virtio.h
> @@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
> --- a/tools/virtio/virtio_test.c
> +++ b/tools/virtio/virtio_test.c
> @@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
>  	assert(r >= 0);
>  	memset(info->ring, 0, vring_size(num, 4096));
>  	vring_init(&info->vring, num, info->ring, 4096);
> -	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
> +	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
> +				       true, info->ring,
>  				       vq_notify, vq_callback, "test");
>  	assert(info->vq);
>  	info->vq->priv = info;
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/2011 01:12 PM, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&  processor was this  ?
>>
>> pined guest memory to numa node 1
> 
> Please try this patch.  How much does the branch cost us?

Ok, I will provide the result later.

Thanks,
Amos

> (Compiles, untested).
> 
> Thanks,
> Rusty.
> 
> From: Rusty Russell <rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
> 
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
> 
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
> 
> By comparison, this branch costs us???
> 
> Reference: https://lkml.org/lkml/2011/12/11/22
> 
> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
> ---
>  drivers/lguest/lguest_device.c |   10 ++++++----
>  drivers/s390/kvm/kvm_virtio.c  |    2 +-
>  drivers/virtio/virtio_mmio.c   |    7 ++++---
>  drivers/virtio/virtio_pci.c    |    4 ++--
>  drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>  include/linux/virtio_ring.h    |    1 +
>  tools/virtio/linux/virtio.h    |    1 +
>  tools/virtio/virtio_test.c     |    3 ++-
>  8 files changed, 38 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
> --- a/drivers/lguest/lguest_device.c
> +++ b/drivers/lguest/lguest_device.c
> @@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
>  	}
>  
>  	/*
> -	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> -	 * and we've got a pointer to its pages.
> +	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
> +	 * and we've got a pointer to its pages.  Note that we set weak_barriers
> +	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
> +	 * barriers.
>  	 */
> -	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
> -				 vdev, lvq->pages, lg_notify, callback, name);
> +	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
> +				 true, lvq->pages, lg_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto unmap;
> diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
> --- a/drivers/s390/kvm/kvm_virtio.c
> +++ b/drivers/s390/kvm/kvm_virtio.c
> @@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
>  		goto out;
>  
>  	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
> -				 vdev, (void *) config->address,
> +				 vdev, true, (void *) config->address,
>  				 kvm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
>  	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
>  			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
>  
> -	/* Create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
> -				 vdev, info->queue, vm_notify, callback, name);
> +	/* Create the vring: no weak barriers, the other side is could
> +	 * be an independent "device". */
> +	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
> +				 false, info->queue, vm_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto error_new_virtqueue;
> diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
> --- a/drivers/virtio/virtio_pci.c
> +++ b/drivers/virtio/virtio_pci.c
> @@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
>  		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
>  
>  	/* create the vring */
> -	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
> -				 vdev, info->queue, vp_notify, callback, name);
> +	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
> +				 true, info->queue, vp_notify, callback, name);
>  	if (!vq) {
>  		err = -ENOMEM;
>  		goto out_activate_queue;
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -28,17 +28,20 @@
>  #ifdef CONFIG_SMP
>  /* Where possible, use SMP barriers which are more lightweight than mandatory
>   * barriers, because mandatory barriers control MMIO effects on accesses
> - * through relaxed memory I/O windows (which virtio does not use). */
> -#define virtio_mb() smp_mb()
> -#define virtio_rmb() smp_rmb()
> -#define virtio_wmb() smp_wmb()
> + * through relaxed memory I/O windows (which virtio-pci does not use). */
> +#define virtio_mb(vq) \
> +	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
> +#define virtio_rmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
> +#define virtio_wmb(vq) \
> +	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
>  #else
>  /* We must force memory ordering even if guest is UP since host could be
>   * running on another CPU, but SMP barriers are defined to barrier() in that
>   * configuration. So fall back to mandatory barriers instead. */
> -#define virtio_mb() mb()
> -#define virtio_rmb() rmb()
> -#define virtio_wmb() wmb()
> +#define virtio_mb(vq) mb()
> +#define virtio_rmb(vq) rmb()
> +#define virtio_wmb(vq) wmb()
>  #endif
>  
>  #ifdef DEBUG
> @@ -77,6 +80,9 @@ struct vring_virtqueue
>  	/* Actual memory layout for this queue */
>  	struct vring vring;
>  
> +	/* Can we use weak barriers? */
> +	bool weak_barriers;
> +
>  	/* Other side has made a mess, don't try any more. */
>  	bool broken;
>  
> @@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
>  	START_USE(vq);
>  	/* Descriptors and available array need to be set before we expose the
>  	 * new available array entries. */
> -	virtio_wmb();
> +	virtio_wmb(vq);
>  
>  	old = vq->vring.avail->idx;
>  	new = vq->vring.avail->idx = old + vq->num_added;
>  	vq->num_added = 0;
>  
>  	/* Need to update avail index before checking if we should notify */
> -	virtio_mb();
> +	virtio_mb(vq);
>  
>  	if (vq->event ?
>  	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
> @@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	}
>  
>  	/* Only get used array entries after they have been exposed by host. */
> -	virtio_rmb();
> +	virtio_rmb(vq);
>  
>  	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
>  	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
> @@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
>  	 * the read in the next get_buf call. */
>  	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
>  		vring_used_event(&vq->vring) = vq->last_used_idx;
> -		virtio_mb();
> +		virtio_mb(vq);
>  	}
>  
>  	END_USE(vq);
> @@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
>  	 * entry. Always do both to keep code simple. */
>  	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
>  	vring_used_event(&vq->vring) = vq->last_used_idx;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely(more_used(vq))) {
>  		END_USE(vq);
>  		return false;
> @@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
>  	/* TODO: tune this threshold */
>  	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
>  	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
> -	virtio_mb();
> +	virtio_mb(vq);
>  	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
>  		END_USE(vq);
>  		return false;
> @@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *),
>  				      void (*callback)(struct virtqueue *),
> @@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
>  	vq->vq.vdev = vdev;
>  	vq->vq.name = name;
>  	vq->notify = notify;
> +	vq->weak_barriers = weak_barriers;
>  	vq->broken = false;
>  	vq->last_used_idx = 0;
>  	vq->num_added = 0;
> diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
> --- a/include/linux/virtio_ring.h
> +++ b/include/linux/virtio_ring.h
> @@ -168,6 +168,7 @@ struct virtqueue;
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
> --- a/tools/virtio/linux/virtio.h
> +++ b/tools/virtio/linux/virtio.h
> @@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
>  struct virtqueue *vring_new_virtqueue(unsigned int num,
>  				      unsigned int vring_align,
>  				      struct virtio_device *vdev,
> +				      bool weak_barriers,
>  				      void *pages,
>  				      void (*notify)(struct virtqueue *vq),
>  				      void (*callback)(struct virtqueue *vq),
> diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
> --- a/tools/virtio/virtio_test.c
> +++ b/tools/virtio/virtio_test.c
> @@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
>  	assert(r >= 0);
>  	memset(info->ring, 0, vring_size(num, 4096));
>  	vring_init(&info->vring, num, info->ring, 4096);
> -	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
> +	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
> +				       true, info->ring,
>  				       vq_notify, vq_callback, "test");
>  	assert(info->vq);
>  	info->vq->priv = info;
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Tue, 13 Dec 2011 07:56:36 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/2011 01:12 PM, Rusty Russell wrote:
> > On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> >> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> >>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >>>
> >>>> Forwarding some results by Amos, who run multiple netperf streams in
> >>>> parallel, from an external box to the guest.  TCP_STREAM results were
> >>>> noisy.  This could be due to buffering done by TCP, where packet size
> >>>> varies even as message size is constant.
> >>>>
> >>>> TCP_RR results were consistent. In this benchmark, after switching
> >>>> to mandatory barriers, CPU utilization increased by up to 35% while
> >>>> throughput went down by up to 14%. the normalized throughput/cpu
> >>>> regressed consistently, between 7 and 35%
> >>>>
> >>>> The "fix" applied was simply this:
> >>>
> >>> What machine&  processor was this  ?
> >>
> >> pined guest memory to numa node 1
> > 
> > Please try this patch.  How much does the branch cost us?
> 
> Ok, I will provide the result later.

Any news?  We're cutting it very fine.  I've CC'd Linus so he knows this
is coming...

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Tue, 13 Dec 2011 07:56:36 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/2011 01:12 PM, Rusty Russell wrote:
> > On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> >> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> >>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >>>
> >>>> Forwarding some results by Amos, who run multiple netperf streams in
> >>>> parallel, from an external box to the guest.  TCP_STREAM results were
> >>>> noisy.  This could be due to buffering done by TCP, where packet size
> >>>> varies even as message size is constant.
> >>>>
> >>>> TCP_RR results were consistent. In this benchmark, after switching
> >>>> to mandatory barriers, CPU utilization increased by up to 35% while
> >>>> throughput went down by up to 14%. the normalized throughput/cpu
> >>>> regressed consistently, between 7 and 35%
> >>>>
> >>>> The "fix" applied was simply this:
> >>>
> >>> What machine&  processor was this  ?
> >>
> >> pined guest memory to numa node 1
> > 
> > Please try this patch.  How much does the branch cost us?
> 
> Ok, I will provide the result later.

Any news?  We're cutting it very fine.  I've CC'd Linus so he knows this
is coming...

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Tue, 13 Dec 2011 07:56:36 +0800, Amos Kong <akong@redhat.com> wrote:
> On 12/12/2011 01:12 PM, Rusty Russell wrote:
> > On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong <akong@redhat.com> wrote:
> >> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
> >>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
> >>>
> >>>> Forwarding some results by Amos, who run multiple netperf streams in
> >>>> parallel, from an external box to the guest.  TCP_STREAM results were
> >>>> noisy.  This could be due to buffering done by TCP, where packet size
> >>>> varies even as message size is constant.
> >>>>
> >>>> TCP_RR results were consistent. In this benchmark, after switching
> >>>> to mandatory barriers, CPU utilization increased by up to 35% while
> >>>> throughput went down by up to 14%. the normalized throughput/cpu
> >>>> regressed consistently, between 7 and 35%
> >>>>
> >>>> The "fix" applied was simply this:
> >>>
> >>> What machine&  processor was this  ?
> >>
> >> pined guest memory to numa node 1
> > 
> > Please try this patch.  How much does the branch cost us?
> 
> Ok, I will provide the result later.

Any news?  We're cutting it very fine.  I've CC'd Linus so he knows this
is coming...

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch costs us???

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 13:12, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com>  wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&   processor was this  ?
>>
>> pined guest memory to numa node 1
>
> Please try this patch.  How much does the branch cost us?
>
> (Compiles, untested).
>
> Thanks,
> Rusty.
>
> From: Rusty Russell<rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
>
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
>
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
>
> By comparison, this branch costs us???
>
> Reference: https://lkml.org/lkml/2011/12/11/22
>
> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
> ---
>   drivers/lguest/lguest_device.c |   10 ++++++----
>   drivers/s390/kvm/kvm_virtio.c  |    2 +-
>   drivers/virtio/virtio_mmio.c   |    7 ++++---
>   drivers/virtio/virtio_pci.c    |    4 ++--
>   drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>   include/linux/virtio_ring.h    |    1 +
>   tools/virtio/linux/virtio.h    |    1 +
>   tools/virtio/virtio_test.c     |    3 ++-
>   8 files changed, 38 insertions(+), 24 deletions(-)

Hi all,

I tested with the same environment and scenarios.
tested one scenarios for three times and compute the average for more 
precision.

Thanks, Amos

--------- compare results -----------
Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.exhost_guest.txt
2 - avg-fixed.netperf.exhost_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1073.54| 10.50|       102|         0|        31| 
         0|      1612|         0|        16|    487641|    489753| 
504764|       0.00|       0.00
2            1|   64|   1079.44| 10.29|       104|         0|        30| 
         0|      1594|         0|        17|    487156|    488828| 
504411|       0.00|       0.00
%             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
         0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
-0.1|          0|          0
1            2|   64|   2141.12| 15.72|       136|         0|        33| 
         0|      1744|         0|        34|    873777|    972303| 
928926|       0.00|       0.00
2            2|   64|   2140.88| 15.64|       137|         0|        33| 
         0|      1744|         0|        34|    926588|    942841| 
974095|       0.00|       0.00
%             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +6.0|      -3.0| 
+4.9|          0|          0
1            4|   64|   4076.80| 19.82|       205|         0|        30| 
         0|      1577|         0|        67|   1422282|   1166425| 
1539219|       0.00|       0.00
2            4|   64|   4094.32| 20.70|       197|         0|        31| 
         0|      1612|         0|        68|   1704330|   1314077| 
1833394|       0.00|       0.00
%             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
         0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
+19.1|          0|          0
1            1|  256|   2867.48| 13.44|       213|         0|        32| 
         0|      1726|         0|        14|    666430|    694922| 
690730|       0.00|       0.00
2            1|  256|   2874.20| 12.71|       226|         0|        32| 
         0|      1709|         0|        14|    697960|    740407| 
721807|       0.00|       0.00
%             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
         0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
+4.5|          0|          0
1            2|  256|   5642.82| 17.61|       320|         0|        30| 
         0|      1594|         0|        30|   1226861|   1236081| 
1268562|       0.00|       0.00
2            2|  256|   5661.06| 17.41|       326|         0|        30| 
         0|      1594|         0|        29|   1175696|   1143490| 
1221528|       0.00|       0.00
%             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
         0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
-3.7|          0|          0
1            4|  256|   9404.27| 23.55|       399|         0|        33| 
         0|      1744|         0|        37|   1692245|    659975| 
1765103|       0.00|       0.00
2            4|  256|   8761.11| 23.18|       376|         0|        32| 
         0|      1726|         0|        36|   1699382|    418992| 
1870804|       0.00|       0.00
%             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
         0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
+6.0|          0|          0
1            1|  512|   3803.66| 14.20|       267|         0|        30| 
         0|      1594|         0|        14|    693992|    750078| 
721107|       0.00|       0.00
2            1|  512|   3838.02| 15.47|       248|         0|        31| 
         0|      1612|         0|        15|    811709|    773505| 
838788|       0.00|       0.00
%             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
         0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
+16.3|          0|          0
1            2|  512|   8606.11| 19.34|       444|         0|        32| 
         0|      1709|         0|        29|   1264624|    647652| 
1309740|       0.00|       0.00
2            2|  512|   8127.80| 18.93|       428|         0|        32| 
         0|      1726|         0|        28|   1216606|   1179269| 
1266260|       0.00|       0.00
%             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
-3.3|          0|          0
1            4|  512|   9409.41| 22.88|       413|         0|        30| 
         0|      1577|         0|        35|   1592587|   1072862| 
1746787|       0.00|       0.00
2            4|  512|   9217.34| 23.05|       400|         0|        30| 
         0|      1594|         0|        34|   1596515|    513742| 
1831538|       0.00|       0.00
%             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
         0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
+4.9|          0|          0
1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
         0|      1726|         0|        14|    663417|    611555| 
689028|       0.00|       0.00
2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
         0|      1726|         0|        13|    609836|    556090| 
634511|       0.00|       0.00
%             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
         0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
-7.9|          0|          0
1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
         0|      1594|         0|        28|   1206853|   1054653| 
1256203|       0.00|       0.00
2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
         0|      1577|         0|        27|   1179555|    884408| 
1226843|       0.00|       0.00
%             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
-2.3|          0|          0
1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
         0|      1726|         0|        36|   1734433|    887190| 
1852486|       0.00|       0.00
2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
         0|      1726|         0|        34|   1695359|    625268| 
1803599|       0.00|       0.00
%             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
         0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
-2.6|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1056.13| 17.51|        59|         0|        30| 
         0|      1594|         3|        86|    476098|   4780186| 
538158|       0.00|       0.00
2            1|   64|   1065.51| 17.64|        60|         0|        30| 
         0|      1594|         3|        87|    476836|   4823602| 
538860|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
         0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
+0.1|          0|          0
1            2|   64|   2472.80| 32.18|        76|         0|        32| 
         0|      1709|         2|       199|    479119|   2089931| 
600119|       0.00|       0.00
2            2|   64|   2468.76| 32.28|        76|         0|        33| 
         0|      1744|         1|       199|    479372|   2179342| 
600328|       0.00|       0.00
%             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
         0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
+0.0|          0|          0
1            4|   64|   2453.22| 32.20|        75|         0|        31| 
         0|      1612|         2|       198|    479223|   2175801| 
600081|       0.00|       0.00
2            4|   64|   2465.68| 32.30|        76|         0|        30| 
         0|      1594|         3|       200|    479404|   2108293| 
600252|       0.00|       0.00
%             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
         0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
+0.0|          0|          0
1            1|  256|   3608.37| 24.59|       146|         0|        32| 
         0|      1726|         2|       277|     88346|    108310| 
150637|       0.00|       0.00
2            1|  256|   3611.12| 24.95|       144|         0|        32| 
         0|      1709|         0|       281|     83319|     64354| 
145511|       0.00|       0.00
%             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
         0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
-3.4|          0|          0
1            2|  256|   7373.32| 26.83|       274|         0|        31| 
         0|      1612|         0|      4185|    278932|    264031| 
377267|       0.00|       0.00
2            2|  256|   7460.03| 26.93|       276|         0|        30| 
         0|      1594|         1|      5913|    270840|    246165| 
371999|       0.00|       0.00
%             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
         0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
-1.4|          0|          0
1            4|  256|   9322.66| 29.73|       313|         0|        32| 
         0|      1709|       848|      3041|    363114|    329308| 
482586|       0.00|       0.00
2            4|  256|   9365.52| 29.59|       316|         0|        33| 
         0|      1744|       414|      2884|    371485|    339191| 
491366|       0.00|       0.00
%             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
         0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
+1.8|          0|          0
1            1|  512|   9360.25| 17.20|       543|         0|        30| 
         0|      1577|         0|       387|    438621|    408062| 
499989|       0.00|       0.00
2            1|  512|   8899.37| 18.17|       495|         0|        30| 
         0|      1594|         0|      5049|    393341|    371545| 
459397|       0.00|       0.00
%             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
         0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
-8.1|          0|          0
1            2|  512|   8713.11| 20.77|       420|         0|        32| 
         0|      1726|         0|     11227|    296157|    246495| 
371632|       0.00|       0.00
2            2|  512|   8686.31| 20.72|       421|         0|        32| 
         0|      1709|         0|     11002|    296608|    246597| 
371797|       0.00|       0.00
%             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
         0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
+0.0|          0|          0
1            4|  512|   8780.40| 22.72|       386|         0|        30| 
         0|      1594|         0|     10337|    303619|    249822| 
385457|       0.00|       0.00
2            4|  512|   8858.51| 22.66|       390|         0|        30| 
         0|      1577|         0|      9968|    301847|    246543| 
382121|       0.00|       0.00
%             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
-0.9|          0|          0
1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
         0|      1726|         0|        84|    452351|    429483| 
504022|       0.00|       0.00
2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
         0|      1726|         2|        76|    446994|    422489| 
499025|       0.00|       0.00
%             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
         0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
-1.0|          0|          0
1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
         0|      1594|         5|       647|    388626|    358303| 
442125|       0.00|       0.00
2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
         0|      1612|        35|       554|    389326|    360169| 
442343|       0.00|       0.00
%             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
         0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
+0.0|          0|          0
1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
         0|      1726|      1189|       648|    405900|    376269| 
459845|       0.00|       0.00
2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
         0|      1726|      1754|       610|    402132|    373768| 
455629|       0.00|       0.00
%             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
         0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
-0.9|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
         0|      1698|         0|         9|    654298|    654362| 
665532|       0.00|       0.00
2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
         0|      1698|         0|        10|    652907|    652973| 
667042|       0.00|       0.00
%             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
         0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
+0.2|          0|          0
1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
         0|      1830|         0|        10|    657257|    657325| 
667088|       0.00|       0.00
2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
         0|      1830|         0|         9|    653044|    653110| 
668883|       0.00|       0.00
%             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
         0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
+0.3|          0|          0
1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
         0|      1716|         0|        10|    654235|    654308| 
665518|       0.00|       0.00
2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
         0|      1664|         0|        10|    652083|    652155| 
666869|       0.00|       0.00
%             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
         0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
+0.2|          0|          0
1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
         0|      1830|         0|        10|    655536|    655619| 
665776|       0.00|       0.00
2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
         0|      1813|         0|        10|    655726|    655803| 
668527|       0.00|       0.00
%             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
         0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
+0.4|          0|          0
1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
         0|      1681|         0|         9|    624516|    624581| 
639159|       0.00|       0.00
2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
         0|      1716|         0|         9|    625432|    625498| 
641343|       0.00|       0.00
%             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
         0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
+0.3|          0|          0
1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
         0|      1830|         0|         9|    624335|    624400| 
637357|       0.00|       0.00
2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
         0|      1848|         0|         9|    626650|    626714| 
639017|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
         0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
+0.3|          0|          0
1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
         0|      1698|         0|         9|    625738|    625822| 
641221|       0.00|       0.00
2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
         0|      1698|         0|         9|    627960|    628040| 
640680|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.4|      +0.4| 
-0.1|          0|          0
1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
         0|      1830|         0|        10|    626608|    626675| 
640081|       0.00|       0.00
2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
         0|      1813|         0|         9|    627904|    627967| 
643152|       0.00|       0.00
%             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
         0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
+0.5|          0|          0
1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
         0|      1716|         0|         7|    510418|    510486| 
524904|       0.00|       0.00
2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
         0|      1664|         0|         8|    513877|    513940| 
526294|       0.00|       0.00
%             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
         0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
+0.3|          0|          0
1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
         0|      1830|         0|         7|    510674|    510756| 
525205|       0.00|       0.00
2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
         0|      1830|         0|         7|    513389|    513471| 
526609|       0.00|       0.00
%             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.5|      +0.5| 
+0.3|          0|          0
1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
         0|      1681|         0|         7|    510518|    510579| 
524294|       0.00|       0.00
2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
         0|      1716|         0|         8|    514765|    514829| 
526481|       0.00|       0.00
%             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
         0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
+0.4|          0|          0
1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
         0|      1813|         0|         8|    511375|    511443| 
523357|       0.00|       0.00
2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
         0|      1830|         0|         7|    514467|    514533| 
526131|       0.00|       0.00
%             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
         0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
+0.5|          0|          0
1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
         0|      1681|         0|         7|    489075|    489155| 
503136|       0.00|       0.00
2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
         0|      1804|         0|         7|    487841|    487915| 
502061|       0.00|       0.00
%             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
         0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
-0.2|          0|          0
1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
         0|      1804|         0|         7|    490039|    490104| 
502990|       0.00|       0.00
2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
         0|      1725|         0|         7|    487171|    487236| 
501939|       0.00|       0.00
%             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
         0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
-0.2|          0|          0
1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
         0|      1848|         0|         7|    490597|    490657| 
503641|       0.00|       0.00
2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
         0|      1813|         0|         7|    488874|    488939| 
502010|       0.00|       0.00
%             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
         0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
-0.3|          0|          0
1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
         0|      1698|         0|         7|    491028|    491095| 
504222|       0.00|       0.00
2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
         0|      1716|         0|         7|    489521|    489593| 
502368|       0.00|       0.00
%             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
         0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
-0.4|          0|          0

Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.host_guest.txt
2 - avg-fixed.netperf.host_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    715.76| 28.44|        24|         0|        31| 
         0|      1629|         0|        36|   3206862|   1733540| 
3245171|       0.00|       0.00
2            1|   64|    714.82| 28.44|        25|         0|        31| 
         0|      1612|         0|        36|   3204256|   1709508| 
3242450|       0.00|       0.00
%             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
         0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
-0.1|          0|          0
1            2|   64|   1449.31| 41.23|        35|         0|        33| 
         0|      1744|         0|        66|   2764635|   1667527| 
2815591|       0.00|       0.00
2            2|   64|   1442.92| 41.32|        34|         0|        32| 
         0|      1709|         0|        66|   2751461|   1659498| 
2799277|       0.00|       0.00
%             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1            4|   64|   2671.25| 48.95|        54|         0|        30| 
         0|      1577|         0|        47|    914085|    289771| 
947054|       0.00|       0.00
2            4|   64|   2652.68| 48.20|        54|         0|        30| 
         0|      1594|         0|        46|    882300|    293273| 
914249|       0.00|       0.00
%             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
         0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
-3.5|          0|          0
1            1|  256|   1478.61| 29.34|        50|         0|        32| 
         0|      1709|         0|        64|   3502384|   1541681| 
3545371|       0.00|       0.00
2            1|  256|   1481.38| 29.27|        50|         0|        32| 
         0|      1726|         0|        64|   3485623|   1526105| 
3528343|       0.00|       0.00
%             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
-0.5|          0|          0
1            2|  256|   3095.93| 42.31|        72|         0|        31| 
         0|      1612|         0|       123|   1521019|    466426| 
1637722|       0.00|       0.00
2            2|  256|   3098.16| 42.03|        73|         0|        30| 
         0|      1594|         0|       123|   1519458|    472632| 
1632462|       0.00|       0.00
%             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
         0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
-0.3|          0|          0
1            4|  256|   6553.60| 38.44|       170|         0|        33| 
         0|      1761|         0|        38|   1343193|    237604| 
1402385|       0.00|       0.00
2            4|  256|   6649.37| 38.34|       172|         0|        32| 
         0|      1726|         0|        32|   1343770|    236320| 
1400875|       0.00|       0.00
%             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
         0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
-0.1|          0|          0
1            1|  512|   1818.40| 29.28|        61|         0|        31| 
         0|      1646|         0|        76|   3326887|   1491961| 
3371189|       0.00|       0.00
2            1|  512|   1822.74| 29.45|        61|         0|        30| 
         0|      1577|         0|        76|   3309934|   1470715| 
3352921|       0.00|       0.00
%             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
         0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
-0.5|          0|          0
1            2|  512|   3931.72| 43.51|        89|         0|        33| 
         0|      1761|         0|       137|   1148331|    301194| 
1270351|       0.00|       0.00
2            2|  512|   3926.16| 43.45|        90|         0|        32| 
         0|      1726|         0|       135|   1166280|    306110| 
1288869|       0.00|       0.00
%             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
         0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
+1.5|          0|          0
1            4|  512|   8634.49| 40.69|       211|         0|        31| 
         0|      1612|         0|        53|   1869998|    353856| 
1948593|       0.00|       0.00
2            4|  512|   9137.12| 40.77|       223|         0|        30| 
         0|      1594|         0|        48|   1879574|    362851| 
1959044|       0.00|       0.00
%             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
         0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
+0.5|          0|          0
1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
         0|      1778|         0|        84|   3087032|   1309160| 
3130736|       0.00|       0.00
2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
         0|      1726|         0|        83|   3082898|   1295936| 
3125229|       0.00|       0.00
%             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
-0.2|          0|          0
1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
         0|      1673|         0|        65|   2522172|    663302| 
2684835|       0.00|       0.00
2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
         0|      1594|         0|        62|   2607281|    698035| 
2755198|       0.00|       0.00
%             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
         0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
+2.6|          0|          0
1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
         0|      1700|         0|        28|   2510318|    576071| 
2596631|       0.00|       0.00
2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
         0|      1726|         0|        25|   2232437|    632785| 
2316035|       0.00|       0.00
%             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
         0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
-10.8|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    706.98| 26.25|        26|         0|        31| 
         0|      1656|         0|        80|   2630204|   4185330| 
2692397|       0.00|       0.00
2            1|   64|    767.45| 27.78|        27|         0|        30| 
         0|      1594|         0|        85|   2626694|   4317273| 
2688799|       0.00|       0.00
%             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
         0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
-0.1|          0|          0
1            2|   64|   2644.57| 42.60|        61|         0|        32| 
         0|      1700|         0|     34863|     49057|     16791| 
204805|       0.00|       0.00
2            2|   64|   2721.19| 42.69|        63|         0|        32| 
         0|      1709|         0|     42365|     52739|     20094| 
215895|       0.00|       0.00
%             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
         0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
+5.4|          0|          0
1            4|   64|   2756.40| 42.69|        64|         0|        32| 
         0|      1708|         0|     52043|     47976|     12694| 
220732|       0.00|       0.00
2            4|   64|   2709.19| 42.79|        62|         0|        30| 
         0|      1594|         0|     50144|     47275|     12743| 
218074|       0.00|       0.00
%             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
         0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
-1.2|          0|          0
1            1|  256|   3785.63| 32.34|       116|         0|        33| 
         0|      1734|         0|     36642|     58940|      1187| 
157652|       0.00|       0.00
2            1|  256|   3593.17| 32.90|       108|         0|        32| 
         0|      1726|         0|     33179|     54980|      1130| 
150168|       0.00|       0.00
%             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
         0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
-4.7|          0|          0
1            2|  256|   8316.26| 40.45|       205|         0|        32| 
         0|      1673|         0|     28302|    114835|      1655| 
251137|       0.00|       0.00
2            2|  256|   8390.41| 41.17|       203|         0|        30| 
         0|      1594|         0|     31222|    115372|      1696| 
257151|       0.00|       0.00
%             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
         0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
+2.4|          0|          0
1            4|  256|   9672.49| 42.40|       227|         0|        33| 
         0|      1761|         0|     30045|    132027|      3059| 
283491|       0.00|       0.00
2            4|  256|   9741.71| 42.57|       228|         0|        32| 
         0|      1726|         6|     28875|    132125|      1999| 
283036|       0.00|       0.00
%             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
         0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
-0.2|          0|          0
1            1|  512|   6052.41| 32.23|       187|         0|        31| 
         0|      1612|         0|     44681|     86084|      2592| 
187465|       0.00|       0.00
2            1|  512|   6267.37| 32.51|       192|         0|        30| 
         0|      1577|         0|     45278|     88449|      2708| 
191374|       0.00|       0.00
%             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
         0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
+2.1|          0|          0
1            2|  512|  12714.52| 38.27|       331|         0|        33| 
         0|      1744|         0|     24843|    169170|      2381| 
292210|       0.00|       0.00
2            2|  512|  11341.65| 37.30|       303|         0|        32| 
         0|      1726|        14|     27393|    152465|      2980| 
270740|       0.00|       0.00
%             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
         0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
-7.3|          0|          0
1            4|  512|  16127.03| 40.80|       394|         0|        31| 
         0|      1612|         0|     15063|    210628|      2234| 
344124|       0.00|       0.00
2            4|  512|  14709.72| 39.94|       367|         0|        31| 
         0|      1612|         0|     17823|    193670|      4457| 
324744|       0.00|       0.00
%             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
         0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
-5.6|          0|          0
1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
         0|      1778|         0|     47820|     83940|      2007| 
172847|       0.00|       0.00
2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
         0|      1726|         0|     47771|     85436|      2038| 
175027|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
+1.3|          0|          0
1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
         0|      1629|         0|     26499|    162142|      2909| 
255717|       0.00|       0.00
2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
         0|      1612|         0|     24196|    171386|      3671| 
266270|       0.00|       0.00
%             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
         0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
+4.1|          0|          0
1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
         0|      1709|         0|     12496|    215031|      6557| 
318039|       0.00|       0.00
2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
         0|      1709|         0|      9444|    224694|      4095| 
328098|       0.00|       0.00
%             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
         0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
+3.2|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  14626.43| 14.93|       980|         0|        32| 
         0|      1681|         0|        13|    877664|    877731| 
902624|       0.00|       0.00
2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
         0|      1681|         0|        13|    878475|    878538| 
897278|       0.00|       0.00
%             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.1|      +0.1| 
-0.6|          0|          0
1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
         0|      1830|         0|        13|    881930|    881991| 
903837|       0.00|       0.00
2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
         0|      1830|         0|        13|    881619|    881679| 
898952|       0.00|       0.00
%             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
         0|      1698|         0|        13|    881083|    881147| 
903071|       0.00|       0.00
2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
         0|      1698|         0|        13|    880702|    880762| 
898190|       0.00|       0.00
%             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
         0|      1830|         0|        13|    881985|    882060| 
905537|       0.00|       0.00
2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
         0|      1830|         0|        13|    877648|    877726| 
896427|       0.00|       0.00
%             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.5|      -0.5| 
-1.0|          0|          0
1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
         0|      1716|         0|        13|    863266|    863336| 
887195|       0.00|       0.00
2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
         0|      1681|         0|        13|    861630|    861698| 
882444|       0.00|       0.00
%             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
-0.5|          0|          0
1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
         0|      1813|         0|        13|    861969|    862033| 
886679|       0.00|       0.00
2          100|  256|  14393.82| 14.53|       999|         0|        34| 
         0|      1813|         0|        13|    863704|    863765| 
884487|       0.00|       0.00
%             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.2|      +0.2| 
-0.2|          0|          0
1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
         0|      1681|         0|        13|    863352|    863429| 
887505|       0.00|       0.00
2          250|  256|  14344.74| 14.62|       987|         0|        32| 
         0|      1698|         0|        13|    860760|    860843| 
882154|       0.00|       0.00
%             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
-0.6|          0|          0
1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
         0|      1769|         0|        13|    862029|    862099| 
885852|       0.00|       0.00
2          500|  256|  14329.59| 14.62|       990|         0|        34| 
         0|      1830|         0|        13|    859852|    861233| 
880082|       0.00|       0.00
%             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
-0.7|          0|          0
1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
         0|      1742|         0|        13|    856460|    856524| 
880356|       0.00|       0.00
2           50|  512|  14222.08| 14.72|       966|         0|        32| 
         0|      1698|         0|        12|    853398|    853468| 
875621|       0.00|       0.00
%             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
         0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
-0.5|          0|          0
1          100|  512|  14236.34| 14.51|       982|         0|        34| 
         0|      1804|         0|        13|    854258|    854335| 
876662|       0.00|       0.00
2          100|  512|  14147.89| 14.85|       953|         0|        34| 
         0|      1813|         0|        13|    848947|    849031| 
871773|       0.00|       0.00
%             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
         0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
-0.6|          0|          0
1          250|  512|  14260.79| 14.53|       983|         0|        33| 
         0|      1760|         0|        13|    855721|    855791| 
879099|       0.00|       0.00
2          250|  512|  14190.68| 14.87|       954|         0|        32| 
         0|      1698|         0|        13|    851517|    851580| 
874119|       0.00|       0.00
%             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
         0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
         0|      1769|         0|        13|    856499|    856562| 
879984|       0.00|       0.00
2          500|  512|  14213.87| 14.72|       967|         0|        34| 
         0|      1830|         0|        13|    852907|    852976| 
875328|       0.00|       0.00
%             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
-0.5|          0|          0
1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
         0|      1708|         0|        12|    843061|    843134| 
863697|       0.00|       0.00
2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
         0|      1698|         0|        12|    838890|    838973| 
858350|       0.00|       0.00
%             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
         0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
         0|      1769|         0|        13|    841616|    841685| 
864460|       0.00|       0.00
2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
         0|      1830|         0|        12|    839676|    839735| 
859288|       0.00|       0.00
%             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
         0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
-0.6|          0|          0
1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
         0|      1848|         0|        12|    843688|    843752| 
864475|       0.00|       0.00
2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
         0|      1830|         0|        12|    839026|    839089| 
859914|       0.00|       0.00
%             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
         0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
-0.5|          0|          0
1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
         0|      1716|         0|        13|    843144|    843212| 
862981|       0.00|       0.00
2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
         0|      1681|         0|        13|    840915|    840983| 
862635|       0.00|       0.00
%             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
-0.0|          0|          0

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 13:12, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com>  wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&   processor was this  ?
>>
>> pined guest memory to numa node 1
>
> Please try this patch.  How much does the branch cost us?
>
> (Compiles, untested).
>
> Thanks,
> Rusty.
>
> From: Rusty Russell<rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
>
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
>
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
>
> By comparison, this branch costs us???
>
> Reference: https://lkml.org/lkml/2011/12/11/22
>
> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
> ---
>   drivers/lguest/lguest_device.c |   10 ++++++----
>   drivers/s390/kvm/kvm_virtio.c  |    2 +-
>   drivers/virtio/virtio_mmio.c   |    7 ++++---
>   drivers/virtio/virtio_pci.c    |    4 ++--
>   drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>   include/linux/virtio_ring.h    |    1 +
>   tools/virtio/linux/virtio.h    |    1 +
>   tools/virtio/virtio_test.c     |    3 ++-
>   8 files changed, 38 insertions(+), 24 deletions(-)

Hi all,

I tested with the same environment and scenarios.
tested one scenarios for three times and compute the average for more 
precision.

Thanks, Amos

--------- compare results -----------
Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.exhost_guest.txt
2 - avg-fixed.netperf.exhost_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1073.54| 10.50|       102|         0|        31| 
         0|      1612|         0|        16|    487641|    489753| 
504764|       0.00|       0.00
2            1|   64|   1079.44| 10.29|       104|         0|        30| 
         0|      1594|         0|        17|    487156|    488828| 
504411|       0.00|       0.00
%             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
         0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
-0.1|          0|          0
1            2|   64|   2141.12| 15.72|       136|         0|        33| 
         0|      1744|         0|        34|    873777|    972303| 
928926|       0.00|       0.00
2            2|   64|   2140.88| 15.64|       137|         0|        33| 
         0|      1744|         0|        34|    926588|    942841| 
974095|       0.00|       0.00
%             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +6.0|      -3.0| 
+4.9|          0|          0
1            4|   64|   4076.80| 19.82|       205|         0|        30| 
         0|      1577|         0|        67|   1422282|   1166425| 
1539219|       0.00|       0.00
2            4|   64|   4094.32| 20.70|       197|         0|        31| 
         0|      1612|         0|        68|   1704330|   1314077| 
1833394|       0.00|       0.00
%             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
         0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
+19.1|          0|          0
1            1|  256|   2867.48| 13.44|       213|         0|        32| 
         0|      1726|         0|        14|    666430|    694922| 
690730|       0.00|       0.00
2            1|  256|   2874.20| 12.71|       226|         0|        32| 
         0|      1709|         0|        14|    697960|    740407| 
721807|       0.00|       0.00
%             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
         0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
+4.5|          0|          0
1            2|  256|   5642.82| 17.61|       320|         0|        30| 
         0|      1594|         0|        30|   1226861|   1236081| 
1268562|       0.00|       0.00
2            2|  256|   5661.06| 17.41|       326|         0|        30| 
         0|      1594|         0|        29|   1175696|   1143490| 
1221528|       0.00|       0.00
%             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
         0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
-3.7|          0|          0
1            4|  256|   9404.27| 23.55|       399|         0|        33| 
         0|      1744|         0|        37|   1692245|    659975| 
1765103|       0.00|       0.00
2            4|  256|   8761.11| 23.18|       376|         0|        32| 
         0|      1726|         0|        36|   1699382|    418992| 
1870804|       0.00|       0.00
%             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
         0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
+6.0|          0|          0
1            1|  512|   3803.66| 14.20|       267|         0|        30| 
         0|      1594|         0|        14|    693992|    750078| 
721107|       0.00|       0.00
2            1|  512|   3838.02| 15.47|       248|         0|        31| 
         0|      1612|         0|        15|    811709|    773505| 
838788|       0.00|       0.00
%             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
         0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
+16.3|          0|          0
1            2|  512|   8606.11| 19.34|       444|         0|        32| 
         0|      1709|         0|        29|   1264624|    647652| 
1309740|       0.00|       0.00
2            2|  512|   8127.80| 18.93|       428|         0|        32| 
         0|      1726|         0|        28|   1216606|   1179269| 
1266260|       0.00|       0.00
%             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
-3.3|          0|          0
1            4|  512|   9409.41| 22.88|       413|         0|        30| 
         0|      1577|         0|        35|   1592587|   1072862| 
1746787|       0.00|       0.00
2            4|  512|   9217.34| 23.05|       400|         0|        30| 
         0|      1594|         0|        34|   1596515|    513742| 
1831538|       0.00|       0.00
%             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
         0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
+4.9|          0|          0
1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
         0|      1726|         0|        14|    663417|    611555| 
689028|       0.00|       0.00
2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
         0|      1726|         0|        13|    609836|    556090| 
634511|       0.00|       0.00
%             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
         0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
-7.9|          0|          0
1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
         0|      1594|         0|        28|   1206853|   1054653| 
1256203|       0.00|       0.00
2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
         0|      1577|         0|        27|   1179555|    884408| 
1226843|       0.00|       0.00
%             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
-2.3|          0|          0
1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
         0|      1726|         0|        36|   1734433|    887190| 
1852486|       0.00|       0.00
2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
         0|      1726|         0|        34|   1695359|    625268| 
1803599|       0.00|       0.00
%             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
         0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
-2.6|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1056.13| 17.51|        59|         0|        30| 
         0|      1594|         3|        86|    476098|   4780186| 
538158|       0.00|       0.00
2            1|   64|   1065.51| 17.64|        60|         0|        30| 
         0|      1594|         3|        87|    476836|   4823602| 
538860|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
         0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
+0.1|          0|          0
1            2|   64|   2472.80| 32.18|        76|         0|        32| 
         0|      1709|         2|       199|    479119|   2089931| 
600119|       0.00|       0.00
2            2|   64|   2468.76| 32.28|        76|         0|        33| 
         0|      1744|         1|       199|    479372|   2179342| 
600328|       0.00|       0.00
%             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
         0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
+0.0|          0|          0
1            4|   64|   2453.22| 32.20|        75|         0|        31| 
         0|      1612|         2|       198|    479223|   2175801| 
600081|       0.00|       0.00
2            4|   64|   2465.68| 32.30|        76|         0|        30| 
         0|      1594|         3|       200|    479404|   2108293| 
600252|       0.00|       0.00
%             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
         0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
+0.0|          0|          0
1            1|  256|   3608.37| 24.59|       146|         0|        32| 
         0|      1726|         2|       277|     88346|    108310| 
150637|       0.00|       0.00
2            1|  256|   3611.12| 24.95|       144|         0|        32| 
         0|      1709|         0|       281|     83319|     64354| 
145511|       0.00|       0.00
%             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
         0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
-3.4|          0|          0
1            2|  256|   7373.32| 26.83|       274|         0|        31| 
         0|      1612|         0|      4185|    278932|    264031| 
377267|       0.00|       0.00
2            2|  256|   7460.03| 26.93|       276|         0|        30| 
         0|      1594|         1|      5913|    270840|    246165| 
371999|       0.00|       0.00
%             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
         0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
-1.4|          0|          0
1            4|  256|   9322.66| 29.73|       313|         0|        32| 
         0|      1709|       848|      3041|    363114|    329308| 
482586|       0.00|       0.00
2            4|  256|   9365.52| 29.59|       316|         0|        33| 
         0|      1744|       414|      2884|    371485|    339191| 
491366|       0.00|       0.00
%             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
         0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
+1.8|          0|          0
1            1|  512|   9360.25| 17.20|       543|         0|        30| 
         0|      1577|         0|       387|    438621|    408062| 
499989|       0.00|       0.00
2            1|  512|   8899.37| 18.17|       495|         0|        30| 
         0|      1594|         0|      5049|    393341|    371545| 
459397|       0.00|       0.00
%             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
         0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
-8.1|          0|          0
1            2|  512|   8713.11| 20.77|       420|         0|        32| 
         0|      1726|         0|     11227|    296157|    246495| 
371632|       0.00|       0.00
2            2|  512|   8686.31| 20.72|       421|         0|        32| 
         0|      1709|         0|     11002|    296608|    246597| 
371797|       0.00|       0.00
%             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
         0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
+0.0|          0|          0
1            4|  512|   8780.40| 22.72|       386|         0|        30| 
         0|      1594|         0|     10337|    303619|    249822| 
385457|       0.00|       0.00
2            4|  512|   8858.51| 22.66|       390|         0|        30| 
         0|      1577|         0|      9968|    301847|    246543| 
382121|       0.00|       0.00
%             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
-0.9|          0|          0
1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
         0|      1726|         0|        84|    452351|    429483| 
504022|       0.00|       0.00
2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
         0|      1726|         2|        76|    446994|    422489| 
499025|       0.00|       0.00
%             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
         0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
-1.0|          0|          0
1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
         0|      1594|         5|       647|    388626|    358303| 
442125|       0.00|       0.00
2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
         0|      1612|        35|       554|    389326|    360169| 
442343|       0.00|       0.00
%             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
         0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
+0.0|          0|          0
1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
         0|      1726|      1189|       648|    405900|    376269| 
459845|       0.00|       0.00
2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
         0|      1726|      1754|       610|    402132|    373768| 
455629|       0.00|       0.00
%             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
         0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
-0.9|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
         0|      1698|         0|         9|    654298|    654362| 
665532|       0.00|       0.00
2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
         0|      1698|         0|        10|    652907|    652973| 
667042|       0.00|       0.00
%             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
         0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
+0.2|          0|          0
1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
         0|      1830|         0|        10|    657257|    657325| 
667088|       0.00|       0.00
2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
         0|      1830|         0|         9|    653044|    653110| 
668883|       0.00|       0.00
%             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
         0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
+0.3|          0|          0
1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
         0|      1716|         0|        10|    654235|    654308| 
665518|       0.00|       0.00
2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
         0|      1664|         0|        10|    652083|    652155| 
666869|       0.00|       0.00
%             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
         0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
+0.2|          0|          0
1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
         0|      1830|         0|        10|    655536|    655619| 
665776|       0.00|       0.00
2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
         0|      1813|         0|        10|    655726|    655803| 
668527|       0.00|       0.00
%             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
         0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
+0.4|          0|          0
1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
         0|      1681|         0|         9|    624516|    624581| 
639159|       0.00|       0.00
2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
         0|      1716|         0|         9|    625432|    625498| 
641343|       0.00|       0.00
%             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
         0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
+0.3|          0|          0
1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
         0|      1830|         0|         9|    624335|    624400| 
637357|       0.00|       0.00
2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
         0|      1848|         0|         9|    626650|    626714| 
639017|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
         0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
+0.3|          0|          0
1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
         0|      1698|         0|         9|    625738|    625822| 
641221|       0.00|       0.00
2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
         0|      1698|         0|         9|    627960|    628040| 
640680|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.4|      +0.4| 
-0.1|          0|          0
1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
         0|      1830|         0|        10|    626608|    626675| 
640081|       0.00|       0.00
2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
         0|      1813|         0|         9|    627904|    627967| 
643152|       0.00|       0.00
%             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
         0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
+0.5|          0|          0
1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
         0|      1716|         0|         7|    510418|    510486| 
524904|       0.00|       0.00
2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
         0|      1664|         0|         8|    513877|    513940| 
526294|       0.00|       0.00
%             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
         0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
+0.3|          0|          0
1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
         0|      1830|         0|         7|    510674|    510756| 
525205|       0.00|       0.00
2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
         0|      1830|         0|         7|    513389|    513471| 
526609|       0.00|       0.00
%             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.5|      +0.5| 
+0.3|          0|          0
1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
         0|      1681|         0|         7|    510518|    510579| 
524294|       0.00|       0.00
2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
         0|      1716|         0|         8|    514765|    514829| 
526481|       0.00|       0.00
%             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
         0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
+0.4|          0|          0
1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
         0|      1813|         0|         8|    511375|    511443| 
523357|       0.00|       0.00
2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
         0|      1830|         0|         7|    514467|    514533| 
526131|       0.00|       0.00
%             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
         0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
+0.5|          0|          0
1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
         0|      1681|         0|         7|    489075|    489155| 
503136|       0.00|       0.00
2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
         0|      1804|         0|         7|    487841|    487915| 
502061|       0.00|       0.00
%             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
         0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
-0.2|          0|          0
1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
         0|      1804|         0|         7|    490039|    490104| 
502990|       0.00|       0.00
2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
         0|      1725|         0|         7|    487171|    487236| 
501939|       0.00|       0.00
%             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
         0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
-0.2|          0|          0
1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
         0|      1848|         0|         7|    490597|    490657| 
503641|       0.00|       0.00
2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
         0|      1813|         0|         7|    488874|    488939| 
502010|       0.00|       0.00
%             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
         0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
-0.3|          0|          0
1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
         0|      1698|         0|         7|    491028|    491095| 
504222|       0.00|       0.00
2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
         0|      1716|         0|         7|    489521|    489593| 
502368|       0.00|       0.00
%             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
         0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
-0.4|          0|          0

Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.host_guest.txt
2 - avg-fixed.netperf.host_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    715.76| 28.44|        24|         0|        31| 
         0|      1629|         0|        36|   3206862|   1733540| 
3245171|       0.00|       0.00
2            1|   64|    714.82| 28.44|        25|         0|        31| 
         0|      1612|         0|        36|   3204256|   1709508| 
3242450|       0.00|       0.00
%             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
         0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
-0.1|          0|          0
1            2|   64|   1449.31| 41.23|        35|         0|        33| 
         0|      1744|         0|        66|   2764635|   1667527| 
2815591|       0.00|       0.00
2            2|   64|   1442.92| 41.32|        34|         0|        32| 
         0|      1709|         0|        66|   2751461|   1659498| 
2799277|       0.00|       0.00
%             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1            4|   64|   2671.25| 48.95|        54|         0|        30| 
         0|      1577|         0|        47|    914085|    289771| 
947054|       0.00|       0.00
2            4|   64|   2652.68| 48.20|        54|         0|        30| 
         0|      1594|         0|        46|    882300|    293273| 
914249|       0.00|       0.00
%             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
         0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
-3.5|          0|          0
1            1|  256|   1478.61| 29.34|        50|         0|        32| 
         0|      1709|         0|        64|   3502384|   1541681| 
3545371|       0.00|       0.00
2            1|  256|   1481.38| 29.27|        50|         0|        32| 
         0|      1726|         0|        64|   3485623|   1526105| 
3528343|       0.00|       0.00
%             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
-0.5|          0|          0
1            2|  256|   3095.93| 42.31|        72|         0|        31| 
         0|      1612|         0|       123|   1521019|    466426| 
1637722|       0.00|       0.00
2            2|  256|   3098.16| 42.03|        73|         0|        30| 
         0|      1594|         0|       123|   1519458|    472632| 
1632462|       0.00|       0.00
%             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
         0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
-0.3|          0|          0
1            4|  256|   6553.60| 38.44|       170|         0|        33| 
         0|      1761|         0|        38|   1343193|    237604| 
1402385|       0.00|       0.00
2            4|  256|   6649.37| 38.34|       172|         0|        32| 
         0|      1726|         0|        32|   1343770|    236320| 
1400875|       0.00|       0.00
%             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
         0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
-0.1|          0|          0
1            1|  512|   1818.40| 29.28|        61|         0|        31| 
         0|      1646|         0|        76|   3326887|   1491961| 
3371189|       0.00|       0.00
2            1|  512|   1822.74| 29.45|        61|         0|        30| 
         0|      1577|         0|        76|   3309934|   1470715| 
3352921|       0.00|       0.00
%             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
         0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
-0.5|          0|          0
1            2|  512|   3931.72| 43.51|        89|         0|        33| 
         0|      1761|         0|       137|   1148331|    301194| 
1270351|       0.00|       0.00
2            2|  512|   3926.16| 43.45|        90|         0|        32| 
         0|      1726|         0|       135|   1166280|    306110| 
1288869|       0.00|       0.00
%             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
         0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
+1.5|          0|          0
1            4|  512|   8634.49| 40.69|       211|         0|        31| 
         0|      1612|         0|        53|   1869998|    353856| 
1948593|       0.00|       0.00
2            4|  512|   9137.12| 40.77|       223|         0|        30| 
         0|      1594|         0|        48|   1879574|    362851| 
1959044|       0.00|       0.00
%             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
         0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
+0.5|          0|          0
1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
         0|      1778|         0|        84|   3087032|   1309160| 
3130736|       0.00|       0.00
2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
         0|      1726|         0|        83|   3082898|   1295936| 
3125229|       0.00|       0.00
%             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
-0.2|          0|          0
1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
         0|      1673|         0|        65|   2522172|    663302| 
2684835|       0.00|       0.00
2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
         0|      1594|         0|        62|   2607281|    698035| 
2755198|       0.00|       0.00
%             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
         0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
+2.6|          0|          0
1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
         0|      1700|         0|        28|   2510318|    576071| 
2596631|       0.00|       0.00
2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
         0|      1726|         0|        25|   2232437|    632785| 
2316035|       0.00|       0.00
%             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
         0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
-10.8|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    706.98| 26.25|        26|         0|        31| 
         0|      1656|         0|        80|   2630204|   4185330| 
2692397|       0.00|       0.00
2            1|   64|    767.45| 27.78|        27|         0|        30| 
         0|      1594|         0|        85|   2626694|   4317273| 
2688799|       0.00|       0.00
%             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
         0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
-0.1|          0|          0
1            2|   64|   2644.57| 42.60|        61|         0|        32| 
         0|      1700|         0|     34863|     49057|     16791| 
204805|       0.00|       0.00
2            2|   64|   2721.19| 42.69|        63|         0|        32| 
         0|      1709|         0|     42365|     52739|     20094| 
215895|       0.00|       0.00
%             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
         0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
+5.4|          0|          0
1            4|   64|   2756.40| 42.69|        64|         0|        32| 
         0|      1708|         0|     52043|     47976|     12694| 
220732|       0.00|       0.00
2            4|   64|   2709.19| 42.79|        62|         0|        30| 
         0|      1594|         0|     50144|     47275|     12743| 
218074|       0.00|       0.00
%             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
         0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
-1.2|          0|          0
1            1|  256|   3785.63| 32.34|       116|         0|        33| 
         0|      1734|         0|     36642|     58940|      1187| 
157652|       0.00|       0.00
2            1|  256|   3593.17| 32.90|       108|         0|        32| 
         0|      1726|         0|     33179|     54980|      1130| 
150168|       0.00|       0.00
%             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
         0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
-4.7|          0|          0
1            2|  256|   8316.26| 40.45|       205|         0|        32| 
         0|      1673|         0|     28302|    114835|      1655| 
251137|       0.00|       0.00
2            2|  256|   8390.41| 41.17|       203|         0|        30| 
         0|      1594|         0|     31222|    115372|      1696| 
257151|       0.00|       0.00
%             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
         0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
+2.4|          0|          0
1            4|  256|   9672.49| 42.40|       227|         0|        33| 
         0|      1761|         0|     30045|    132027|      3059| 
283491|       0.00|       0.00
2            4|  256|   9741.71| 42.57|       228|         0|        32| 
         0|      1726|         6|     28875|    132125|      1999| 
283036|       0.00|       0.00
%             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
         0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
-0.2|          0|          0
1            1|  512|   6052.41| 32.23|       187|         0|        31| 
         0|      1612|         0|     44681|     86084|      2592| 
187465|       0.00|       0.00
2            1|  512|   6267.37| 32.51|       192|         0|        30| 
         0|      1577|         0|     45278|     88449|      2708| 
191374|       0.00|       0.00
%             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
         0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
+2.1|          0|          0
1            2|  512|  12714.52| 38.27|       331|         0|        33| 
         0|      1744|         0|     24843|    169170|      2381| 
292210|       0.00|       0.00
2            2|  512|  11341.65| 37.30|       303|         0|        32| 
         0|      1726|        14|     27393|    152465|      2980| 
270740|       0.00|       0.00
%             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
         0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
-7.3|          0|          0
1            4|  512|  16127.03| 40.80|       394|         0|        31| 
         0|      1612|         0|     15063|    210628|      2234| 
344124|       0.00|       0.00
2            4|  512|  14709.72| 39.94|       367|         0|        31| 
         0|      1612|         0|     17823|    193670|      4457| 
324744|       0.00|       0.00
%             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
         0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
-5.6|          0|          0
1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
         0|      1778|         0|     47820|     83940|      2007| 
172847|       0.00|       0.00
2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
         0|      1726|         0|     47771|     85436|      2038| 
175027|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
+1.3|          0|          0
1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
         0|      1629|         0|     26499|    162142|      2909| 
255717|       0.00|       0.00
2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
         0|      1612|         0|     24196|    171386|      3671| 
266270|       0.00|       0.00
%             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
         0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
+4.1|          0|          0
1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
         0|      1709|         0|     12496|    215031|      6557| 
318039|       0.00|       0.00
2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
         0|      1709|         0|      9444|    224694|      4095| 
328098|       0.00|       0.00
%             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
         0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
+3.2|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  14626.43| 14.93|       980|         0|        32| 
         0|      1681|         0|        13|    877664|    877731| 
902624|       0.00|       0.00
2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
         0|      1681|         0|        13|    878475|    878538| 
897278|       0.00|       0.00
%             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.1|      +0.1| 
-0.6|          0|          0
1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
         0|      1830|         0|        13|    881930|    881991| 
903837|       0.00|       0.00
2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
         0|      1830|         0|        13|    881619|    881679| 
898952|       0.00|       0.00
%             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
         0|      1698|         0|        13|    881083|    881147| 
903071|       0.00|       0.00
2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
         0|      1698|         0|        13|    880702|    880762| 
898190|       0.00|       0.00
%             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
         0|      1830|         0|        13|    881985|    882060| 
905537|       0.00|       0.00
2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
         0|      1830|         0|        13|    877648|    877726| 
896427|       0.00|       0.00
%             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.5|      -0.5| 
-1.0|          0|          0
1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
         0|      1716|         0|        13|    863266|    863336| 
887195|       0.00|       0.00
2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
         0|      1681|         0|        13|    861630|    861698| 
882444|       0.00|       0.00
%             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
-0.5|          0|          0
1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
         0|      1813|         0|        13|    861969|    862033| 
886679|       0.00|       0.00
2          100|  256|  14393.82| 14.53|       999|         0|        34| 
         0|      1813|         0|        13|    863704|    863765| 
884487|       0.00|       0.00
%             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.2|      +0.2| 
-0.2|          0|          0
1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
         0|      1681|         0|        13|    863352|    863429| 
887505|       0.00|       0.00
2          250|  256|  14344.74| 14.62|       987|         0|        32| 
         0|      1698|         0|        13|    860760|    860843| 
882154|       0.00|       0.00
%             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
-0.6|          0|          0
1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
         0|      1769|         0|        13|    862029|    862099| 
885852|       0.00|       0.00
2          500|  256|  14329.59| 14.62|       990|         0|        34| 
         0|      1830|         0|        13|    859852|    861233| 
880082|       0.00|       0.00
%             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
-0.7|          0|          0
1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
         0|      1742|         0|        13|    856460|    856524| 
880356|       0.00|       0.00
2           50|  512|  14222.08| 14.72|       966|         0|        32| 
         0|      1698|         0|        12|    853398|    853468| 
875621|       0.00|       0.00
%             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
         0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
-0.5|          0|          0
1          100|  512|  14236.34| 14.51|       982|         0|        34| 
         0|      1804|         0|        13|    854258|    854335| 
876662|       0.00|       0.00
2          100|  512|  14147.89| 14.85|       953|         0|        34| 
         0|      1813|         0|        13|    848947|    849031| 
871773|       0.00|       0.00
%             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
         0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
-0.6|          0|          0
1          250|  512|  14260.79| 14.53|       983|         0|        33| 
         0|      1760|         0|        13|    855721|    855791| 
879099|       0.00|       0.00
2          250|  512|  14190.68| 14.87|       954|         0|        32| 
         0|      1698|         0|        13|    851517|    851580| 
874119|       0.00|       0.00
%             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
         0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
         0|      1769|         0|        13|    856499|    856562| 
879984|       0.00|       0.00
2          500|  512|  14213.87| 14.72|       967|         0|        34| 
         0|      1830|         0|        13|    852907|    852976| 
875328|       0.00|       0.00
%             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
-0.5|          0|          0
1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
         0|      1708|         0|        12|    843061|    843134| 
863697|       0.00|       0.00
2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
         0|      1698|         0|        12|    838890|    838973| 
858350|       0.00|       0.00
%             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
         0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
         0|      1769|         0|        13|    841616|    841685| 
864460|       0.00|       0.00
2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
         0|      1830|         0|        12|    839676|    839735| 
859288|       0.00|       0.00
%             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
         0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
-0.6|          0|          0
1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
         0|      1848|         0|        12|    843688|    843752| 
864475|       0.00|       0.00
2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
         0|      1830|         0|        12|    839026|    839089| 
859914|       0.00|       0.00
%             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
         0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
-0.5|          0|          0
1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
         0|      1716|         0|        13|    843144|    843212| 
862981|       0.00|       0.00
2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
         0|      1681|         0|        13|    840915|    840983| 
862635|       0.00|       0.00
%             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
-0.0|          0|          0

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 12/12/11 13:12, Rusty Russell wrote:
> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com>  wrote:
>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>
>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>> parallel, from an external box to the guest.  TCP_STREAM results were
>>>> noisy.  This could be due to buffering done by TCP, where packet size
>>>> varies even as message size is constant.
>>>>
>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>> regressed consistently, between 7 and 35%
>>>>
>>>> The "fix" applied was simply this:
>>>
>>> What machine&   processor was this  ?
>>
>> pined guest memory to numa node 1
>
> Please try this patch.  How much does the branch cost us?
>
> (Compiles, untested).
>
> Thanks,
> Rusty.
>
> From: Rusty Russell<rusty@rustcorp.com.au>
> Subject: virtio: harsher barriers for virtio-mmio.
>
> We were cheating with our barriers; using the smp ones rather than the
> real device ones.  That was fine, until virtio-mmio came along, which
> could be talking to a real device (a non-SMP CPU).
>
> Unfortunately, just putting back the real barriers (reverting
> d57ed95d) causes a performance regression on virtio-pci.  In
> particular, Amos reports netbench's TCP_RR over virtio_net CPU
> utilization increased up to 35% while throughput went down by up to
> 14%.
>
> By comparison, this branch costs us???
>
> Reference: https://lkml.org/lkml/2011/12/11/22
>
> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
> ---
>   drivers/lguest/lguest_device.c |   10 ++++++----
>   drivers/s390/kvm/kvm_virtio.c  |    2 +-
>   drivers/virtio/virtio_mmio.c   |    7 ++++---
>   drivers/virtio/virtio_pci.c    |    4 ++--
>   drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
>   include/linux/virtio_ring.h    |    1 +
>   tools/virtio/linux/virtio.h    |    1 +
>   tools/virtio/virtio_test.c     |    3 ++-
>   8 files changed, 38 insertions(+), 24 deletions(-)

Hi all,

I tested with the same environment and scenarios.
tested one scenarios for three times and compute the average for more 
precision.

Thanks, Amos

--------- compare results -----------
Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.exhost_guest.txt
2 - avg-fixed.netperf.exhost_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1073.54| 10.50|       102|         0|        31| 
         0|      1612|         0|        16|    487641|    489753| 
504764|       0.00|       0.00
2            1|   64|   1079.44| 10.29|       104|         0|        30| 
         0|      1594|         0|        17|    487156|    488828| 
504411|       0.00|       0.00
%             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
         0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
-0.1|          0|          0
1            2|   64|   2141.12| 15.72|       136|         0|        33| 
         0|      1744|         0|        34|    873777|    972303| 
928926|       0.00|       0.00
2            2|   64|   2140.88| 15.64|       137|         0|        33| 
         0|      1744|         0|        34|    926588|    942841| 
974095|       0.00|       0.00
%             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +6.0|      -3.0| 
+4.9|          0|          0
1            4|   64|   4076.80| 19.82|       205|         0|        30| 
         0|      1577|         0|        67|   1422282|   1166425| 
1539219|       0.00|       0.00
2            4|   64|   4094.32| 20.70|       197|         0|        31| 
         0|      1612|         0|        68|   1704330|   1314077| 
1833394|       0.00|       0.00
%             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
         0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
+19.1|          0|          0
1            1|  256|   2867.48| 13.44|       213|         0|        32| 
         0|      1726|         0|        14|    666430|    694922| 
690730|       0.00|       0.00
2            1|  256|   2874.20| 12.71|       226|         0|        32| 
         0|      1709|         0|        14|    697960|    740407| 
721807|       0.00|       0.00
%             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
         0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
+4.5|          0|          0
1            2|  256|   5642.82| 17.61|       320|         0|        30| 
         0|      1594|         0|        30|   1226861|   1236081| 
1268562|       0.00|       0.00
2            2|  256|   5661.06| 17.41|       326|         0|        30| 
         0|      1594|         0|        29|   1175696|   1143490| 
1221528|       0.00|       0.00
%             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
         0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
-3.7|          0|          0
1            4|  256|   9404.27| 23.55|       399|         0|        33| 
         0|      1744|         0|        37|   1692245|    659975| 
1765103|       0.00|       0.00
2            4|  256|   8761.11| 23.18|       376|         0|        32| 
         0|      1726|         0|        36|   1699382|    418992| 
1870804|       0.00|       0.00
%             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
         0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
+6.0|          0|          0
1            1|  512|   3803.66| 14.20|       267|         0|        30| 
         0|      1594|         0|        14|    693992|    750078| 
721107|       0.00|       0.00
2            1|  512|   3838.02| 15.47|       248|         0|        31| 
         0|      1612|         0|        15|    811709|    773505| 
838788|       0.00|       0.00
%             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
         0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
+16.3|          0|          0
1            2|  512|   8606.11| 19.34|       444|         0|        32| 
         0|      1709|         0|        29|   1264624|    647652| 
1309740|       0.00|       0.00
2            2|  512|   8127.80| 18.93|       428|         0|        32| 
         0|      1726|         0|        28|   1216606|   1179269| 
1266260|       0.00|       0.00
%             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
-3.3|          0|          0
1            4|  512|   9409.41| 22.88|       413|         0|        30| 
         0|      1577|         0|        35|   1592587|   1072862| 
1746787|       0.00|       0.00
2            4|  512|   9217.34| 23.05|       400|         0|        30| 
         0|      1594|         0|        34|   1596515|    513742| 
1831538|       0.00|       0.00
%             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
         0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
+4.9|          0|          0
1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
         0|      1726|         0|        14|    663417|    611555| 
689028|       0.00|       0.00
2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
         0|      1726|         0|        13|    609836|    556090| 
634511|       0.00|       0.00
%             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
         0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
-7.9|          0|          0
1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
         0|      1594|         0|        28|   1206853|   1054653| 
1256203|       0.00|       0.00
2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
         0|      1577|         0|        27|   1179555|    884408| 
1226843|       0.00|       0.00
%             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
-2.3|          0|          0
1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
         0|      1726|         0|        36|   1734433|    887190| 
1852486|       0.00|       0.00
2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
         0|      1726|         0|        34|   1695359|    625268| 
1803599|       0.00|       0.00
%             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
         0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
-2.6|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1056.13| 17.51|        59|         0|        30| 
         0|      1594|         3|        86|    476098|   4780186| 
538158|       0.00|       0.00
2            1|   64|   1065.51| 17.64|        60|         0|        30| 
         0|      1594|         3|        87|    476836|   4823602| 
538860|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
         0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
+0.1|          0|          0
1            2|   64|   2472.80| 32.18|        76|         0|        32| 
         0|      1709|         2|       199|    479119|   2089931| 
600119|       0.00|       0.00
2            2|   64|   2468.76| 32.28|        76|         0|        33| 
         0|      1744|         1|       199|    479372|   2179342| 
600328|       0.00|       0.00
%             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
         0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
+0.0|          0|          0
1            4|   64|   2453.22| 32.20|        75|         0|        31| 
         0|      1612|         2|       198|    479223|   2175801| 
600081|       0.00|       0.00
2            4|   64|   2465.68| 32.30|        76|         0|        30| 
         0|      1594|         3|       200|    479404|   2108293| 
600252|       0.00|       0.00
%             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
         0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
+0.0|          0|          0
1            1|  256|   3608.37| 24.59|       146|         0|        32| 
         0|      1726|         2|       277|     88346|    108310| 
150637|       0.00|       0.00
2            1|  256|   3611.12| 24.95|       144|         0|        32| 
         0|      1709|         0|       281|     83319|     64354| 
145511|       0.00|       0.00
%             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
         0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
-3.4|          0|          0
1            2|  256|   7373.32| 26.83|       274|         0|        31| 
         0|      1612|         0|      4185|    278932|    264031| 
377267|       0.00|       0.00
2            2|  256|   7460.03| 26.93|       276|         0|        30| 
         0|      1594|         1|      5913|    270840|    246165| 
371999|       0.00|       0.00
%             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
         0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
-1.4|          0|          0
1            4|  256|   9322.66| 29.73|       313|         0|        32| 
         0|      1709|       848|      3041|    363114|    329308| 
482586|       0.00|       0.00
2            4|  256|   9365.52| 29.59|       316|         0|        33| 
         0|      1744|       414|      2884|    371485|    339191| 
491366|       0.00|       0.00
%             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
         0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
+1.8|          0|          0
1            1|  512|   9360.25| 17.20|       543|         0|        30| 
         0|      1577|         0|       387|    438621|    408062| 
499989|       0.00|       0.00
2            1|  512|   8899.37| 18.17|       495|         0|        30| 
         0|      1594|         0|      5049|    393341|    371545| 
459397|       0.00|       0.00
%             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
         0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
-8.1|          0|          0
1            2|  512|   8713.11| 20.77|       420|         0|        32| 
         0|      1726|         0|     11227|    296157|    246495| 
371632|       0.00|       0.00
2            2|  512|   8686.31| 20.72|       421|         0|        32| 
         0|      1709|         0|     11002|    296608|    246597| 
371797|       0.00|       0.00
%             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
         0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
+0.0|          0|          0
1            4|  512|   8780.40| 22.72|       386|         0|        30| 
         0|      1594|         0|     10337|    303619|    249822| 
385457|       0.00|       0.00
2            4|  512|   8858.51| 22.66|       390|         0|        30| 
         0|      1577|         0|      9968|    301847|    246543| 
382121|       0.00|       0.00
%             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
         0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
-0.9|          0|          0
1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
         0|      1726|         0|        84|    452351|    429483| 
504022|       0.00|       0.00
2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
         0|      1726|         2|        76|    446994|    422489| 
499025|       0.00|       0.00
%             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
         0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
-1.0|          0|          0
1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
         0|      1594|         5|       647|    388626|    358303| 
442125|       0.00|       0.00
2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
         0|      1612|        35|       554|    389326|    360169| 
442343|       0.00|       0.00
%             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
         0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
+0.0|          0|          0
1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
         0|      1726|      1189|       648|    405900|    376269| 
459845|       0.00|       0.00
2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
         0|      1726|      1754|       610|    402132|    373768| 
455629|       0.00|       0.00
%             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
         0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
-0.9|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
         0|      1698|         0|         9|    654298|    654362| 
665532|       0.00|       0.00
2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
         0|      1698|         0|        10|    652907|    652973| 
667042|       0.00|       0.00
%             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
         0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
+0.2|          0|          0
1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
         0|      1830|         0|        10|    657257|    657325| 
667088|       0.00|       0.00
2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
         0|      1830|         0|         9|    653044|    653110| 
668883|       0.00|       0.00
%             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
         0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
+0.3|          0|          0
1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
         0|      1716|         0|        10|    654235|    654308| 
665518|       0.00|       0.00
2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
         0|      1664|         0|        10|    652083|    652155| 
666869|       0.00|       0.00
%             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
         0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
+0.2|          0|          0
1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
         0|      1830|         0|        10|    655536|    655619| 
665776|       0.00|       0.00
2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
         0|      1813|         0|        10|    655726|    655803| 
668527|       0.00|       0.00
%             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
         0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
+0.4|          0|          0
1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
         0|      1681|         0|         9|    624516|    624581| 
639159|       0.00|       0.00
2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
         0|      1716|         0|         9|    625432|    625498| 
641343|       0.00|       0.00
%             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
         0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
+0.3|          0|          0
1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
         0|      1830|         0|         9|    624335|    624400| 
637357|       0.00|       0.00
2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
         0|      1848|         0|         9|    626650|    626714| 
639017|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
         0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
+0.3|          0|          0
1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
         0|      1698|         0|         9|    625738|    625822| 
641221|       0.00|       0.00
2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
         0|      1698|         0|         9|    627960|    628040| 
640680|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.4|      +0.4| 
-0.1|          0|          0
1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
         0|      1830|         0|        10|    626608|    626675| 
640081|       0.00|       0.00
2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
         0|      1813|         0|         9|    627904|    627967| 
643152|       0.00|       0.00
%             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
         0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
+0.5|          0|          0
1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
         0|      1716|         0|         7|    510418|    510486| 
524904|       0.00|       0.00
2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
         0|      1664|         0|         8|    513877|    513940| 
526294|       0.00|       0.00
%             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
         0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
+0.3|          0|          0
1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
         0|      1830|         0|         7|    510674|    510756| 
525205|       0.00|       0.00
2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
         0|      1830|         0|         7|    513389|    513471| 
526609|       0.00|       0.00
%             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.5|      +0.5| 
+0.3|          0|          0
1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
         0|      1681|         0|         7|    510518|    510579| 
524294|       0.00|       0.00
2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
         0|      1716|         0|         8|    514765|    514829| 
526481|       0.00|       0.00
%             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
         0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
+0.4|          0|          0
1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
         0|      1813|         0|         8|    511375|    511443| 
523357|       0.00|       0.00
2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
         0|      1830|         0|         7|    514467|    514533| 
526131|       0.00|       0.00
%             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
         0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
+0.5|          0|          0
1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
         0|      1681|         0|         7|    489075|    489155| 
503136|       0.00|       0.00
2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
         0|      1804|         0|         7|    487841|    487915| 
502061|       0.00|       0.00
%             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
         0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
-0.2|          0|          0
1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
         0|      1804|         0|         7|    490039|    490104| 
502990|       0.00|       0.00
2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
         0|      1725|         0|         7|    487171|    487236| 
501939|       0.00|       0.00
%             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
         0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
-0.2|          0|          0
1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
         0|      1848|         0|         7|    490597|    490657| 
503641|       0.00|       0.00
2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
         0|      1813|         0|         7|    488874|    488939| 
502010|       0.00|       0.00
%             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
         0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
-0.3|          0|          0
1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
         0|      1698|         0|         7|    491028|    491095| 
504222|       0.00|       0.00
2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
         0|      1716|         0|         7|    489521|    489593| 
502368|       0.00|       0.00
%             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
         0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
-0.4|          0|          0

Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.host_guest.txt
2 - avg-fixed.netperf.host_guest.txt

======
TCP_STREAM
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    715.76| 28.44|        24|         0|        31| 
         0|      1629|         0|        36|   3206862|   1733540| 
3245171|       0.00|       0.00
2            1|   64|    714.82| 28.44|        25|         0|        31| 
         0|      1612|         0|        36|   3204256|   1709508| 
3242450|       0.00|       0.00
%             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
         0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
-0.1|          0|          0
1            2|   64|   1449.31| 41.23|        35|         0|        33| 
         0|      1744|         0|        66|   2764635|   1667527| 
2815591|       0.00|       0.00
2            2|   64|   1442.92| 41.32|        34|         0|        32| 
         0|      1709|         0|        66|   2751461|   1659498| 
2799277|       0.00|       0.00
%             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1            4|   64|   2671.25| 48.95|        54|         0|        30| 
         0|      1577|         0|        47|    914085|    289771| 
947054|       0.00|       0.00
2            4|   64|   2652.68| 48.20|        54|         0|        30| 
         0|      1594|         0|        46|    882300|    293273| 
914249|       0.00|       0.00
%             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
         0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
-3.5|          0|          0
1            1|  256|   1478.61| 29.34|        50|         0|        32| 
         0|      1709|         0|        64|   3502384|   1541681| 
3545371|       0.00|       0.00
2            1|  256|   1481.38| 29.27|        50|         0|        32| 
         0|      1726|         0|        64|   3485623|   1526105| 
3528343|       0.00|       0.00
%             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
-0.5|          0|          0
1            2|  256|   3095.93| 42.31|        72|         0|        31| 
         0|      1612|         0|       123|   1521019|    466426| 
1637722|       0.00|       0.00
2            2|  256|   3098.16| 42.03|        73|         0|        30| 
         0|      1594|         0|       123|   1519458|    472632| 
1632462|       0.00|       0.00
%             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
         0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
-0.3|          0|          0
1            4|  256|   6553.60| 38.44|       170|         0|        33| 
         0|      1761|         0|        38|   1343193|    237604| 
1402385|       0.00|       0.00
2            4|  256|   6649.37| 38.34|       172|         0|        32| 
         0|      1726|         0|        32|   1343770|    236320| 
1400875|       0.00|       0.00
%             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
         0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
-0.1|          0|          0
1            1|  512|   1818.40| 29.28|        61|         0|        31| 
         0|      1646|         0|        76|   3326887|   1491961| 
3371189|       0.00|       0.00
2            1|  512|   1822.74| 29.45|        61|         0|        30| 
         0|      1577|         0|        76|   3309934|   1470715| 
3352921|       0.00|       0.00
%             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
         0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
-0.5|          0|          0
1            2|  512|   3931.72| 43.51|        89|         0|        33| 
         0|      1761|         0|       137|   1148331|    301194| 
1270351|       0.00|       0.00
2            2|  512|   3926.16| 43.45|        90|         0|        32| 
         0|      1726|         0|       135|   1166280|    306110| 
1288869|       0.00|       0.00
%             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
         0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
+1.5|          0|          0
1            4|  512|   8634.49| 40.69|       211|         0|        31| 
         0|      1612|         0|        53|   1869998|    353856| 
1948593|       0.00|       0.00
2            4|  512|   9137.12| 40.77|       223|         0|        30| 
         0|      1594|         0|        48|   1879574|    362851| 
1959044|       0.00|       0.00
%             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
         0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
+0.5|          0|          0
1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
         0|      1778|         0|        84|   3087032|   1309160| 
3130736|       0.00|       0.00
2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
         0|      1726|         0|        83|   3082898|   1295936| 
3125229|       0.00|       0.00
%             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
-0.2|          0|          0
1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
         0|      1673|         0|        65|   2522172|    663302| 
2684835|       0.00|       0.00
2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
         0|      1594|         0|        62|   2607281|    698035| 
2755198|       0.00|       0.00
%             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
         0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
+2.6|          0|          0
1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
         0|      1700|         0|        28|   2510318|    576071| 
2596631|       0.00|       0.00
2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
         0|      1726|         0|        25|   2232437|    632785| 
2316035|       0.00|       0.00
%             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
         0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
-10.8|          0|          0
TCP_MAERTS
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    706.98| 26.25|        26|         0|        31| 
         0|      1656|         0|        80|   2630204|   4185330| 
2692397|       0.00|       0.00
2            1|   64|    767.45| 27.78|        27|         0|        30| 
         0|      1594|         0|        85|   2626694|   4317273| 
2688799|       0.00|       0.00
%             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
         0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
-0.1|          0|          0
1            2|   64|   2644.57| 42.60|        61|         0|        32| 
         0|      1700|         0|     34863|     49057|     16791| 
204805|       0.00|       0.00
2            2|   64|   2721.19| 42.69|        63|         0|        32| 
         0|      1709|         0|     42365|     52739|     20094| 
215895|       0.00|       0.00
%             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
         0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
+5.4|          0|          0
1            4|   64|   2756.40| 42.69|        64|         0|        32| 
         0|      1708|         0|     52043|     47976|     12694| 
220732|       0.00|       0.00
2            4|   64|   2709.19| 42.79|        62|         0|        30| 
         0|      1594|         0|     50144|     47275|     12743| 
218074|       0.00|       0.00
%             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
         0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
-1.2|          0|          0
1            1|  256|   3785.63| 32.34|       116|         0|        33| 
         0|      1734|         0|     36642|     58940|      1187| 
157652|       0.00|       0.00
2            1|  256|   3593.17| 32.90|       108|         0|        32| 
         0|      1726|         0|     33179|     54980|      1130| 
150168|       0.00|       0.00
%             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
         0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
-4.7|          0|          0
1            2|  256|   8316.26| 40.45|       205|         0|        32| 
         0|      1673|         0|     28302|    114835|      1655| 
251137|       0.00|       0.00
2            2|  256|   8390.41| 41.17|       203|         0|        30| 
         0|      1594|         0|     31222|    115372|      1696| 
257151|       0.00|       0.00
%             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
         0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
+2.4|          0|          0
1            4|  256|   9672.49| 42.40|       227|         0|        33| 
         0|      1761|         0|     30045|    132027|      3059| 
283491|       0.00|       0.00
2            4|  256|   9741.71| 42.57|       228|         0|        32| 
         0|      1726|         6|     28875|    132125|      1999| 
283036|       0.00|       0.00
%             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
         0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
-0.2|          0|          0
1            1|  512|   6052.41| 32.23|       187|         0|        31| 
         0|      1612|         0|     44681|     86084|      2592| 
187465|       0.00|       0.00
2            1|  512|   6267.37| 32.51|       192|         0|        30| 
         0|      1577|         0|     45278|     88449|      2708| 
191374|       0.00|       0.00
%             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
         0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
+2.1|          0|          0
1            2|  512|  12714.52| 38.27|       331|         0|        33| 
         0|      1744|         0|     24843|    169170|      2381| 
292210|       0.00|       0.00
2            2|  512|  11341.65| 37.30|       303|         0|        32| 
         0|      1726|        14|     27393|    152465|      2980| 
270740|       0.00|       0.00
%             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
         0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
-7.3|          0|          0
1            4|  512|  16127.03| 40.80|       394|         0|        31| 
         0|      1612|         0|     15063|    210628|      2234| 
344124|       0.00|       0.00
2            4|  512|  14709.72| 39.94|       367|         0|        31| 
         0|      1612|         0|     17823|    193670|      4457| 
324744|       0.00|       0.00
%             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
         0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
-5.6|          0|          0
1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
         0|      1778|         0|     47820|     83940|      2007| 
172847|       0.00|       0.00
2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
         0|      1726|         0|     47771|     85436|      2038| 
175027|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
         0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
+1.3|          0|          0
1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
         0|      1629|         0|     26499|    162142|      2909| 
255717|       0.00|       0.00
2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
         0|      1612|         0|     24196|    171386|      3671| 
266270|       0.00|       0.00
%             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
         0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
+4.1|          0|          0
1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
         0|      1709|         0|     12496|    215031|      6557| 
318039|       0.00|       0.00
2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
         0|      1709|         0|      9444|    224694|      4095| 
328098|       0.00|       0.00
%             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
         0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
+3.2|          0|          0
TCP_RR
       sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
#rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
#io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  14626.43| 14.93|       980|         0|        32| 
         0|      1681|         0|        13|    877664|    877731| 
902624|       0.00|       0.00
2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
         0|      1681|         0|        13|    878475|    878538| 
897278|       0.00|       0.00
%             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.1|      +0.1| 
-0.6|          0|          0
1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
         0|      1830|         0|        13|    881930|    881991| 
903837|       0.00|       0.00
2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
         0|      1830|         0|        13|    881619|    881679| 
898952|       0.00|       0.00
%             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
         0|      1698|         0|        13|    881083|    881147| 
903071|       0.00|       0.00
2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
         0|      1698|         0|        13|    880702|    880762| 
898190|       0.00|       0.00
%             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.0|      -0.0| 
-0.5|          0|          0
1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
         0|      1830|         0|        13|    881985|    882060| 
905537|       0.00|       0.00
2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
         0|      1830|         0|        13|    877648|    877726| 
896427|       0.00|       0.00
%             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
         0|       0.0|         0|       0.0|      -0.5|      -0.5| 
-1.0|          0|          0
1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
         0|      1716|         0|        13|    863266|    863336| 
887195|       0.00|       0.00
2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
         0|      1681|         0|        13|    861630|    861698| 
882444|       0.00|       0.00
%             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
-0.5|          0|          0
1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
         0|      1813|         0|        13|    861969|    862033| 
886679|       0.00|       0.00
2          100|  256|  14393.82| 14.53|       999|         0|        34| 
         0|      1813|         0|        13|    863704|    863765| 
884487|       0.00|       0.00
%             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
         0|       0.0|         0|       0.0|      +0.2|      +0.2| 
-0.2|          0|          0
1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
         0|      1681|         0|        13|    863352|    863429| 
887505|       0.00|       0.00
2          250|  256|  14344.74| 14.62|       987|         0|        32| 
         0|      1698|         0|        13|    860760|    860843| 
882154|       0.00|       0.00
%             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
         0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
-0.6|          0|          0
1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
         0|      1769|         0|        13|    862029|    862099| 
885852|       0.00|       0.00
2          500|  256|  14329.59| 14.62|       990|         0|        34| 
         0|      1830|         0|        13|    859852|    861233| 
880082|       0.00|       0.00
%             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
-0.7|          0|          0
1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
         0|      1742|         0|        13|    856460|    856524| 
880356|       0.00|       0.00
2           50|  512|  14222.08| 14.72|       966|         0|        32| 
         0|      1698|         0|        12|    853398|    853468| 
875621|       0.00|       0.00
%             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
         0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
-0.5|          0|          0
1          100|  512|  14236.34| 14.51|       982|         0|        34| 
         0|      1804|         0|        13|    854258|    854335| 
876662|       0.00|       0.00
2          100|  512|  14147.89| 14.85|       953|         0|        34| 
         0|      1813|         0|        13|    848947|    849031| 
871773|       0.00|       0.00
%             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
         0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
-0.6|          0|          0
1          250|  512|  14260.79| 14.53|       983|         0|        33| 
         0|      1760|         0|        13|    855721|    855791| 
879099|       0.00|       0.00
2          250|  512|  14190.68| 14.87|       954|         0|        32| 
         0|      1698|         0|        13|    851517|    851580| 
874119|       0.00|       0.00
%             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
         0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
         0|      1769|         0|        13|    856499|    856562| 
879984|       0.00|       0.00
2          500|  512|  14213.87| 14.72|       967|         0|        34| 
         0|      1830|         0|        13|    852907|    852976| 
875328|       0.00|       0.00
%             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
         0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
-0.5|          0|          0
1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
         0|      1708|         0|        12|    843061|    843134| 
863697|       0.00|       0.00
2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
         0|      1698|         0|        12|    838890|    838973| 
858350|       0.00|       0.00
%             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
         0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
-0.6|          0|          0
1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
         0|      1769|         0|        13|    841616|    841685| 
864460|       0.00|       0.00
2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
         0|      1830|         0|        12|    839676|    839735| 
859288|       0.00|       0.00
%             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
         0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
-0.6|          0|          0
1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
         0|      1848|         0|        12|    843688|    843752| 
864475|       0.00|       0.00
2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
         0|      1830|         0|        12|    839026|    839089| 
859914|       0.00|       0.00
%             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
         0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
-0.5|          0|          0
1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
         0|      1716|         0|        13|    843144|    843212| 
862981|       0.00|       0.00
2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
         0|      1681|         0|        13|    840915|    840983| 
862635|       0.00|       0.00
%             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
         0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
-0.0|          0|          0

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:

> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more 
> precision.
> 
> Thanks, Amos
> 
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt

The output is word wrapped and generally unreadable. Any chance you can
provide us with a summary of the outcome ?

Cheers,
Ben.

> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1073.54| 10.50|       102|         0|        31| 
>          0|      1612|         0|        16|    487641|    489753| 
> 504764|       0.00|       0.00
> 2            1|   64|   1079.44| 10.29|       104|         0|        30| 
>          0|      1594|         0|        17|    487156|    488828| 
> 504411|       0.00|       0.00
> %             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
>          0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
> -0.1|          0|          0
> 1            2|   64|   2141.12| 15.72|       136|         0|        33| 
>          0|      1744|         0|        34|    873777|    972303| 
> 928926|       0.00|       0.00
> 2            2|   64|   2140.88| 15.64|       137|         0|        33| 
>          0|      1744|         0|        34|    926588|    942841| 
> 974095|       0.00|       0.00
> %             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +6.0|      -3.0| 
> +4.9|          0|          0
> 1            4|   64|   4076.80| 19.82|       205|         0|        30| 
>          0|      1577|         0|        67|   1422282|   1166425| 
> 1539219|       0.00|       0.00
> 2            4|   64|   4094.32| 20.70|       197|         0|        31| 
>          0|      1612|         0|        68|   1704330|   1314077| 
> 1833394|       0.00|       0.00
> %             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
>          0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
> +19.1|          0|          0
> 1            1|  256|   2867.48| 13.44|       213|         0|        32| 
>          0|      1726|         0|        14|    666430|    694922| 
> 690730|       0.00|       0.00
> 2            1|  256|   2874.20| 12.71|       226|         0|        32| 
>          0|      1709|         0|        14|    697960|    740407| 
> 721807|       0.00|       0.00
> %             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
> +4.5|          0|          0
> 1            2|  256|   5642.82| 17.61|       320|         0|        30| 
>          0|      1594|         0|        30|   1226861|   1236081| 
> 1268562|       0.00|       0.00
> 2            2|  256|   5661.06| 17.41|       326|         0|        30| 
>          0|      1594|         0|        29|   1175696|   1143490| 
> 1221528|       0.00|       0.00
> %             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
>          0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
> -3.7|          0|          0
> 1            4|  256|   9404.27| 23.55|       399|         0|        33| 
>          0|      1744|         0|        37|   1692245|    659975| 
> 1765103|       0.00|       0.00
> 2            4|  256|   8761.11| 23.18|       376|         0|        32| 
>          0|      1726|         0|        36|   1699382|    418992| 
> 1870804|       0.00|       0.00
> %             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
>          0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
> +6.0|          0|          0
> 1            1|  512|   3803.66| 14.20|       267|         0|        30| 
>          0|      1594|         0|        14|    693992|    750078| 
> 721107|       0.00|       0.00
> 2            1|  512|   3838.02| 15.47|       248|         0|        31| 
>          0|      1612|         0|        15|    811709|    773505| 
> 838788|       0.00|       0.00
> %             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
>          0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
> +16.3|          0|          0
> 1            2|  512|   8606.11| 19.34|       444|         0|        32| 
>          0|      1709|         0|        29|   1264624|    647652| 
> 1309740|       0.00|       0.00
> 2            2|  512|   8127.80| 18.93|       428|         0|        32| 
>          0|      1726|         0|        28|   1216606|   1179269| 
> 1266260|       0.00|       0.00
> %             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
> -3.3|          0|          0
> 1            4|  512|   9409.41| 22.88|       413|         0|        30| 
>          0|      1577|         0|        35|   1592587|   1072862| 
> 1746787|       0.00|       0.00
> 2            4|  512|   9217.34| 23.05|       400|         0|        30| 
>          0|      1594|         0|        34|   1596515|    513742| 
> 1831538|       0.00|       0.00
> %             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
>          0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
> +4.9|          0|          0
> 1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
>          0|      1726|         0|        14|    663417|    611555| 
> 689028|       0.00|       0.00
> 2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
>          0|      1726|         0|        13|    609836|    556090| 
> 634511|       0.00|       0.00
> %             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
>          0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
> -7.9|          0|          0
> 1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
>          0|      1594|         0|        28|   1206853|   1054653| 
> 1256203|       0.00|       0.00
> 2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
>          0|      1577|         0|        27|   1179555|    884408| 
> 1226843|       0.00|       0.00
> %             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
> -2.3|          0|          0
> 1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
>          0|      1726|         0|        36|   1734433|    887190| 
> 1852486|       0.00|       0.00
> 2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
>          0|      1726|         0|        34|   1695359|    625268| 
> 1803599|       0.00|       0.00
> %             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
>          0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
> -2.6|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1056.13| 17.51|        59|         0|        30| 
>          0|      1594|         3|        86|    476098|   4780186| 
> 538158|       0.00|       0.00
> 2            1|   64|   1065.51| 17.64|        60|         0|        30| 
>          0|      1594|         3|        87|    476836|   4823602| 
> 538860|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
>          0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
> +0.1|          0|          0
> 1            2|   64|   2472.80| 32.18|        76|         0|        32| 
>          0|      1709|         2|       199|    479119|   2089931| 
> 600119|       0.00|       0.00
> 2            2|   64|   2468.76| 32.28|        76|         0|        33| 
>          0|      1744|         1|       199|    479372|   2179342| 
> 600328|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
>          0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
> +0.0|          0|          0
> 1            4|   64|   2453.22| 32.20|        75|         0|        31| 
>          0|      1612|         2|       198|    479223|   2175801| 
> 600081|       0.00|       0.00
> 2            4|   64|   2465.68| 32.30|        76|         0|        30| 
>          0|      1594|         3|       200|    479404|   2108293| 
> 600252|       0.00|       0.00
> %             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
>          0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
> +0.0|          0|          0
> 1            1|  256|   3608.37| 24.59|       146|         0|        32| 
>          0|      1726|         2|       277|     88346|    108310| 
> 150637|       0.00|       0.00
> 2            1|  256|   3611.12| 24.95|       144|         0|        32| 
>          0|      1709|         0|       281|     83319|     64354| 
> 145511|       0.00|       0.00
> %             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
>          0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
> -3.4|          0|          0
> 1            2|  256|   7373.32| 26.83|       274|         0|        31| 
>          0|      1612|         0|      4185|    278932|    264031| 
> 377267|       0.00|       0.00
> 2            2|  256|   7460.03| 26.93|       276|         0|        30| 
>          0|      1594|         1|      5913|    270840|    246165| 
> 371999|       0.00|       0.00
> %             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
>          0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
> -1.4|          0|          0
> 1            4|  256|   9322.66| 29.73|       313|         0|        32| 
>          0|      1709|       848|      3041|    363114|    329308| 
> 482586|       0.00|       0.00
> 2            4|  256|   9365.52| 29.59|       316|         0|        33| 
>          0|      1744|       414|      2884|    371485|    339191| 
> 491366|       0.00|       0.00
> %             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
>          0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
> +1.8|          0|          0
> 1            1|  512|   9360.25| 17.20|       543|         0|        30| 
>          0|      1577|         0|       387|    438621|    408062| 
> 499989|       0.00|       0.00
> 2            1|  512|   8899.37| 18.17|       495|         0|        30| 
>          0|      1594|         0|      5049|    393341|    371545| 
> 459397|       0.00|       0.00
> %             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
>          0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
> -8.1|          0|          0
> 1            2|  512|   8713.11| 20.77|       420|         0|        32| 
>          0|      1726|         0|     11227|    296157|    246495| 
> 371632|       0.00|       0.00
> 2            2|  512|   8686.31| 20.72|       421|         0|        32| 
>          0|      1709|         0|     11002|    296608|    246597| 
> 371797|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
>          0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
> +0.0|          0|          0
> 1            4|  512|   8780.40| 22.72|       386|         0|        30| 
>          0|      1594|         0|     10337|    303619|    249822| 
> 385457|       0.00|       0.00
> 2            4|  512|   8858.51| 22.66|       390|         0|        30| 
>          0|      1577|         0|      9968|    301847|    246543| 
> 382121|       0.00|       0.00
> %             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
> -0.9|          0|          0
> 1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
>          0|      1726|         0|        84|    452351|    429483| 
> 504022|       0.00|       0.00
> 2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
>          0|      1726|         2|        76|    446994|    422489| 
> 499025|       0.00|       0.00
> %             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
>          0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
> -1.0|          0|          0
> 1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
>          0|      1594|         5|       647|    388626|    358303| 
> 442125|       0.00|       0.00
> 2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
>          0|      1612|        35|       554|    389326|    360169| 
> 442343|       0.00|       0.00
> %             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
>          0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
> +0.0|          0|          0
> 1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
>          0|      1726|      1189|       648|    405900|    376269| 
> 459845|       0.00|       0.00
> 2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
>          0|      1726|      1754|       610|    402132|    373768| 
> 455629|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
>          0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
> -0.9|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
>          0|      1698|         0|         9|    654298|    654362| 
> 665532|       0.00|       0.00
> 2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
>          0|      1698|         0|        10|    652907|    652973| 
> 667042|       0.00|       0.00
> %             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
>          0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
> +0.2|          0|          0
> 1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
>          0|      1830|         0|        10|    657257|    657325| 
> 667088|       0.00|       0.00
> 2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
>          0|      1830|         0|         9|    653044|    653110| 
> 668883|       0.00|       0.00
> %             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
>          0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
> +0.3|          0|          0
> 1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
>          0|      1716|         0|        10|    654235|    654308| 
> 665518|       0.00|       0.00
> 2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
>          0|      1664|         0|        10|    652083|    652155| 
> 666869|       0.00|       0.00
> %             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
>          0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
> +0.2|          0|          0
> 1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
>          0|      1830|         0|        10|    655536|    655619| 
> 665776|       0.00|       0.00
> 2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
>          0|      1813|         0|        10|    655726|    655803| 
> 668527|       0.00|       0.00
> %             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
>          0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
> +0.4|          0|          0
> 1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
>          0|      1681|         0|         9|    624516|    624581| 
> 639159|       0.00|       0.00
> 2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
>          0|      1716|         0|         9|    625432|    625498| 
> 641343|       0.00|       0.00
> %             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
>          0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
> +0.3|          0|          0
> 1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
>          0|      1830|         0|         9|    624335|    624400| 
> 637357|       0.00|       0.00
> 2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
>          0|      1848|         0|         9|    626650|    626714| 
> 639017|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
>          0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
> +0.3|          0|          0
> 1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
>          0|      1698|         0|         9|    625738|    625822| 
> 641221|       0.00|       0.00
> 2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
>          0|      1698|         0|         9|    627960|    628040| 
> 640680|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.4|      +0.4| 
> -0.1|          0|          0
> 1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
>          0|      1830|         0|        10|    626608|    626675| 
> 640081|       0.00|       0.00
> 2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
>          0|      1813|         0|         9|    627904|    627967| 
> 643152|       0.00|       0.00
> %             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
>          0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
> +0.5|          0|          0
> 1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
>          0|      1716|         0|         7|    510418|    510486| 
> 524904|       0.00|       0.00
> 2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
>          0|      1664|         0|         8|    513877|    513940| 
> 526294|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
>          0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
> +0.3|          0|          0
> 1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
>          0|      1830|         0|         7|    510674|    510756| 
> 525205|       0.00|       0.00
> 2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
>          0|      1830|         0|         7|    513389|    513471| 
> 526609|       0.00|       0.00
> %             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.5|      +0.5| 
> +0.3|          0|          0
> 1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
>          0|      1681|         0|         7|    510518|    510579| 
> 524294|       0.00|       0.00
> 2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
>          0|      1716|         0|         8|    514765|    514829| 
> 526481|       0.00|       0.00
> %             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
>          0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
> +0.4|          0|          0
> 1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
>          0|      1813|         0|         8|    511375|    511443| 
> 523357|       0.00|       0.00
> 2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
>          0|      1830|         0|         7|    514467|    514533| 
> 526131|       0.00|       0.00
> %             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
>          0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
> +0.5|          0|          0
> 1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
>          0|      1681|         0|         7|    489075|    489155| 
> 503136|       0.00|       0.00
> 2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
>          0|      1804|         0|         7|    487841|    487915| 
> 502061|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
>          0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
> -0.2|          0|          0
> 1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
>          0|      1804|         0|         7|    490039|    490104| 
> 502990|       0.00|       0.00
> 2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
>          0|      1725|         0|         7|    487171|    487236| 
> 501939|       0.00|       0.00
> %             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
>          0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
> -0.2|          0|          0
> 1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
>          0|      1848|         0|         7|    490597|    490657| 
> 503641|       0.00|       0.00
> 2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
>          0|      1813|         0|         7|    488874|    488939| 
> 502010|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
>          0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
> -0.3|          0|          0
> 1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
>          0|      1698|         0|         7|    491028|    491095| 
> 504222|       0.00|       0.00
> 2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
>          0|      1716|         0|         7|    489521|    489593| 
> 502368|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
>          0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
> -0.4|          0|          0
> 
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.host_guest.txt
> 2 - avg-fixed.netperf.host_guest.txt
> 
> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    715.76| 28.44|        24|         0|        31| 
>          0|      1629|         0|        36|   3206862|   1733540| 
> 3245171|       0.00|       0.00
> 2            1|   64|    714.82| 28.44|        25|         0|        31| 
>          0|      1612|         0|        36|   3204256|   1709508| 
> 3242450|       0.00|       0.00
> %             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
> -0.1|          0|          0
> 1            2|   64|   1449.31| 41.23|        35|         0|        33| 
>          0|      1744|         0|        66|   2764635|   1667527| 
> 2815591|       0.00|       0.00
> 2            2|   64|   1442.92| 41.32|        34|         0|        32| 
>          0|      1709|         0|        66|   2751461|   1659498| 
> 2799277|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1            4|   64|   2671.25| 48.95|        54|         0|        30| 
>          0|      1577|         0|        47|    914085|    289771| 
> 947054|       0.00|       0.00
> 2            4|   64|   2652.68| 48.20|        54|         0|        30| 
>          0|      1594|         0|        46|    882300|    293273| 
> 914249|       0.00|       0.00
> %             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
>          0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
> -3.5|          0|          0
> 1            1|  256|   1478.61| 29.34|        50|         0|        32| 
>          0|      1709|         0|        64|   3502384|   1541681| 
> 3545371|       0.00|       0.00
> 2            1|  256|   1481.38| 29.27|        50|         0|        32| 
>          0|      1726|         0|        64|   3485623|   1526105| 
> 3528343|       0.00|       0.00
> %             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
> -0.5|          0|          0
> 1            2|  256|   3095.93| 42.31|        72|         0|        31| 
>          0|      1612|         0|       123|   1521019|    466426| 
> 1637722|       0.00|       0.00
> 2            2|  256|   3098.16| 42.03|        73|         0|        30| 
>          0|      1594|         0|       123|   1519458|    472632| 
> 1632462|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
>          0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
> -0.3|          0|          0
> 1            4|  256|   6553.60| 38.44|       170|         0|        33| 
>          0|      1761|         0|        38|   1343193|    237604| 
> 1402385|       0.00|       0.00
> 2            4|  256|   6649.37| 38.34|       172|         0|        32| 
>          0|      1726|         0|        32|   1343770|    236320| 
> 1400875|       0.00|       0.00
> %             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
>          0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
> -0.1|          0|          0
> 1            1|  512|   1818.40| 29.28|        61|         0|        31| 
>          0|      1646|         0|        76|   3326887|   1491961| 
> 3371189|       0.00|       0.00
> 2            1|  512|   1822.74| 29.45|        61|         0|        30| 
>          0|      1577|         0|        76|   3309934|   1470715| 
> 3352921|       0.00|       0.00
> %             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
>          0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
> -0.5|          0|          0
> 1            2|  512|   3931.72| 43.51|        89|         0|        33| 
>          0|      1761|         0|       137|   1148331|    301194| 
> 1270351|       0.00|       0.00
> 2            2|  512|   3926.16| 43.45|        90|         0|        32| 
>          0|      1726|         0|       135|   1166280|    306110| 
> 1288869|       0.00|       0.00
> %             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
>          0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
> +1.5|          0|          0
> 1            4|  512|   8634.49| 40.69|       211|         0|        31| 
>          0|      1612|         0|        53|   1869998|    353856| 
> 1948593|       0.00|       0.00
> 2            4|  512|   9137.12| 40.77|       223|         0|        30| 
>          0|      1594|         0|        48|   1879574|    362851| 
> 1959044|       0.00|       0.00
> %             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
>          0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
> +0.5|          0|          0
> 1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
>          0|      1778|         0|        84|   3087032|   1309160| 
> 3130736|       0.00|       0.00
> 2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
>          0|      1726|         0|        83|   3082898|   1295936| 
> 3125229|       0.00|       0.00
> %             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
> -0.2|          0|          0
> 1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
>          0|      1673|         0|        65|   2522172|    663302| 
> 2684835|       0.00|       0.00
> 2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
>          0|      1594|         0|        62|   2607281|    698035| 
> 2755198|       0.00|       0.00
> %             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
>          0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
> +2.6|          0|          0
> 1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
>          0|      1700|         0|        28|   2510318|    576071| 
> 2596631|       0.00|       0.00
> 2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
>          0|      1726|         0|        25|   2232437|    632785| 
> 2316035|       0.00|       0.00
> %             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
>          0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
> -10.8|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    706.98| 26.25|        26|         0|        31| 
>          0|      1656|         0|        80|   2630204|   4185330| 
> 2692397|       0.00|       0.00
> 2            1|   64|    767.45| 27.78|        27|         0|        30| 
>          0|      1594|         0|        85|   2626694|   4317273| 
> 2688799|       0.00|       0.00
> %             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
>          0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
> -0.1|          0|          0
> 1            2|   64|   2644.57| 42.60|        61|         0|        32| 
>          0|      1700|         0|     34863|     49057|     16791| 
> 204805|       0.00|       0.00
> 2            2|   64|   2721.19| 42.69|        63|         0|        32| 
>          0|      1709|         0|     42365|     52739|     20094| 
> 215895|       0.00|       0.00
> %             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
>          0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
> +5.4|          0|          0
> 1            4|   64|   2756.40| 42.69|        64|         0|        32| 
>          0|      1708|         0|     52043|     47976|     12694| 
> 220732|       0.00|       0.00
> 2            4|   64|   2709.19| 42.79|        62|         0|        30| 
>          0|      1594|         0|     50144|     47275|     12743| 
> 218074|       0.00|       0.00
> %             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
>          0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
> -1.2|          0|          0
> 1            1|  256|   3785.63| 32.34|       116|         0|        33| 
>          0|      1734|         0|     36642|     58940|      1187| 
> 157652|       0.00|       0.00
> 2            1|  256|   3593.17| 32.90|       108|         0|        32| 
>          0|      1726|         0|     33179|     54980|      1130| 
> 150168|       0.00|       0.00
> %             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
>          0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
> -4.7|          0|          0
> 1            2|  256|   8316.26| 40.45|       205|         0|        32| 
>          0|      1673|         0|     28302|    114835|      1655| 
> 251137|       0.00|       0.00
> 2            2|  256|   8390.41| 41.17|       203|         0|        30| 
>          0|      1594|         0|     31222|    115372|      1696| 
> 257151|       0.00|       0.00
> %             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
>          0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
> +2.4|          0|          0
> 1            4|  256|   9672.49| 42.40|       227|         0|        33| 
>          0|      1761|         0|     30045|    132027|      3059| 
> 283491|       0.00|       0.00
> 2            4|  256|   9741.71| 42.57|       228|         0|        32| 
>          0|      1726|         6|     28875|    132125|      1999| 
> 283036|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
>          0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
> -0.2|          0|          0
> 1            1|  512|   6052.41| 32.23|       187|         0|        31| 
>          0|      1612|         0|     44681|     86084|      2592| 
> 187465|       0.00|       0.00
> 2            1|  512|   6267.37| 32.51|       192|         0|        30| 
>          0|      1577|         0|     45278|     88449|      2708| 
> 191374|       0.00|       0.00
> %             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
>          0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
> +2.1|          0|          0
> 1            2|  512|  12714.52| 38.27|       331|         0|        33| 
>          0|      1744|         0|     24843|    169170|      2381| 
> 292210|       0.00|       0.00
> 2            2|  512|  11341.65| 37.30|       303|         0|        32| 
>          0|      1726|        14|     27393|    152465|      2980| 
> 270740|       0.00|       0.00
> %             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
>          0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
> -7.3|          0|          0
> 1            4|  512|  16127.03| 40.80|       394|         0|        31| 
>          0|      1612|         0|     15063|    210628|      2234| 
> 344124|       0.00|       0.00
> 2            4|  512|  14709.72| 39.94|       367|         0|        31| 
>          0|      1612|         0|     17823|    193670|      4457| 
> 324744|       0.00|       0.00
> %             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
>          0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
> -5.6|          0|          0
> 1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
>          0|      1778|         0|     47820|     83940|      2007| 
> 172847|       0.00|       0.00
> 2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
>          0|      1726|         0|     47771|     85436|      2038| 
> 175027|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
> +1.3|          0|          0
> 1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
>          0|      1629|         0|     26499|    162142|      2909| 
> 255717|       0.00|       0.00
> 2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
>          0|      1612|         0|     24196|    171386|      3671| 
> 266270|       0.00|       0.00
> %             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
>          0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
> +4.1|          0|          0
> 1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
>          0|      1709|         0|     12496|    215031|      6557| 
> 318039|       0.00|       0.00
> 2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
>          0|      1709|         0|      9444|    224694|      4095| 
> 328098|       0.00|       0.00
> %             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
>          0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
> +3.2|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  14626.43| 14.93|       980|         0|        32| 
>          0|      1681|         0|        13|    877664|    877731| 
> 902624|       0.00|       0.00
> 2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
>          0|      1681|         0|        13|    878475|    878538| 
> 897278|       0.00|       0.00
> %             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.1|      +0.1| 
> -0.6|          0|          0
> 1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
>          0|      1830|         0|        13|    881930|    881991| 
> 903837|       0.00|       0.00
> 2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
>          0|      1830|         0|        13|    881619|    881679| 
> 898952|       0.00|       0.00
> %             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
>          0|      1698|         0|        13|    881083|    881147| 
> 903071|       0.00|       0.00
> 2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
>          0|      1698|         0|        13|    880702|    880762| 
> 898190|       0.00|       0.00
> %             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
>          0|      1830|         0|        13|    881985|    882060| 
> 905537|       0.00|       0.00
> 2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
>          0|      1830|         0|        13|    877648|    877726| 
> 896427|       0.00|       0.00
> %             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.5|      -0.5| 
> -1.0|          0|          0
> 1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
>          0|      1716|         0|        13|    863266|    863336| 
> 887195|       0.00|       0.00
> 2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
>          0|      1681|         0|        13|    861630|    861698| 
> 882444|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
> -0.5|          0|          0
> 1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
>          0|      1813|         0|        13|    861969|    862033| 
> 886679|       0.00|       0.00
> 2          100|  256|  14393.82| 14.53|       999|         0|        34| 
>          0|      1813|         0|        13|    863704|    863765| 
> 884487|       0.00|       0.00
> %             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.2|      +0.2| 
> -0.2|          0|          0
> 1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
>          0|      1681|         0|        13|    863352|    863429| 
> 887505|       0.00|       0.00
> 2          250|  256|  14344.74| 14.62|       987|         0|        32| 
>          0|      1698|         0|        13|    860760|    860843| 
> 882154|       0.00|       0.00
> %             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
> -0.6|          0|          0
> 1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
>          0|      1769|         0|        13|    862029|    862099| 
> 885852|       0.00|       0.00
> 2          500|  256|  14329.59| 14.62|       990|         0|        34| 
>          0|      1830|         0|        13|    859852|    861233| 
> 880082|       0.00|       0.00
> %             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
> -0.7|          0|          0
> 1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
>          0|      1742|         0|        13|    856460|    856524| 
> 880356|       0.00|       0.00
> 2           50|  512|  14222.08| 14.72|       966|         0|        32| 
>          0|      1698|         0|        12|    853398|    853468| 
> 875621|       0.00|       0.00
> %             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
>          0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1          100|  512|  14236.34| 14.51|       982|         0|        34| 
>          0|      1804|         0|        13|    854258|    854335| 
> 876662|       0.00|       0.00
> 2          100|  512|  14147.89| 14.85|       953|         0|        34| 
>          0|      1813|         0|        13|    848947|    849031| 
> 871773|       0.00|       0.00
> %             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
>          0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
> -0.6|          0|          0
> 1          250|  512|  14260.79| 14.53|       983|         0|        33| 
>          0|      1760|         0|        13|    855721|    855791| 
> 879099|       0.00|       0.00
> 2          250|  512|  14190.68| 14.87|       954|         0|        32| 
>          0|      1698|         0|        13|    851517|    851580| 
> 874119|       0.00|       0.00
> %             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
>          0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
>          0|      1769|         0|        13|    856499|    856562| 
> 879984|       0.00|       0.00
> 2          500|  512|  14213.87| 14.72|       967|         0|        34| 
>          0|      1830|         0|        13|    852907|    852976| 
> 875328|       0.00|       0.00
> %             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
>          0|      1708|         0|        12|    843061|    843134| 
> 863697|       0.00|       0.00
> 2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
>          0|      1698|         0|        12|    838890|    838973| 
> 858350|       0.00|       0.00
> %             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
>          0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
>          0|      1769|         0|        13|    841616|    841685| 
> 864460|       0.00|       0.00
> 2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
>          0|      1830|         0|        12|    839676|    839735| 
> 859288|       0.00|       0.00
> %             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
>          0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
> -0.6|          0|          0
> 1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
>          0|      1848|         0|        12|    843688|    843752| 
> 864475|       0.00|       0.00
> 2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
>          0|      1830|         0|        12|    839026|    839089| 
> 859914|       0.00|       0.00
> %             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
>          0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
> -0.5|          0|          0
> 1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
>          0|      1716|         0|        13|    843144|    843212| 
> 862981|       0.00|       0.00
> 2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
>          0|      1681|         0|        13|    840915|    840983| 
> 862635|       0.00|       0.00
> %             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
> -0.0|          0|          0
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:

> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more 
> precision.
> 
> Thanks, Amos
> 
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt

The output is word wrapped and generally unreadable. Any chance you can
provide us with a summary of the outcome ?

Cheers,
Ben.

> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1073.54| 10.50|       102|         0|        31| 
>          0|      1612|         0|        16|    487641|    489753| 
> 504764|       0.00|       0.00
> 2            1|   64|   1079.44| 10.29|       104|         0|        30| 
>          0|      1594|         0|        17|    487156|    488828| 
> 504411|       0.00|       0.00
> %             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
>          0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
> -0.1|          0|          0
> 1            2|   64|   2141.12| 15.72|       136|         0|        33| 
>          0|      1744|         0|        34|    873777|    972303| 
> 928926|       0.00|       0.00
> 2            2|   64|   2140.88| 15.64|       137|         0|        33| 
>          0|      1744|         0|        34|    926588|    942841| 
> 974095|       0.00|       0.00
> %             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +6.0|      -3.0| 
> +4.9|          0|          0
> 1            4|   64|   4076.80| 19.82|       205|         0|        30| 
>          0|      1577|         0|        67|   1422282|   1166425| 
> 1539219|       0.00|       0.00
> 2            4|   64|   4094.32| 20.70|       197|         0|        31| 
>          0|      1612|         0|        68|   1704330|   1314077| 
> 1833394|       0.00|       0.00
> %             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
>          0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
> +19.1|          0|          0
> 1            1|  256|   2867.48| 13.44|       213|         0|        32| 
>          0|      1726|         0|        14|    666430|    694922| 
> 690730|       0.00|       0.00
> 2            1|  256|   2874.20| 12.71|       226|         0|        32| 
>          0|      1709|         0|        14|    697960|    740407| 
> 721807|       0.00|       0.00
> %             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
> +4.5|          0|          0
> 1            2|  256|   5642.82| 17.61|       320|         0|        30| 
>          0|      1594|         0|        30|   1226861|   1236081| 
> 1268562|       0.00|       0.00
> 2            2|  256|   5661.06| 17.41|       326|         0|        30| 
>          0|      1594|         0|        29|   1175696|   1143490| 
> 1221528|       0.00|       0.00
> %             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
>          0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
> -3.7|          0|          0
> 1            4|  256|   9404.27| 23.55|       399|         0|        33| 
>          0|      1744|         0|        37|   1692245|    659975| 
> 1765103|       0.00|       0.00
> 2            4|  256|   8761.11| 23.18|       376|         0|        32| 
>          0|      1726|         0|        36|   1699382|    418992| 
> 1870804|       0.00|       0.00
> %             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
>          0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
> +6.0|          0|          0
> 1            1|  512|   3803.66| 14.20|       267|         0|        30| 
>          0|      1594|         0|        14|    693992|    750078| 
> 721107|       0.00|       0.00
> 2            1|  512|   3838.02| 15.47|       248|         0|        31| 
>          0|      1612|         0|        15|    811709|    773505| 
> 838788|       0.00|       0.00
> %             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
>          0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
> +16.3|          0|          0
> 1            2|  512|   8606.11| 19.34|       444|         0|        32| 
>          0|      1709|         0|        29|   1264624|    647652| 
> 1309740|       0.00|       0.00
> 2            2|  512|   8127.80| 18.93|       428|         0|        32| 
>          0|      1726|         0|        28|   1216606|   1179269| 
> 1266260|       0.00|       0.00
> %             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
> -3.3|          0|          0
> 1            4|  512|   9409.41| 22.88|       413|         0|        30| 
>          0|      1577|         0|        35|   1592587|   1072862| 
> 1746787|       0.00|       0.00
> 2            4|  512|   9217.34| 23.05|       400|         0|        30| 
>          0|      1594|         0|        34|   1596515|    513742| 
> 1831538|       0.00|       0.00
> %             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
>          0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
> +4.9|          0|          0
> 1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
>          0|      1726|         0|        14|    663417|    611555| 
> 689028|       0.00|       0.00
> 2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
>          0|      1726|         0|        13|    609836|    556090| 
> 634511|       0.00|       0.00
> %             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
>          0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
> -7.9|          0|          0
> 1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
>          0|      1594|         0|        28|   1206853|   1054653| 
> 1256203|       0.00|       0.00
> 2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
>          0|      1577|         0|        27|   1179555|    884408| 
> 1226843|       0.00|       0.00
> %             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
> -2.3|          0|          0
> 1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
>          0|      1726|         0|        36|   1734433|    887190| 
> 1852486|       0.00|       0.00
> 2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
>          0|      1726|         0|        34|   1695359|    625268| 
> 1803599|       0.00|       0.00
> %             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
>          0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
> -2.6|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1056.13| 17.51|        59|         0|        30| 
>          0|      1594|         3|        86|    476098|   4780186| 
> 538158|       0.00|       0.00
> 2            1|   64|   1065.51| 17.64|        60|         0|        30| 
>          0|      1594|         3|        87|    476836|   4823602| 
> 538860|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
>          0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
> +0.1|          0|          0
> 1            2|   64|   2472.80| 32.18|        76|         0|        32| 
>          0|      1709|         2|       199|    479119|   2089931| 
> 600119|       0.00|       0.00
> 2            2|   64|   2468.76| 32.28|        76|         0|        33| 
>          0|      1744|         1|       199|    479372|   2179342| 
> 600328|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
>          0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
> +0.0|          0|          0
> 1            4|   64|   2453.22| 32.20|        75|         0|        31| 
>          0|      1612|         2|       198|    479223|   2175801| 
> 600081|       0.00|       0.00
> 2            4|   64|   2465.68| 32.30|        76|         0|        30| 
>          0|      1594|         3|       200|    479404|   2108293| 
> 600252|       0.00|       0.00
> %             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
>          0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
> +0.0|          0|          0
> 1            1|  256|   3608.37| 24.59|       146|         0|        32| 
>          0|      1726|         2|       277|     88346|    108310| 
> 150637|       0.00|       0.00
> 2            1|  256|   3611.12| 24.95|       144|         0|        32| 
>          0|      1709|         0|       281|     83319|     64354| 
> 145511|       0.00|       0.00
> %             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
>          0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
> -3.4|          0|          0
> 1            2|  256|   7373.32| 26.83|       274|         0|        31| 
>          0|      1612|         0|      4185|    278932|    264031| 
> 377267|       0.00|       0.00
> 2            2|  256|   7460.03| 26.93|       276|         0|        30| 
>          0|      1594|         1|      5913|    270840|    246165| 
> 371999|       0.00|       0.00
> %             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
>          0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
> -1.4|          0|          0
> 1            4|  256|   9322.66| 29.73|       313|         0|        32| 
>          0|      1709|       848|      3041|    363114|    329308| 
> 482586|       0.00|       0.00
> 2            4|  256|   9365.52| 29.59|       316|         0|        33| 
>          0|      1744|       414|      2884|    371485|    339191| 
> 491366|       0.00|       0.00
> %             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
>          0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
> +1.8|          0|          0
> 1            1|  512|   9360.25| 17.20|       543|         0|        30| 
>          0|      1577|         0|       387|    438621|    408062| 
> 499989|       0.00|       0.00
> 2            1|  512|   8899.37| 18.17|       495|         0|        30| 
>          0|      1594|         0|      5049|    393341|    371545| 
> 459397|       0.00|       0.00
> %             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
>          0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
> -8.1|          0|          0
> 1            2|  512|   8713.11| 20.77|       420|         0|        32| 
>          0|      1726|         0|     11227|    296157|    246495| 
> 371632|       0.00|       0.00
> 2            2|  512|   8686.31| 20.72|       421|         0|        32| 
>          0|      1709|         0|     11002|    296608|    246597| 
> 371797|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
>          0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
> +0.0|          0|          0
> 1            4|  512|   8780.40| 22.72|       386|         0|        30| 
>          0|      1594|         0|     10337|    303619|    249822| 
> 385457|       0.00|       0.00
> 2            4|  512|   8858.51| 22.66|       390|         0|        30| 
>          0|      1577|         0|      9968|    301847|    246543| 
> 382121|       0.00|       0.00
> %             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
> -0.9|          0|          0
> 1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
>          0|      1726|         0|        84|    452351|    429483| 
> 504022|       0.00|       0.00
> 2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
>          0|      1726|         2|        76|    446994|    422489| 
> 499025|       0.00|       0.00
> %             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
>          0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
> -1.0|          0|          0
> 1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
>          0|      1594|         5|       647|    388626|    358303| 
> 442125|       0.00|       0.00
> 2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
>          0|      1612|        35|       554|    389326|    360169| 
> 442343|       0.00|       0.00
> %             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
>          0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
> +0.0|          0|          0
> 1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
>          0|      1726|      1189|       648|    405900|    376269| 
> 459845|       0.00|       0.00
> 2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
>          0|      1726|      1754|       610|    402132|    373768| 
> 455629|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
>          0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
> -0.9|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
>          0|      1698|         0|         9|    654298|    654362| 
> 665532|       0.00|       0.00
> 2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
>          0|      1698|         0|        10|    652907|    652973| 
> 667042|       0.00|       0.00
> %             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
>          0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
> +0.2|          0|          0
> 1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
>          0|      1830|         0|        10|    657257|    657325| 
> 667088|       0.00|       0.00
> 2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
>          0|      1830|         0|         9|    653044|    653110| 
> 668883|       0.00|       0.00
> %             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
>          0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
> +0.3|          0|          0
> 1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
>          0|      1716|         0|        10|    654235|    654308| 
> 665518|       0.00|       0.00
> 2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
>          0|      1664|         0|        10|    652083|    652155| 
> 666869|       0.00|       0.00
> %             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
>          0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
> +0.2|          0|          0
> 1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
>          0|      1830|         0|        10|    655536|    655619| 
> 665776|       0.00|       0.00
> 2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
>          0|      1813|         0|        10|    655726|    655803| 
> 668527|       0.00|       0.00
> %             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
>          0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
> +0.4|          0|          0
> 1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
>          0|      1681|         0|         9|    624516|    624581| 
> 639159|       0.00|       0.00
> 2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
>          0|      1716|         0|         9|    625432|    625498| 
> 641343|       0.00|       0.00
> %             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
>          0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
> +0.3|          0|          0
> 1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
>          0|      1830|         0|         9|    624335|    624400| 
> 637357|       0.00|       0.00
> 2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
>          0|      1848|         0|         9|    626650|    626714| 
> 639017|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
>          0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
> +0.3|          0|          0
> 1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
>          0|      1698|         0|         9|    625738|    625822| 
> 641221|       0.00|       0.00
> 2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
>          0|      1698|         0|         9|    627960|    628040| 
> 640680|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.4|      +0.4| 
> -0.1|          0|          0
> 1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
>          0|      1830|         0|        10|    626608|    626675| 
> 640081|       0.00|       0.00
> 2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
>          0|      1813|         0|         9|    627904|    627967| 
> 643152|       0.00|       0.00
> %             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
>          0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
> +0.5|          0|          0
> 1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
>          0|      1716|         0|         7|    510418|    510486| 
> 524904|       0.00|       0.00
> 2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
>          0|      1664|         0|         8|    513877|    513940| 
> 526294|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
>          0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
> +0.3|          0|          0
> 1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
>          0|      1830|         0|         7|    510674|    510756| 
> 525205|       0.00|       0.00
> 2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
>          0|      1830|         0|         7|    513389|    513471| 
> 526609|       0.00|       0.00
> %             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.5|      +0.5| 
> +0.3|          0|          0
> 1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
>          0|      1681|         0|         7|    510518|    510579| 
> 524294|       0.00|       0.00
> 2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
>          0|      1716|         0|         8|    514765|    514829| 
> 526481|       0.00|       0.00
> %             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
>          0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
> +0.4|          0|          0
> 1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
>          0|      1813|         0|         8|    511375|    511443| 
> 523357|       0.00|       0.00
> 2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
>          0|      1830|         0|         7|    514467|    514533| 
> 526131|       0.00|       0.00
> %             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
>          0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
> +0.5|          0|          0
> 1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
>          0|      1681|         0|         7|    489075|    489155| 
> 503136|       0.00|       0.00
> 2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
>          0|      1804|         0|         7|    487841|    487915| 
> 502061|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
>          0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
> -0.2|          0|          0
> 1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
>          0|      1804|         0|         7|    490039|    490104| 
> 502990|       0.00|       0.00
> 2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
>          0|      1725|         0|         7|    487171|    487236| 
> 501939|       0.00|       0.00
> %             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
>          0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
> -0.2|          0|          0
> 1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
>          0|      1848|         0|         7|    490597|    490657| 
> 503641|       0.00|       0.00
> 2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
>          0|      1813|         0|         7|    488874|    488939| 
> 502010|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
>          0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
> -0.3|          0|          0
> 1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
>          0|      1698|         0|         7|    491028|    491095| 
> 504222|       0.00|       0.00
> 2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
>          0|      1716|         0|         7|    489521|    489593| 
> 502368|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
>          0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
> -0.4|          0|          0
> 
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.host_guest.txt
> 2 - avg-fixed.netperf.host_guest.txt
> 
> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    715.76| 28.44|        24|         0|        31| 
>          0|      1629|         0|        36|   3206862|   1733540| 
> 3245171|       0.00|       0.00
> 2            1|   64|    714.82| 28.44|        25|         0|        31| 
>          0|      1612|         0|        36|   3204256|   1709508| 
> 3242450|       0.00|       0.00
> %             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
> -0.1|          0|          0
> 1            2|   64|   1449.31| 41.23|        35|         0|        33| 
>          0|      1744|         0|        66|   2764635|   1667527| 
> 2815591|       0.00|       0.00
> 2            2|   64|   1442.92| 41.32|        34|         0|        32| 
>          0|      1709|         0|        66|   2751461|   1659498| 
> 2799277|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1            4|   64|   2671.25| 48.95|        54|         0|        30| 
>          0|      1577|         0|        47|    914085|    289771| 
> 947054|       0.00|       0.00
> 2            4|   64|   2652.68| 48.20|        54|         0|        30| 
>          0|      1594|         0|        46|    882300|    293273| 
> 914249|       0.00|       0.00
> %             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
>          0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
> -3.5|          0|          0
> 1            1|  256|   1478.61| 29.34|        50|         0|        32| 
>          0|      1709|         0|        64|   3502384|   1541681| 
> 3545371|       0.00|       0.00
> 2            1|  256|   1481.38| 29.27|        50|         0|        32| 
>          0|      1726|         0|        64|   3485623|   1526105| 
> 3528343|       0.00|       0.00
> %             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
> -0.5|          0|          0
> 1            2|  256|   3095.93| 42.31|        72|         0|        31| 
>          0|      1612|         0|       123|   1521019|    466426| 
> 1637722|       0.00|       0.00
> 2            2|  256|   3098.16| 42.03|        73|         0|        30| 
>          0|      1594|         0|       123|   1519458|    472632| 
> 1632462|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
>          0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
> -0.3|          0|          0
> 1            4|  256|   6553.60| 38.44|       170|         0|        33| 
>          0|      1761|         0|        38|   1343193|    237604| 
> 1402385|       0.00|       0.00
> 2            4|  256|   6649.37| 38.34|       172|         0|        32| 
>          0|      1726|         0|        32|   1343770|    236320| 
> 1400875|       0.00|       0.00
> %             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
>          0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
> -0.1|          0|          0
> 1            1|  512|   1818.40| 29.28|        61|         0|        31| 
>          0|      1646|         0|        76|   3326887|   1491961| 
> 3371189|       0.00|       0.00
> 2            1|  512|   1822.74| 29.45|        61|         0|        30| 
>          0|      1577|         0|        76|   3309934|   1470715| 
> 3352921|       0.00|       0.00
> %             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
>          0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
> -0.5|          0|          0
> 1            2|  512|   3931.72| 43.51|        89|         0|        33| 
>          0|      1761|         0|       137|   1148331|    301194| 
> 1270351|       0.00|       0.00
> 2            2|  512|   3926.16| 43.45|        90|         0|        32| 
>          0|      1726|         0|       135|   1166280|    306110| 
> 1288869|       0.00|       0.00
> %             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
>          0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
> +1.5|          0|          0
> 1            4|  512|   8634.49| 40.69|       211|         0|        31| 
>          0|      1612|         0|        53|   1869998|    353856| 
> 1948593|       0.00|       0.00
> 2            4|  512|   9137.12| 40.77|       223|         0|        30| 
>          0|      1594|         0|        48|   1879574|    362851| 
> 1959044|       0.00|       0.00
> %             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
>          0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
> +0.5|          0|          0
> 1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
>          0|      1778|         0|        84|   3087032|   1309160| 
> 3130736|       0.00|       0.00
> 2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
>          0|      1726|         0|        83|   3082898|   1295936| 
> 3125229|       0.00|       0.00
> %             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
> -0.2|          0|          0
> 1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
>          0|      1673|         0|        65|   2522172|    663302| 
> 2684835|       0.00|       0.00
> 2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
>          0|      1594|         0|        62|   2607281|    698035| 
> 2755198|       0.00|       0.00
> %             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
>          0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
> +2.6|          0|          0
> 1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
>          0|      1700|         0|        28|   2510318|    576071| 
> 2596631|       0.00|       0.00
> 2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
>          0|      1726|         0|        25|   2232437|    632785| 
> 2316035|       0.00|       0.00
> %             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
>          0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
> -10.8|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    706.98| 26.25|        26|         0|        31| 
>          0|      1656|         0|        80|   2630204|   4185330| 
> 2692397|       0.00|       0.00
> 2            1|   64|    767.45| 27.78|        27|         0|        30| 
>          0|      1594|         0|        85|   2626694|   4317273| 
> 2688799|       0.00|       0.00
> %             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
>          0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
> -0.1|          0|          0
> 1            2|   64|   2644.57| 42.60|        61|         0|        32| 
>          0|      1700|         0|     34863|     49057|     16791| 
> 204805|       0.00|       0.00
> 2            2|   64|   2721.19| 42.69|        63|         0|        32| 
>          0|      1709|         0|     42365|     52739|     20094| 
> 215895|       0.00|       0.00
> %             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
>          0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
> +5.4|          0|          0
> 1            4|   64|   2756.40| 42.69|        64|         0|        32| 
>          0|      1708|         0|     52043|     47976|     12694| 
> 220732|       0.00|       0.00
> 2            4|   64|   2709.19| 42.79|        62|         0|        30| 
>          0|      1594|         0|     50144|     47275|     12743| 
> 218074|       0.00|       0.00
> %             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
>          0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
> -1.2|          0|          0
> 1            1|  256|   3785.63| 32.34|       116|         0|        33| 
>          0|      1734|         0|     36642|     58940|      1187| 
> 157652|       0.00|       0.00
> 2            1|  256|   3593.17| 32.90|       108|         0|        32| 
>          0|      1726|         0|     33179|     54980|      1130| 
> 150168|       0.00|       0.00
> %             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
>          0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
> -4.7|          0|          0
> 1            2|  256|   8316.26| 40.45|       205|         0|        32| 
>          0|      1673|         0|     28302|    114835|      1655| 
> 251137|       0.00|       0.00
> 2            2|  256|   8390.41| 41.17|       203|         0|        30| 
>          0|      1594|         0|     31222|    115372|      1696| 
> 257151|       0.00|       0.00
> %             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
>          0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
> +2.4|          0|          0
> 1            4|  256|   9672.49| 42.40|       227|         0|        33| 
>          0|      1761|         0|     30045|    132027|      3059| 
> 283491|       0.00|       0.00
> 2            4|  256|   9741.71| 42.57|       228|         0|        32| 
>          0|      1726|         6|     28875|    132125|      1999| 
> 283036|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
>          0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
> -0.2|          0|          0
> 1            1|  512|   6052.41| 32.23|       187|         0|        31| 
>          0|      1612|         0|     44681|     86084|      2592| 
> 187465|       0.00|       0.00
> 2            1|  512|   6267.37| 32.51|       192|         0|        30| 
>          0|      1577|         0|     45278|     88449|      2708| 
> 191374|       0.00|       0.00
> %             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
>          0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
> +2.1|          0|          0
> 1            2|  512|  12714.52| 38.27|       331|         0|        33| 
>          0|      1744|         0|     24843|    169170|      2381| 
> 292210|       0.00|       0.00
> 2            2|  512|  11341.65| 37.30|       303|         0|        32| 
>          0|      1726|        14|     27393|    152465|      2980| 
> 270740|       0.00|       0.00
> %             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
>          0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
> -7.3|          0|          0
> 1            4|  512|  16127.03| 40.80|       394|         0|        31| 
>          0|      1612|         0|     15063|    210628|      2234| 
> 344124|       0.00|       0.00
> 2            4|  512|  14709.72| 39.94|       367|         0|        31| 
>          0|      1612|         0|     17823|    193670|      4457| 
> 324744|       0.00|       0.00
> %             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
>          0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
> -5.6|          0|          0
> 1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
>          0|      1778|         0|     47820|     83940|      2007| 
> 172847|       0.00|       0.00
> 2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
>          0|      1726|         0|     47771|     85436|      2038| 
> 175027|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
> +1.3|          0|          0
> 1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
>          0|      1629|         0|     26499|    162142|      2909| 
> 255717|       0.00|       0.00
> 2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
>          0|      1612|         0|     24196|    171386|      3671| 
> 266270|       0.00|       0.00
> %             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
>          0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
> +4.1|          0|          0
> 1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
>          0|      1709|         0|     12496|    215031|      6557| 
> 318039|       0.00|       0.00
> 2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
>          0|      1709|         0|      9444|    224694|      4095| 
> 328098|       0.00|       0.00
> %             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
>          0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
> +3.2|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  14626.43| 14.93|       980|         0|        32| 
>          0|      1681|         0|        13|    877664|    877731| 
> 902624|       0.00|       0.00
> 2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
>          0|      1681|         0|        13|    878475|    878538| 
> 897278|       0.00|       0.00
> %             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.1|      +0.1| 
> -0.6|          0|          0
> 1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
>          0|      1830|         0|        13|    881930|    881991| 
> 903837|       0.00|       0.00
> 2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
>          0|      1830|         0|        13|    881619|    881679| 
> 898952|       0.00|       0.00
> %             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
>          0|      1698|         0|        13|    881083|    881147| 
> 903071|       0.00|       0.00
> 2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
>          0|      1698|         0|        13|    880702|    880762| 
> 898190|       0.00|       0.00
> %             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
>          0|      1830|         0|        13|    881985|    882060| 
> 905537|       0.00|       0.00
> 2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
>          0|      1830|         0|        13|    877648|    877726| 
> 896427|       0.00|       0.00
> %             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.5|      -0.5| 
> -1.0|          0|          0
> 1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
>          0|      1716|         0|        13|    863266|    863336| 
> 887195|       0.00|       0.00
> 2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
>          0|      1681|         0|        13|    861630|    861698| 
> 882444|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
> -0.5|          0|          0
> 1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
>          0|      1813|         0|        13|    861969|    862033| 
> 886679|       0.00|       0.00
> 2          100|  256|  14393.82| 14.53|       999|         0|        34| 
>          0|      1813|         0|        13|    863704|    863765| 
> 884487|       0.00|       0.00
> %             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.2|      +0.2| 
> -0.2|          0|          0
> 1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
>          0|      1681|         0|        13|    863352|    863429| 
> 887505|       0.00|       0.00
> 2          250|  256|  14344.74| 14.62|       987|         0|        32| 
>          0|      1698|         0|        13|    860760|    860843| 
> 882154|       0.00|       0.00
> %             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
> -0.6|          0|          0
> 1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
>          0|      1769|         0|        13|    862029|    862099| 
> 885852|       0.00|       0.00
> 2          500|  256|  14329.59| 14.62|       990|         0|        34| 
>          0|      1830|         0|        13|    859852|    861233| 
> 880082|       0.00|       0.00
> %             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
> -0.7|          0|          0
> 1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
>          0|      1742|         0|        13|    856460|    856524| 
> 880356|       0.00|       0.00
> 2           50|  512|  14222.08| 14.72|       966|         0|        32| 
>          0|      1698|         0|        12|    853398|    853468| 
> 875621|       0.00|       0.00
> %             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
>          0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1          100|  512|  14236.34| 14.51|       982|         0|        34| 
>          0|      1804|         0|        13|    854258|    854335| 
> 876662|       0.00|       0.00
> 2          100|  512|  14147.89| 14.85|       953|         0|        34| 
>          0|      1813|         0|        13|    848947|    849031| 
> 871773|       0.00|       0.00
> %             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
>          0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
> -0.6|          0|          0
> 1          250|  512|  14260.79| 14.53|       983|         0|        33| 
>          0|      1760|         0|        13|    855721|    855791| 
> 879099|       0.00|       0.00
> 2          250|  512|  14190.68| 14.87|       954|         0|        32| 
>          0|      1698|         0|        13|    851517|    851580| 
> 874119|       0.00|       0.00
> %             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
>          0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
>          0|      1769|         0|        13|    856499|    856562| 
> 879984|       0.00|       0.00
> 2          500|  512|  14213.87| 14.72|       967|         0|        34| 
>          0|      1830|         0|        13|    852907|    852976| 
> 875328|       0.00|       0.00
> %             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
>          0|      1708|         0|        12|    843061|    843134| 
> 863697|       0.00|       0.00
> 2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
>          0|      1698|         0|        12|    838890|    838973| 
> 858350|       0.00|       0.00
> %             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
>          0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
>          0|      1769|         0|        13|    841616|    841685| 
> 864460|       0.00|       0.00
> 2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
>          0|      1830|         0|        12|    839676|    839735| 
> 859288|       0.00|       0.00
> %             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
>          0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
> -0.6|          0|          0
> 1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
>          0|      1848|         0|        12|    843688|    843752| 
> 864475|       0.00|       0.00
> 2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
>          0|      1830|         0|        12|    839026|    839089| 
> 859914|       0.00|       0.00
> %             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
>          0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
> -0.5|          0|          0
> 1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
>          0|      1716|         0|        13|    843144|    843212| 
> 862981|       0.00|       0.00
> 2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
>          0|      1681|         0|        13|    840915|    840983| 
> 862635|       0.00|       0.00
> %             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
> -0.0|          0|          0
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Benjamin Herrenschmidt]

On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:

> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more 
> precision.
> 
> Thanks, Amos
> 
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt

The output is word wrapped and generally unreadable. Any chance you can
provide us with a summary of the outcome ?

Cheers,
Ben.

> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1073.54| 10.50|       102|         0|        31| 
>          0|      1612|         0|        16|    487641|    489753| 
> 504764|       0.00|       0.00
> 2            1|   64|   1079.44| 10.29|       104|         0|        30| 
>          0|      1594|         0|        17|    487156|    488828| 
> 504411|       0.00|       0.00
> %             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2| 
>          0|      -1.1|         0|      +6.2|      -0.1|      -0.2| 
> -0.1|          0|          0
> 1            2|   64|   2141.12| 15.72|       136|         0|        33| 
>          0|      1744|         0|        34|    873777|    972303| 
> 928926|       0.00|       0.00
> 2            2|   64|   2140.88| 15.64|       137|         0|        33| 
>          0|      1744|         0|        34|    926588|    942841| 
> 974095|       0.00|       0.00
> %             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +6.0|      -3.0| 
> +4.9|          0|          0
> 1            4|   64|   4076.80| 19.82|       205|         0|        30| 
>          0|      1577|         0|        67|   1422282|   1166425| 
> 1539219|       0.00|       0.00
> 2            4|   64|   4094.32| 20.70|       197|         0|        31| 
>          0|      1612|         0|        68|   1704330|   1314077| 
> 1833394|       0.00|       0.00
> %             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3| 
>          0|      +2.2|         0|      +1.5|     +19.8|     +12.7| 
> +19.1|          0|          0
> 1            1|  256|   2867.48| 13.44|       213|         0|        32| 
>          0|      1726|         0|        14|    666430|    694922| 
> 690730|       0.00|       0.00
> 2            1|  256|   2874.20| 12.71|       226|         0|        32| 
>          0|      1709|         0|        14|    697960|    740407| 
> 721807|       0.00|       0.00
> %             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      +4.7|      +6.5| 
> +4.5|          0|          0
> 1            2|  256|   5642.82| 17.61|       320|         0|        30| 
>          0|      1594|         0|        30|   1226861|   1236081| 
> 1268562|       0.00|       0.00
> 2            2|  256|   5661.06| 17.41|       326|         0|        30| 
>          0|      1594|         0|        29|   1175696|   1143490| 
> 1221528|       0.00|       0.00
> %             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0| 
>          0|       0.0|         0|      -3.3|      -4.2|      -7.5| 
> -3.7|          0|          0
> 1            4|  256|   9404.27| 23.55|       399|         0|        33| 
>          0|      1744|         0|        37|   1692245|    659975| 
> 1765103|       0.00|       0.00
> 2            4|  256|   8761.11| 23.18|       376|         0|        32| 
>          0|      1726|         0|        36|   1699382|    418992| 
> 1870804|       0.00|       0.00
> %             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0| 
>          0|      -1.0|         0|      -2.7|      +0.4|     -36.5| 
> +6.0|          0|          0
> 1            1|  512|   3803.66| 14.20|       267|         0|        30| 
>          0|      1594|         0|        14|    693992|    750078| 
> 721107|       0.00|       0.00
> 2            1|  512|   3838.02| 15.47|       248|         0|        31| 
>          0|      1612|         0|        15|    811709|    773505| 
> 838788|       0.00|       0.00
> %             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3| 
>          0|      +1.1|         0|      +7.1|     +17.0|      +3.1| 
> +16.3|          0|          0
> 1            2|  512|   8606.11| 19.34|       444|         0|        32| 
>          0|      1709|         0|        29|   1264624|    647652| 
> 1309740|       0.00|       0.00
> 2            2|  512|   8127.80| 18.93|       428|         0|        32| 
>          0|      1726|         0|        28|   1216606|   1179269| 
> 1266260|       0.00|       0.00
> %             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|      -3.4|      -3.8|     +82.1| 
> -3.3|          0|          0
> 1            4|  512|   9409.41| 22.88|       413|         0|        30| 
>          0|      1577|         0|        35|   1592587|   1072862| 
> 1746787|       0.00|       0.00
> 2            4|  512|   9217.34| 23.05|       400|         0|        30| 
>          0|      1594|         0|        34|   1596515|    513742| 
> 1831538|       0.00|       0.00
> %             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0| 
>          0|      +1.1|         0|      -2.9|      +0.2|     -52.1| 
> +4.9|          0|          0
> 1            1| 1024|   4389.75| 14.62|       303|         0|        32| 
>          0|      1726|         0|        14|    663417|    611555| 
> 689028|       0.00|       0.00
> 2            1| 1024|   4390.44| 13.66|       321|         0|        32| 
>          0|      1726|         0|        13|    609836|    556090| 
> 634511|       0.00|       0.00
> %             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0| 
>          0|       0.0|         0|      -7.1|      -8.1|      -9.1| 
> -7.9|          0|          0
> 1            2| 1024|   9046.18| 19.63|       460|         0|        30| 
>          0|      1594|         0|        28|   1206853|   1054653| 
> 1256203|       0.00|       0.00
> 2            2| 1024|   9027.54| 20.15|       447|         0|        30| 
>          0|      1577|         0|        27|   1179555|    884408| 
> 1226843|       0.00|       0.00
> %             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -2.3|     -16.1| 
> -2.3|          0|          0
> 1            4| 1024|   9410.41| 22.01|       427|         0|        32| 
>          0|      1726|         0|        36|   1734433|    887190| 
> 1852486|       0.00|       0.00
> 2            4| 1024|   9409.11| 23.18|       405|         0|        32| 
>          0|      1726|         0|        34|   1695359|    625268| 
> 1803599|       0.00|       0.00
> %             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0| 
>          0|       0.0|         0|      -5.6|      -2.3|     -29.5| 
> -2.6|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|   1056.13| 17.51|        59|         0|        30| 
>          0|      1594|         3|        86|    476098|   4780186| 
> 538158|       0.00|       0.00
> 2            1|   64|   1065.51| 17.64|        60|         0|        30| 
>          0|      1594|         3|        87|    476836|   4823602| 
> 538860|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0| 
>          0|       0.0|       0.0|      +1.2|      +0.2|      +0.9| 
> +0.1|          0|          0
> 1            2|   64|   2472.80| 32.18|        76|         0|        32| 
>          0|      1709|         2|       199|    479119|   2089931| 
> 600119|       0.00|       0.00
> 2            2|   64|   2468.76| 32.28|        76|         0|        33| 
>          0|      1744|         1|       199|    479372|   2179342| 
> 600328|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1| 
>          0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3| 
> +0.0|          0|          0
> 1            4|   64|   2453.22| 32.20|        75|         0|        31| 
>          0|      1612|         2|       198|    479223|   2175801| 
> 600081|       0.00|       0.00
> 2            4|   64|   2465.68| 32.30|        76|         0|        30| 
>          0|      1594|         3|       200|    479404|   2108293| 
> 600252|       0.00|       0.00
> %             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2| 
>          0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1| 
> +0.0|          0|          0
> 1            1|  256|   3608.37| 24.59|       146|         0|        32| 
>          0|      1726|         2|       277|     88346|    108310| 
> 150637|       0.00|       0.00
> 2            1|  256|   3611.12| 24.95|       144|         0|        32| 
>          0|      1709|         0|       281|     83319|     64354| 
> 145511|       0.00|       0.00
> %             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0| 
>          0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6| 
> -3.4|          0|          0
> 1            2|  256|   7373.32| 26.83|       274|         0|        31| 
>          0|      1612|         0|      4185|    278932|    264031| 
> 377267|       0.00|       0.00
> 2            2|  256|   7460.03| 26.93|       276|         0|        30| 
>          0|      1594|         1|      5913|    270840|    246165| 
> 371999|       0.00|       0.00
> %             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2| 
>          0|      -1.1|         0|     +41.3|      -2.9|      -6.8| 
> -1.4|          0|          0
> 1            4|  256|   9322.66| 29.73|       313|         0|        32| 
>          0|      1709|       848|      3041|    363114|    329308| 
> 482586|       0.00|       0.00
> 2            4|  256|   9365.52| 29.59|       316|         0|        33| 
>          0|      1744|       414|      2884|    371485|    339191| 
> 491366|       0.00|       0.00
> %             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1| 
>          0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0| 
> +1.8|          0|          0
> 1            1|  512|   9360.25| 17.20|       543|         0|        30| 
>          0|      1577|         0|       387|    438621|    408062| 
> 499989|       0.00|       0.00
> 2            1|  512|   8899.37| 18.17|       495|         0|        30| 
>          0|      1594|         0|      5049|    393341|    371545| 
> 459397|       0.00|       0.00
> %             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0| 
>          0|      +1.1|         0|   +1204.7|     -10.3|      -8.9| 
> -8.1|          0|          0
> 1            2|  512|   8713.11| 20.77|       420|         0|        32| 
>          0|      1726|         0|     11227|    296157|    246495| 
> 371632|       0.00|       0.00
> 2            2|  512|   8686.31| 20.72|       421|         0|        32| 
>          0|      1709|         0|     11002|    296608|    246597| 
> 371797|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0| 
>          0|      -1.0|         0|      -2.0|      +0.2|      +0.0| 
> +0.0|          0|          0
> 1            4|  512|   8780.40| 22.72|       386|         0|        30| 
>          0|      1594|         0|     10337|    303619|    249822| 
> 385457|       0.00|       0.00
> 2            4|  512|   8858.51| 22.66|       390|         0|        30| 
>          0|      1577|         0|      9968|    301847|    246543| 
> 382121|       0.00|       0.00
> %             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0| 
>          0|      -1.1|         0|      -3.6|      -0.6|      -1.3| 
> -0.9|          0|          0
> 1            1| 1024|   9387.09| 16.43|       571|         0|        32| 
>          0|      1726|         0|        84|    452351|    429483| 
> 504022|       0.00|       0.00
> 2            1| 1024|   9388.90| 16.51|       568|         0|        32| 
>          0|      1726|         2|        76|    446994|    422489| 
> 499025|       0.00|       0.00
> %             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0| 
>          0|       0.0|         0|      -9.5|      -1.2|      -1.6| 
> -1.0|          0|          0
> 1            2| 1024|   9361.43| 17.26|       542|         0|        30| 
>          0|      1594|         5|       647|    388626|    358303| 
> 442125|       0.00|       0.00
> 2            2| 1024|   9364.78| 17.25|       542|         0|        31| 
>          0|      1612|        35|       554|    389326|    360169| 
> 442343|       0.00|       0.00
> %             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3| 
>          0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5| 
> +0.0|          0|          0
> 1            4| 1024|   9381.57| 18.35|       510|         0|        32| 
>          0|      1726|      1189|       648|    405900|    376269| 
> 459845|       0.00|       0.00
> 2            4| 1024|   9393.48| 18.27|       513|         0|        32| 
>          0|      1726|      1754|       610|    402132|    373768| 
> 455629|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0| 
>          0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7| 
> -0.9|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  10903.70|  8.03|      1382|         0|        32| 
>          0|      1698|         0|         9|    654298|    654362| 
> 665532|       0.00|       0.00
> 2           50|   64|  10880.50|  7.40|      1488|         0|        32| 
>          0|      1698|         0|        10|    652907|    652973| 
> 667042|       0.00|       0.00
> %             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0| 
>          0|       0.0|         0|     +11.1|      -0.2|      -0.2| 
> +0.2|          0|          0
> 1          100|   64|  10953.00|  8.40|      1332|         0|        34| 
>          0|      1830|         0|        10|    657257|    657325| 
> 667088|       0.00|       0.00
> 2          100|   64|  10882.78|  7.67|      1439|         0|        34| 
>          0|      1830|         0|         9|    653044|    653110| 
> 668883|       0.00|       0.00
> %             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0| 
>          0|       0.0|         0|     -10.0|      -0.6|      -0.6| 
> +0.3|          0|          0
> 1          250|   64|  10902.66|  8.57|      1300|         0|        33| 
>          0|      1716|         0|        10|    654235|    654308| 
> 665518|       0.00|       0.00
> 2          250|   64|  10866.79|  7.58|      1449|         0|        32| 
>          0|      1664|         0|        10|    652083|    652155| 
> 666869|       0.00|       0.00
> %             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0| 
>          0|      -3.0|         0|       0.0|      -0.3|      -0.3| 
> +0.2|          0|          0
> 1          500|   64|  10924.34|  7.25|      1509|         0|        34| 
>          0|      1830|         0|        10|    655536|    655619| 
> 665776|       0.00|       0.00
> 2          500|   64|  10927.52|  6.50|      1684|         0|        34| 
>          0|      1813|         0|        10|    655726|    655803| 
> 668527|       0.00|       0.00
> %             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0| 
>          0|      -0.9|         0|       0.0|      +0.0|      +0.0| 
> +0.4|          0|          0
> 1           50|  256|  10407.35|  9.38|      1112|         0|        32| 
>          0|      1681|         0|         9|    624516|    624581| 
> 639159|       0.00|       0.00
> 2           50|  256|  10422.60|  9.45|      1108|         0|        33| 
>          0|      1716|         0|         9|    625432|    625498| 
> 641343|       0.00|       0.00
> %             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1| 
>          0|      +2.1|         0|       0.0|      +0.1|      +0.1| 
> +0.3|          0|          0
> 1          100|  256|  10404.30|  8.99|      1159|         0|        34| 
>          0|      1830|         0|         9|    624335|    624400| 
> 637357|       0.00|       0.00
> 2          100|  256|  10442.84|  9.57|      1093|         0|        35| 
>          0|      1848|         0|         9|    626650|    626714| 
> 639017|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9| 
>          0|      +1.0|         0|       0.0|      +0.4|      +0.4| 
> +0.3|          0|          0
> 1          250|  256|  10427.71|  9.10|      1157|         0|        32| 
>          0|      1698|         0|         9|    625738|    625822| 
> 641221|       0.00|       0.00
> 2          250|  256|  10464.71|  9.69|      1085|         0|        32| 
>          0|      1698|         0|         9|    627960|    628040| 
> 640680|       0.00|       0.00
> %             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.4|      +0.4| 
> -0.1|          0|          0
> 1          500|  256|  10442.18|  8.65|      1217|         0|        34| 
>          0|      1830|         0|        10|    626608|    626675| 
> 640081|       0.00|       0.00
> 2          500|  256|  10463.74|  9.63|      1089|         0|        34| 
>          0|      1813|         0|         9|    627904|    627967| 
> 643152|       0.00|       0.00
> %             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0| 
>          0|      -0.9|         0|     -10.0|      +0.2|      +0.2| 
> +0.5|          0|          0
> 1           50|  512|   8505.70|  7.20|      1191|         0|        33| 
>          0|      1716|         0|         7|    510418|    510486| 
> 524904|       0.00|       0.00
> 2           50|  512|   8563.34|  7.24|      1185|         0|        32| 
>          0|      1664|         0|         8|    513877|    513940| 
> 526294|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0| 
>          0|      -3.0|         0|     +14.3|      +0.7|      +0.7| 
> +0.3|          0|          0
> 1          100|  512|   8509.97|  7.00|      1226|         0|        34| 
>          0|      1830|         0|         7|    510674|    510756| 
> 525205|       0.00|       0.00
> 2          100|  512|   8555.22|  7.52|      1144|         0|        34| 
>          0|      1830|         0|         7|    513389|    513471| 
> 526609|       0.00|       0.00
> %             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.5|      +0.5| 
> +0.3|          0|          0
> 1          250|  512|   8507.35|  7.21|      1183|         0|        32| 
>          0|      1681|         0|         7|    510518|    510579| 
> 524294|       0.00|       0.00
> 2          250|  512|   8578.13|  7.01|      1223|         0|        33| 
>          0|      1716|         0|         8|    514765|    514829| 
> 526481|       0.00|       0.00
> %             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1| 
>          0|      +2.1|         0|     +14.3|      +0.8|      +0.8| 
> +0.4|          0|          0
> 1          500|  512|   8521.68|  6.54|      1302|         0|        34| 
>          0|      1813|         0|         8|    511375|    511443| 
> 523357|       0.00|       0.00
> 2          500|  512|   8573.17|  6.94|      1235|         0|        34| 
>          0|      1830|         0|         7|    514467|    514533| 
> 526131|       0.00|       0.00
> %             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0| 
>          0|      +0.9|         0|     -12.5|      +0.6|      +0.6| 
> +0.5|          0|          0
> 1           50| 1024|   8150.01|  7.69|      1060|         0|        32| 
>          0|      1681|         0|         7|    489075|    489155| 
> 503136|       0.00|       0.00
> 2           50| 1024|   8129.35|  7.65|      1062|         0|        34| 
>          0|      1804|         0|         7|    487841|    487915| 
> 502061|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2| 
>          0|      +7.3|         0|       0.0|      -0.3|      -0.3| 
> -0.2|          0|          0
> 1          100| 1024|   8166.06|  7.53|      1083|         0|        34| 
>          0|      1804|         0|         7|    490039|    490104| 
> 502990|       0.00|       0.00
> 2          100| 1024|   8118.23|  7.77|      1044|         0|        33| 
>          0|      1725|         0|         7|    487171|    487236| 
> 501939|       0.00|       0.00
> %             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9| 
>          0|      -4.4|         0|       0.0|      -0.6|      -0.6| 
> -0.2|          0|          0
> 1          250| 1024|   8175.35|  7.58|      1079|         0|        35| 
>          0|      1848|         0|         7|    490597|    490657| 
> 503641|       0.00|       0.00
> 2          250| 1024|   8146.63|  7.59|      1072|         0|        34| 
>          0|      1813|         0|         7|    488874|    488939| 
> 502010|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9| 
>          0|      -1.9|         0|       0.0|      -0.4|      -0.4| 
> -0.3|          0|          0
> 1          500| 1024|   8182.51|  7.61|      1075|         0|        32| 
>          0|      1698|         0|         7|    491028|    491095| 
> 504222|       0.00|       0.00
> 2          500| 1024|   8157.39|  7.58|      1075|         0|        33| 
>          0|      1716|         0|         7|    489521|    489593| 
> 502368|       0.00|       0.00
> %             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1| 
>          0|      +1.1|         0|       0.0|      -0.3|      -0.3| 
> -0.4|          0|          0
> 
> Mon Dec 19 09:51:09 2011
> 
> 1 - avg-old.netperf.host_guest.txt
> 2 - avg-fixed.netperf.host_guest.txt
> 
> ======
> TCP_STREAM
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    715.76| 28.44|        24|         0|        31| 
>          0|      1629|         0|        36|   3206862|   1733540| 
> 3245171|       0.00|       0.00
> 2            1|   64|    714.82| 28.44|        25|         0|        31| 
>          0|      1612|         0|        36|   3204256|   1709508| 
> 3242450|       0.00|       0.00
> %             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0| 
>          0|      -1.0|         0|       0.0|      -0.1|      -1.4| 
> -0.1|          0|          0
> 1            2|   64|   1449.31| 41.23|        35|         0|        33| 
>          0|      1744|         0|        66|   2764635|   1667527| 
> 2815591|       0.00|       0.00
> 2            2|   64|   1442.92| 41.32|        34|         0|        32| 
>          0|      1709|         0|        66|   2751461|   1659498| 
> 2799277|       0.00|       0.00
> %             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1            4|   64|   2671.25| 48.95|        54|         0|        30| 
>          0|      1577|         0|        47|    914085|    289771| 
> 947054|       0.00|       0.00
> 2            4|   64|   2652.68| 48.20|        54|         0|        30| 
>          0|      1594|         0|        46|    882300|    293273| 
> 914249|       0.00|       0.00
> %             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0| 
>          0|      +1.1|         0|      -2.1|      -3.5|      +1.2| 
> -3.5|          0|          0
> 1            1|  256|   1478.61| 29.34|        50|         0|        32| 
>          0|      1709|         0|        64|   3502384|   1541681| 
> 3545371|       0.00|       0.00
> 2            1|  256|   1481.38| 29.27|        50|         0|        32| 
>          0|      1726|         0|        64|   3485623|   1526105| 
> 3528343|       0.00|       0.00
> %             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.5|      -1.0| 
> -0.5|          0|          0
> 1            2|  256|   3095.93| 42.31|        72|         0|        31| 
>          0|      1612|         0|       123|   1521019|    466426| 
> 1637722|       0.00|       0.00
> 2            2|  256|   3098.16| 42.03|        73|         0|        30| 
>          0|      1594|         0|       123|   1519458|    472632| 
> 1632462|       0.00|       0.00
> %             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2| 
>          0|      -1.1|         0|       0.0|      -0.1|      +1.3| 
> -0.3|          0|          0
> 1            4|  256|   6553.60| 38.44|       170|         0|        33| 
>          0|      1761|         0|        38|   1343193|    237604| 
> 1402385|       0.00|       0.00
> 2            4|  256|   6649.37| 38.34|       172|         0|        32| 
>          0|      1726|         0|        32|   1343770|    236320| 
> 1400875|       0.00|       0.00
> %             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0| 
>          0|      -2.0|         0|     -15.8|      +0.0|      -0.5| 
> -0.1|          0|          0
> 1            1|  512|   1818.40| 29.28|        61|         0|        31| 
>          0|      1646|         0|        76|   3326887|   1491961| 
> 3371189|       0.00|       0.00
> 2            1|  512|   1822.74| 29.45|        61|         0|        30| 
>          0|      1577|         0|        76|   3309934|   1470715| 
> 3352921|       0.00|       0.00
> %             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2| 
>          0|      -4.2|         0|       0.0|      -0.5|      -1.4| 
> -0.5|          0|          0
> 1            2|  512|   3931.72| 43.51|        89|         0|        33| 
>          0|      1761|         0|       137|   1148331|    301194| 
> 1270351|       0.00|       0.00
> 2            2|  512|   3926.16| 43.45|        90|         0|        32| 
>          0|      1726|         0|       135|   1166280|    306110| 
> 1288869|       0.00|       0.00
> %             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0| 
>          0|      -2.0|         0|      -1.5|      +1.6|      +1.6| 
> +1.5|          0|          0
> 1            4|  512|   8634.49| 40.69|       211|         0|        31| 
>          0|      1612|         0|        53|   1869998|    353856| 
> 1948593|       0.00|       0.00
> 2            4|  512|   9137.12| 40.77|       223|         0|        30| 
>          0|      1594|         0|        48|   1879574|    362851| 
> 1959044|       0.00|       0.00
> %             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2| 
>          0|      -1.1|         0|      -9.4|      +0.5|      +2.5| 
> +0.5|          0|          0
> 1            1| 1024|   2042.67| 29.50|        68|         0|        33| 
>          0|      1778|         0|        84|   3087032|   1309160| 
> 3130736|       0.00|       0.00
> 2            1| 1024|   2033.24| 29.51|        68|         0|        32| 
>          0|      1726|         0|        83|   3082898|   1295936| 
> 3125229|       0.00|       0.00
> %             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -1.2|      -0.1|      -1.0| 
> -0.2|          0|          0
> 1            2| 1024|   7703.26| 39.58|       194|         0|        32| 
>          0|      1673|         0|        65|   2522172|    663302| 
> 2684835|       0.00|       0.00
> 2            2| 1024|   7657.00| 39.27|       194|         0|        30| 
>          0|      1594|         0|        62|   2607281|    698035| 
> 2755198|       0.00|       0.00
> %             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2| 
>          0|      -4.7|         0|      -4.6|      +3.4|      +5.2| 
> +2.6|          0|          0
> 1            4| 1024|  11814.84| 39.38|       299|         0|        32| 
>          0|      1700|         0|        28|   2510318|    576071| 
> 2596631|       0.00|       0.00
> 2            4| 1024|  12871.76| 38.93|       330|         0|        32| 
>          0|      1726|         0|        25|   2232437|    632785| 
> 2316035|       0.00|       0.00
> %             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0| 
>          0|      +1.5|         0|     -10.7|     -11.1|      +9.8| 
> -10.8|          0|          0
> TCP_MAERTS
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1            1|   64|    706.98| 26.25|        26|         0|        31| 
>          0|      1656|         0|        80|   2630204|   4185330| 
> 2692397|       0.00|       0.00
> 2            1|   64|    767.45| 27.78|        27|         0|        30| 
>          0|      1594|         0|        85|   2626694|   4317273| 
> 2688799|       0.00|       0.00
> %             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2| 
>          0|      -3.7|         0|      +6.2|      -0.1|      +3.2| 
> -0.1|          0|          0
> 1            2|   64|   2644.57| 42.60|        61|         0|        32| 
>          0|      1700|         0|     34863|     49057|     16791| 
> 204805|       0.00|       0.00
> 2            2|   64|   2721.19| 42.69|        63|         0|        32| 
>          0|      1709|         0|     42365|     52739|     20094| 
> 215895|       0.00|       0.00
> %             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0| 
>          0|      +0.5|         0|     +21.5|      +7.5|     +19.7| 
> +5.4|          0|          0
> 1            4|   64|   2756.40| 42.69|        64|         0|        32| 
>          0|      1708|         0|     52043|     47976|     12694| 
> 220732|       0.00|       0.00
> 2            4|   64|   2709.19| 42.79|        62|         0|        30| 
>          0|      1594|         0|     50144|     47275|     12743| 
> 218074|       0.00|       0.00
> %             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2| 
>          0|      -6.7|         0|      -3.6|      -1.5|      +0.4| 
> -1.2|          0|          0
> 1            1|  256|   3785.63| 32.34|       116|         0|        33| 
>          0|      1734|         0|     36642|     58940|      1187| 
> 157652|       0.00|       0.00
> 2            1|  256|   3593.17| 32.90|       108|         0|        32| 
>          0|      1726|         0|     33179|     54980|      1130| 
> 150168|       0.00|       0.00
> %             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0| 
>          0|      -0.5|         0|      -9.5|      -6.7|      -4.8| 
> -4.7|          0|          0
> 1            2|  256|   8316.26| 40.45|       205|         0|        32| 
>          0|      1673|         0|     28302|    114835|      1655| 
> 251137|       0.00|       0.00
> 2            2|  256|   8390.41| 41.17|       203|         0|        30| 
>          0|      1594|         0|     31222|    115372|      1696| 
> 257151|       0.00|       0.00
> %             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2| 
>          0|      -4.7|         0|     +10.3|      +0.5|      +2.5| 
> +2.4|          0|          0
> 1            4|  256|   9672.49| 42.40|       227|         0|        33| 
>          0|      1761|         0|     30045|    132027|      3059| 
> 283491|       0.00|       0.00
> 2            4|  256|   9741.71| 42.57|       228|         0|        32| 
>          0|      1726|         6|     28875|    132125|      1999| 
> 283036|       0.00|       0.00
> %             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0| 
>          0|      -2.0|         0|      -3.9|      +0.1|     -34.7| 
> -0.2|          0|          0
> 1            1|  512|   6052.41| 32.23|       187|         0|        31| 
>          0|      1612|         0|     44681|     86084|      2592| 
> 187465|       0.00|       0.00
> 2            1|  512|   6267.37| 32.51|       192|         0|        30| 
>          0|      1577|         0|     45278|     88449|      2708| 
> 191374|       0.00|       0.00
> %             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2| 
>          0|      -2.2|         0|      +1.3|      +2.7|      +4.5| 
> +2.1|          0|          0
> 1            2|  512|  12714.52| 38.27|       331|         0|        33| 
>          0|      1744|         0|     24843|    169170|      2381| 
> 292210|       0.00|       0.00
> 2            2|  512|  11341.65| 37.30|       303|         0|        32| 
>          0|      1726|        14|     27393|    152465|      2980| 
> 270740|       0.00|       0.00
> %             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0| 
>          0|      -1.0|         0|     +10.3|      -9.9|     +25.2| 
> -7.3|          0|          0
> 1            4|  512|  16127.03| 40.80|       394|         0|        31| 
>          0|      1612|         0|     15063|    210628|      2234| 
> 344124|       0.00|       0.00
> 2            4|  512|  14709.72| 39.94|       367|         0|        31| 
>          0|      1612|         0|     17823|    193670|      4457| 
> 324744|       0.00|       0.00
> %             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0| 
>          0|       0.0|         0|     +18.3|      -8.1|     +99.5| 
> -5.6|          0|          0
> 1            1| 1024|   6436.88| 30.31|       212|         0|        33| 
>          0|      1778|         0|     47820|     83940|      2007| 
> 172847|       0.00|       0.00
> 2            1| 1024|   6495.48| 30.51|       212|         0|        32| 
>          0|      1726|         0|     47771|     85436|      2038| 
> 175027|       0.00|       0.00
> %             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0| 
>          0|      -2.9|         0|      -0.1|      +1.8|      +1.5| 
> +1.3|          0|          0
> 1            2| 1024|  12245.34| 33.44|       365|         0|        31| 
>          0|      1629|         0|     26499|    162142|      2909| 
> 255717|       0.00|       0.00
> 2            2| 1024|  12913.57| 33.93|       380|         0|        31| 
>          0|      1612|         0|     24196|    171386|      3671| 
> 266270|       0.00|       0.00
> %             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0| 
>          0|      -1.0|         0|      -8.7|      +5.7|     +26.2| 
> +4.1|          0|          0
> 1            4| 1024|  16428.48| 34.81|       471|         0|        32| 
>          0|      1709|         0|     12496|    215031|      6557| 
> 318039|       0.00|       0.00
> 2            4| 1024|  17294.33| 35.05|       492|         0|        32| 
>          0|      1709|         0|      9444|    224694|      4095| 
> 328098|       0.00|       0.00
> %             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0| 
>          0|       0.0|         0|     -24.4|      +4.5|     -37.5| 
> +3.2|          0|          0
> TCP_RR
>        sessions| size|throughput|   cpu| normalize|  #tx-pkts| 
> #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr| 
> #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
> 1           50|   64|  14626.43| 14.93|       980|         0|        32| 
>          0|      1681|         0|        13|    877664|    877731| 
> 902624|       0.00|       0.00
> 2           50|   64|  14639.95| 13.10|      1123|         0|        32| 
>          0|      1681|         0|        13|    878475|    878538| 
> 897278|       0.00|       0.00
> %             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.1|      +0.1| 
> -0.6|          0|          0
> 1          100|   64|  14697.50| 14.31|      1029|         0|        34| 
>          0|      1830|         0|        13|    881930|    881991| 
> 903837|       0.00|       0.00
> 2          100|   64|  14692.35| 13.08|      1128|         0|        34| 
>          0|      1830|         0|        13|    881619|    881679| 
> 898952|       0.00|       0.00
> %             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          250|   64|  14683.40| 14.34|      1027|         0|        32| 
>          0|      1698|         0|        13|    881083|    881147| 
> 903071|       0.00|       0.00
> 2          250|   64|  14677.06| 12.85|      1144|         0|        32| 
>          0|      1698|         0|        13|    880702|    880762| 
> 898190|       0.00|       0.00
> %             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.0|      -0.0| 
> -0.5|          0|          0
> 1          500|   64|  14698.45| 14.39|      1025|         0|        34| 
>          0|      1830|         0|        13|    881985|    882060| 
> 905537|       0.00|       0.00
> 2          500|   64|  14626.17| 13.54|      1083|         0|        34| 
>          0|      1830|         0|        13|    877648|    877726| 
> 896427|       0.00|       0.00
> %             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0| 
>          0|       0.0|         0|       0.0|      -0.5|      -0.5| 
> -1.0|          0|          0
> 1           50|  256|  14386.52| 14.14|      1018|         0|        33| 
>          0|      1716|         0|        13|    863266|    863336| 
> 887195|       0.00|       0.00
> 2           50|  256|  14359.26| 14.22|      1016|         0|        32| 
>          0|      1681|         0|        13|    861630|    861698| 
> 882444|       0.00|       0.00
> %             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.2|      -0.2| 
> -0.5|          0|          0
> 1          100|  256|  14364.89| 14.25|      1008|         0|        34| 
>          0|      1813|         0|        13|    861969|    862033| 
> 886679|       0.00|       0.00
> 2          100|  256|  14393.82| 14.53|       999|         0|        34| 
>          0|      1813|         0|        13|    863704|    863765| 
> 884487|       0.00|       0.00
> %             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0| 
>          0|       0.0|         0|       0.0|      +0.2|      +0.2| 
> -0.2|          0|          0
> 1          250|  256|  14387.95| 14.05|      1024|         0|        32| 
>          0|      1681|         0|        13|    863352|    863429| 
> 887505|       0.00|       0.00
> 2          250|  256|  14344.74| 14.62|       987|         0|        32| 
>          0|      1698|         0|        13|    860760|    860843| 
> 882154|       0.00|       0.00
> %             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0| 
>          0|      +1.0|         0|       0.0|      -0.3|      -0.3| 
> -0.6|          0|          0
> 1          500|  256|  14365.89| 14.19|      1013|         0|        33| 
>          0|      1769|         0|        13|    862029|    862099| 
> 885852|       0.00|       0.00
> 2          500|  256|  14329.59| 14.62|       990|         0|        34| 
>          0|      1830|         0|        13|    859852|    861233| 
> 880082|       0.00|       0.00
> %             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.3|      -0.1| 
> -0.7|          0|          0
> 1           50|  512|  14273.07| 14.12|      1010|         0|        33| 
>          0|      1742|         0|        13|    856460|    856524| 
> 880356|       0.00|       0.00
> 2           50|  512|  14222.08| 14.72|       966|         0|        32| 
>          0|      1698|         0|        12|    853398|    853468| 
> 875621|       0.00|       0.00
> %             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0| 
>          0|      -2.5|         0|      -7.7|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1          100|  512|  14236.34| 14.51|       982|         0|        34| 
>          0|      1804|         0|        13|    854258|    854335| 
> 876662|       0.00|       0.00
> 2          100|  512|  14147.89| 14.85|       953|         0|        34| 
>          0|      1813|         0|        13|    848947|    849031| 
> 871773|       0.00|       0.00
> %             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0| 
>          0|      +0.5|         0|       0.0|      -0.6|      -0.6| 
> -0.6|          0|          0
> 1          250|  512|  14260.79| 14.53|       983|         0|        33| 
>          0|      1760|         0|        13|    855721|    855791| 
> 879099|       0.00|       0.00
> 2          250|  512|  14190.68| 14.87|       954|         0|        32| 
>          0|      1698|         0|        13|    851517|    851580| 
> 874119|       0.00|       0.00
> %             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0| 
>          0|      -3.5|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          500|  512|  14273.71| 14.18|      1008|         0|        33| 
>          0|      1769|         0|        13|    856499|    856562| 
> 879984|       0.00|       0.00
> 2          500|  512|  14213.87| 14.72|       967|         0|        34| 
>          0|      1830|         0|        13|    852907|    852976| 
> 875328|       0.00|       0.00
> %             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0| 
>          0|      +3.4|         0|       0.0|      -0.4|      -0.4| 
> -0.5|          0|          0
> 1           50| 1024|  14049.77| 13.89|      1024|         0|        32| 
>          0|      1708|         0|        12|    843061|    843134| 
> 863697|       0.00|       0.00
> 2           50| 1024|  13980.25| 14.38|       980|         0|        32| 
>          0|      1698|         0|        12|    838890|    838973| 
> 858350|       0.00|       0.00
> %             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0| 
>          0|      -0.6|         0|       0.0|      -0.5|      -0.5| 
> -0.6|          0|          0
> 1          100| 1024|  14025.66| 13.32|      1066|         0|        33| 
>          0|      1769|         0|        13|    841616|    841685| 
> 864460|       0.00|       0.00
> 2          100| 1024|  13993.30| 14.82|       946|         0|        34| 
>          0|      1830|         0|        12|    839676|    839735| 
> 859288|       0.00|       0.00
> %             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0| 
>          0|      +3.4|         0|      -7.7|      -0.2|      -0.2| 
> -0.6|          0|          0
> 1          250| 1024|  14060.15| 13.90|      1023|         0|        35| 
>          0|      1848|         0|        12|    843688|    843752| 
> 864475|       0.00|       0.00
> 2          250| 1024|  13982.47| 15.34|       913|         0|        34| 
>          0|      1830|         0|        12|    839026|    839089| 
> 859914|       0.00|       0.00
> %             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9| 
>          0|      -1.0|         0|       0.0|      -0.6|      -0.6| 
> -0.5|          0|          0
> 1          500| 1024|  14051.14| 13.80|      1024|         0|        33| 
>          0|      1716|         0|        13|    843144|    843212| 
> 862981|       0.00|       0.00
> 2          500| 1024|  14013.95| 15.27|       918|         0|        32| 
>          0|      1681|         0|        13|    840915|    840983| 
> 862635|       0.00|       0.00
> %             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0| 
>          0|      -2.0|         0|       0.0|      -0.3|      -0.3| 
> -0.0|          0|          0
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 19/12/11 10:41, Benjamin Herrenschmidt wrote:
> On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:
>
>> I tested with the same environment and scenarios.
>> tested one scenarios for three times and compute the average for more
>> precision.
>>
>> Thanks, Amos
>>
>> --------- compare results -----------
>> Mon Dec 19 09:51:09 2011
>>
>> 1 - avg-old.netperf.exhost_guest.txt
>> 2 - avg-fixed.netperf.exhost_guest.txt
>
> The output is word wrapped and generally unreadable. Any chance you can
> provide us with a summary of the outcome ?
>
> Cheers,
> Ben.

Hi Ben,

The change of TCP_RR Throughput is very small.
external host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
reduced a little.
local host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
increased a little.


About compare result format:
---------------------------
>> 1 - avg-old.netperf.exhost_guest.txt

average result (tested 3 times) file of test 1
>> 2 - avg-fixed.netperf.exhost_guest.txt

average result file of test 2
>>
>> ======
>> TCP_STREAM

^^^ protocol

>>       sessions| size|throughput|   cpu| normalize|  #tx-pkts| #rx-pkts| #tx-byts|  #rx-byts| #re-trans| #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
>> 1            1|   64|   1073.54| 10.50|       102|  ....

^^^ average result of old kernel, start netserver in guest, start 
netperf client(s) in external host

>> 2            1|   64|   1079.44| 10.29|       104|  ....

^^^ average result of fixed kernel

>> %             |  0.0|      +0.5|  -2.0|      +2.0|  ....

^^^ augment rate between test1 and test2

--------
sessions: netperf clients number
size: request/response sizes
#rx-pkts: received packets number
#rx-byts: received bytes number
#rx-intr: interrupt number for receive
#io_exit: io exit number
#irq_inj: injected irq number


Thanks, Amos.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 19/12/11 10:41, Benjamin Herrenschmidt wrote:
> On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:
>
>> I tested with the same environment and scenarios.
>> tested one scenarios for three times and compute the average for more
>> precision.
>>
>> Thanks, Amos
>>
>> --------- compare results -----------
>> Mon Dec 19 09:51:09 2011
>>
>> 1 - avg-old.netperf.exhost_guest.txt
>> 2 - avg-fixed.netperf.exhost_guest.txt
>
> The output is word wrapped and generally unreadable. Any chance you can
> provide us with a summary of the outcome ?
>
> Cheers,
> Ben.

Hi Ben,

The change of TCP_RR Throughput is very small.
external host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
reduced a little.
local host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
increased a little.


About compare result format:
---------------------------
>> 1 - avg-old.netperf.exhost_guest.txt

average result (tested 3 times) file of test 1
>> 2 - avg-fixed.netperf.exhost_guest.txt

average result file of test 2
>>
>> ======
>> TCP_STREAM

^^^ protocol

>>       sessions| size|throughput|   cpu| normalize|  #tx-pkts| #rx-pkts| #tx-byts|  #rx-byts| #re-trans| #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
>> 1            1|   64|   1073.54| 10.50|       102|  ....

^^^ average result of old kernel, start netserver in guest, start 
netperf client(s) in external host

>> 2            1|   64|   1079.44| 10.29|       104|  ....

^^^ average result of fixed kernel

>> %             |  0.0|      +0.5|  -2.0|      +2.0|  ....

^^^ augment rate between test1 and test2

--------
sessions: netperf clients number
size: request/response sizes
#rx-pkts: received packets number
#rx-byts: received bytes number
#rx-intr: interrupt number for receive
#io_exit: io exit number
#irq_inj: injected irq number


Thanks, Amos.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 19/12/11 10:41, Benjamin Herrenschmidt wrote:
> On Mon, 2011-12-19 at 10:19 +0800, Amos Kong wrote:
>
>> I tested with the same environment and scenarios.
>> tested one scenarios for three times and compute the average for more
>> precision.
>>
>> Thanks, Amos
>>
>> --------- compare results -----------
>> Mon Dec 19 09:51:09 2011
>>
>> 1 - avg-old.netperf.exhost_guest.txt
>> 2 - avg-fixed.netperf.exhost_guest.txt
>
> The output is word wrapped and generally unreadable. Any chance you can
> provide us with a summary of the outcome ?
>
> Cheers,
> Ben.

Hi Ben,

The change of TCP_RR Throughput is very small.
external host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
reduced a little.
local host -> guest: Some of throughput of TCP_STREAM and TCP_MAERTS 
increased a little.


About compare result format:
---------------------------
>> 1 - avg-old.netperf.exhost_guest.txt

average result (tested 3 times) file of test 1
>> 2 - avg-fixed.netperf.exhost_guest.txt

average result file of test 2
>>
>> ======
>> TCP_STREAM

^^^ protocol

>>       sessions| size|throughput|   cpu| normalize|  #tx-pkts| #rx-pkts| #tx-byts|  #rx-byts| #re-trans| #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
>> 1            1|   64|   1073.54| 10.50|       102|  ....

^^^ average result of old kernel, start netserver in guest, start 
netperf client(s) in external host

>> 2            1|   64|   1079.44| 10.29|       104|  ....

^^^ average result of fixed kernel

>> %             |  0.0|      +0.5|  -2.0|      +2.0|  ....

^^^ augment rate between test1 and test2

--------
sessions: netperf clients number
size: request/response sizes
#rx-pkts: received packets number
#rx-byts: received bytes number
#rx-intr: interrupt number for receive
#io_exit: io exit number
#irq_inj: injected irq number


Thanks, Amos.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

[-- Attachment #1: Type: text/plain, Size: 3122 bytes --]

On 19/12/11 10:19, Amos Kong wrote:
> On 12/12/11 13:12, Rusty Russell wrote:
>> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com> wrote:
>>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>>
>>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>>> parallel, from an external box to the guest. TCP_STREAM results were
>>>>> noisy. This could be due to buffering done by TCP, where packet size
>>>>> varies even as message size is constant.
>>>>>
>>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>>> regressed consistently, between 7 and 35%
>>>>>
>>>>> The "fix" applied was simply this:
>>>>
>>>> What machine& processor was this ?
>>>
>>> pined guest memory to numa node 1
>>
>> Please try this patch. How much does the branch cost us?
>>
>> (Compiles, untested).
>>
>> Thanks,
>> Rusty.
>>
>> From: Rusty Russell<rusty@rustcorp.com.au>
>> Subject: virtio: harsher barriers for virtio-mmio.
>>
>> We were cheating with our barriers; using the smp ones rather than the
>> real device ones. That was fine, until virtio-mmio came along, which
>> could be talking to a real device (a non-SMP CPU).
>>
>> Unfortunately, just putting back the real barriers (reverting
>> d57ed95d) causes a performance regression on virtio-pci. In
>> particular, Amos reports netbench's TCP_RR over virtio_net CPU
>> utilization increased up to 35% while throughput went down by up to
>> 14%.
>>
>> By comparison, this branch costs us???
>>
>> Reference: https://lkml.org/lkml/2011/12/11/22
>>
>> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
>> ---
>> drivers/lguest/lguest_device.c | 10 ++++++----
>> drivers/s390/kvm/kvm_virtio.c | 2 +-
>> drivers/virtio/virtio_mmio.c | 7 ++++---
>> drivers/virtio/virtio_pci.c | 4 ++--
>> drivers/virtio/virtio_ring.c | 34 +++++++++++++++++++++-------------
>> include/linux/virtio_ring.h | 1 +
>> tools/virtio/linux/virtio.h | 1 +
>> tools/virtio/virtio_test.c | 3 ++-
>> 8 files changed, 38 insertions(+), 24 deletions(-)
>
> Hi all,
>
> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more
> precision.
>
> Thanks, Amos
>
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
>
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt
>
> ======
> TCP_STREAM
> sessions| size|throughput| cpu| normalize| #tx-pkts| #rx-pkts| #tx-byts|
> #rx-byts| #re-trans| #tx-intr| #rx-intr| #io_exit| #irq_inj|#tpkt/#exit|
> #rpkt/#irq
> 1 1| 64| 1073.54| 10.50| 102| 0| 31| 0| 1612| 0| 16| 487641| 489753|
> 504764| 0.00| 0.00
> 2 1| 64| 1079.44| 10.29| 104| 0| 30| 0| 1594| 0| 17| 487156| 488828|
> 504411| 0.00| 0.00
> % | 0.0| +0.5| -2.0| +2.0| 0| -3.2| 0| -1.1| 0| +6.2| -0.1| -0.2| -0.1|

The format is broken in webpage, attached the result file.
it's also available here: http://amosk.info/download/rusty-fix-perf.txt

[-- Attachment #2: rusty-fix-perf.txt --]
[-- Type: text/plain, Size: 42872 bytes --]

--------- compare results -----------
Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.exhost_guest.txt
2 - avg-fixed.netperf.exhost_guest.txt

======
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1073.54| 10.50|       102|         0|        31|         0|      1612|         0|        16|    487641|    489753|    504764|       0.00|       0.00
2            1|   64|   1079.44| 10.29|       104|         0|        30|         0|      1594|         0|        17|    487156|    488828|    504411|       0.00|       0.00
%             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2|         0|      -1.1|         0|      +6.2|      -0.1|      -0.2|      -0.1|          0|          0
1            2|   64|   2141.12| 15.72|       136|         0|        33|         0|      1744|         0|        34|    873777|    972303|    928926|       0.00|       0.00
2            2|   64|   2140.88| 15.64|       137|         0|        33|         0|      1744|         0|        34|    926588|    942841|    974095|       0.00|       0.00
%             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0|         0|       0.0|         0|       0.0|      +6.0|      -3.0|      +4.9|          0|          0
1            4|   64|   4076.80| 19.82|       205|         0|        30|         0|      1577|         0|        67|   1422282|   1166425|   1539219|       0.00|       0.00
2            4|   64|   4094.32| 20.70|       197|         0|        31|         0|      1612|         0|        68|   1704330|   1314077|   1833394|       0.00|       0.00
%             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3|         0|      +2.2|         0|      +1.5|     +19.8|     +12.7|     +19.1|          0|          0
1            1|  256|   2867.48| 13.44|       213|         0|        32|         0|      1726|         0|        14|    666430|    694922|    690730|       0.00|       0.00
2            1|  256|   2874.20| 12.71|       226|         0|        32|         0|      1709|         0|        14|    697960|    740407|    721807|       0.00|       0.00
%             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0|         0|      -1.0|         0|       0.0|      +4.7|      +6.5|      +4.5|          0|          0
1            2|  256|   5642.82| 17.61|       320|         0|        30|         0|      1594|         0|        30|   1226861|   1236081|   1268562|       0.00|       0.00
2            2|  256|   5661.06| 17.41|       326|         0|        30|         0|      1594|         0|        29|   1175696|   1143490|   1221528|       0.00|       0.00
%             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0|         0|       0.0|         0|      -3.3|      -4.2|      -7.5|      -3.7|          0|          0
1            4|  256|   9404.27| 23.55|       399|         0|        33|         0|      1744|         0|        37|   1692245|    659975|   1765103|       0.00|       0.00
2            4|  256|   8761.11| 23.18|       376|         0|        32|         0|      1726|         0|        36|   1699382|    418992|   1870804|       0.00|       0.00
%             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0|         0|      -1.0|         0|      -2.7|      +0.4|     -36.5|      +6.0|          0|          0
1            1|  512|   3803.66| 14.20|       267|         0|        30|         0|      1594|         0|        14|    693992|    750078|    721107|       0.00|       0.00
2            1|  512|   3838.02| 15.47|       248|         0|        31|         0|      1612|         0|        15|    811709|    773505|    838788|       0.00|       0.00
%             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3|         0|      +1.1|         0|      +7.1|     +17.0|      +3.1|     +16.3|          0|          0
1            2|  512|   8606.11| 19.34|       444|         0|        32|         0|      1709|         0|        29|   1264624|    647652|   1309740|       0.00|       0.00
2            2|  512|   8127.80| 18.93|       428|         0|        32|         0|      1726|         0|        28|   1216606|   1179269|   1266260|       0.00|       0.00
%             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0|         0|      +1.0|         0|      -3.4|      -3.8|     +82.1|      -3.3|          0|          0
1            4|  512|   9409.41| 22.88|       413|         0|        30|         0|      1577|         0|        35|   1592587|   1072862|   1746787|       0.00|       0.00
2            4|  512|   9217.34| 23.05|       400|         0|        30|         0|      1594|         0|        34|   1596515|    513742|   1831538|       0.00|       0.00
%             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0|         0|      +1.1|         0|      -2.9|      +0.2|     -52.1|      +4.9|          0|          0
1            1| 1024|   4389.75| 14.62|       303|         0|        32|         0|      1726|         0|        14|    663417|    611555|    689028|       0.00|       0.00
2            1| 1024|   4390.44| 13.66|       321|         0|        32|         0|      1726|         0|        13|    609836|    556090|    634511|       0.00|       0.00
%             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0|         0|       0.0|         0|      -7.1|      -8.1|      -9.1|      -7.9|          0|          0
1            2| 1024|   9046.18| 19.63|       460|         0|        30|         0|      1594|         0|        28|   1206853|   1054653|   1256203|       0.00|       0.00
2            2| 1024|   9027.54| 20.15|       447|         0|        30|         0|      1577|         0|        27|   1179555|    884408|   1226843|       0.00|       0.00
%             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0|         0|      -1.1|         0|      -3.6|      -2.3|     -16.1|      -2.3|          0|          0
1            4| 1024|   9410.41| 22.01|       427|         0|        32|         0|      1726|         0|        36|   1734433|    887190|   1852486|       0.00|       0.00
2            4| 1024|   9409.11| 23.18|       405|         0|        32|         0|      1726|         0|        34|   1695359|    625268|   1803599|       0.00|       0.00
%             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0|         0|       0.0|         0|      -5.6|      -2.3|     -29.5|      -2.6|          0|          0
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1056.13| 17.51|        59|         0|        30|         0|      1594|         3|        86|    476098|   4780186|    538158|       0.00|       0.00
2            1|   64|   1065.51| 17.64|        60|         0|        30|         0|      1594|         3|        87|    476836|   4823602|    538860|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0|         0|       0.0|       0.0|      +1.2|      +0.2|      +0.9|      +0.1|          0|          0
1            2|   64|   2472.80| 32.18|        76|         0|        32|         0|      1709|         2|       199|    479119|   2089931|    600119|       0.00|       0.00
2            2|   64|   2468.76| 32.28|        76|         0|        33|         0|      1744|         1|       199|    479372|   2179342|    600328|       0.00|       0.00
%             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1|         0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3|      +0.0|          0|          0
1            4|   64|   2453.22| 32.20|        75|         0|        31|         0|      1612|         2|       198|    479223|   2175801|    600081|       0.00|       0.00
2            4|   64|   2465.68| 32.30|        76|         0|        30|         0|      1594|         3|       200|    479404|   2108293|    600252|       0.00|       0.00
%             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2|         0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1|      +0.0|          0|          0
1            1|  256|   3608.37| 24.59|       146|         0|        32|         0|      1726|         2|       277|     88346|    108310|    150637|       0.00|       0.00
2            1|  256|   3611.12| 24.95|       144|         0|        32|         0|      1709|         0|       281|     83319|     64354|    145511|       0.00|       0.00
%             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0|         0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6|      -3.4|          0|          0
1            2|  256|   7373.32| 26.83|       274|         0|        31|         0|      1612|         0|      4185|    278932|    264031|    377267|       0.00|       0.00
2            2|  256|   7460.03| 26.93|       276|         0|        30|         0|      1594|         1|      5913|    270840|    246165|    371999|       0.00|       0.00
%             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2|         0|      -1.1|         0|     +41.3|      -2.9|      -6.8|      -1.4|          0|          0
1            4|  256|   9322.66| 29.73|       313|         0|        32|         0|      1709|       848|      3041|    363114|    329308|    482586|       0.00|       0.00
2            4|  256|   9365.52| 29.59|       316|         0|        33|         0|      1744|       414|      2884|    371485|    339191|    491366|       0.00|       0.00
%             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1|         0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0|      +1.8|          0|          0
1            1|  512|   9360.25| 17.20|       543|         0|        30|         0|      1577|         0|       387|    438621|    408062|    499989|       0.00|       0.00
2            1|  512|   8899.37| 18.17|       495|         0|        30|         0|      1594|         0|      5049|    393341|    371545|    459397|       0.00|       0.00
%             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0|         0|      +1.1|         0|   +1204.7|     -10.3|      -8.9|      -8.1|          0|          0
1            2|  512|   8713.11| 20.77|       420|         0|        32|         0|      1726|         0|     11227|    296157|    246495|    371632|       0.00|       0.00
2            2|  512|   8686.31| 20.72|       421|         0|        32|         0|      1709|         0|     11002|    296608|    246597|    371797|       0.00|       0.00
%             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0|         0|      -1.0|         0|      -2.0|      +0.2|      +0.0|      +0.0|          0|          0
1            4|  512|   8780.40| 22.72|       386|         0|        30|         0|      1594|         0|     10337|    303619|    249822|    385457|       0.00|       0.00
2            4|  512|   8858.51| 22.66|       390|         0|        30|         0|      1577|         0|      9968|    301847|    246543|    382121|       0.00|       0.00
%             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0|         0|      -1.1|         0|      -3.6|      -0.6|      -1.3|      -0.9|          0|          0
1            1| 1024|   9387.09| 16.43|       571|         0|        32|         0|      1726|         0|        84|    452351|    429483|    504022|       0.00|       0.00
2            1| 1024|   9388.90| 16.51|       568|         0|        32|         0|      1726|         2|        76|    446994|    422489|    499025|       0.00|       0.00
%             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0|         0|       0.0|         0|      -9.5|      -1.2|      -1.6|      -1.0|          0|          0
1            2| 1024|   9361.43| 17.26|       542|         0|        30|         0|      1594|         5|       647|    388626|    358303|    442125|       0.00|       0.00
2            2| 1024|   9364.78| 17.25|       542|         0|        31|         0|      1612|        35|       554|    389326|    360169|    442343|       0.00|       0.00
%             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3|         0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5|      +0.0|          0|          0
1            4| 1024|   9381.57| 18.35|       510|         0|        32|         0|      1726|      1189|       648|    405900|    376269|    459845|       0.00|       0.00
2            4| 1024|   9393.48| 18.27|       513|         0|        32|         0|      1726|      1754|       610|    402132|    373768|    455629|       0.00|       0.00
%             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0|         0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7|      -0.9|          0|          0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10903.70|  8.03|      1382|         0|        32|         0|      1698|         0|         9|    654298|    654362|    665532|       0.00|       0.00
2           50|   64|  10880.50|  7.40|      1488|         0|        32|         0|      1698|         0|        10|    652907|    652973|    667042|       0.00|       0.00
%             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0|         0|       0.0|         0|     +11.1|      -0.2|      -0.2|      +0.2|          0|          0
1          100|   64|  10953.00|  8.40|      1332|         0|        34|         0|      1830|         0|        10|    657257|    657325|    667088|       0.00|       0.00
2          100|   64|  10882.78|  7.67|      1439|         0|        34|         0|      1830|         0|         9|    653044|    653110|    668883|       0.00|       0.00
%             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0|         0|       0.0|         0|     -10.0|      -0.6|      -0.6|      +0.3|          0|          0
1          250|   64|  10902.66|  8.57|      1300|         0|        33|         0|      1716|         0|        10|    654235|    654308|    665518|       0.00|       0.00
2          250|   64|  10866.79|  7.58|      1449|         0|        32|         0|      1664|         0|        10|    652083|    652155|    666869|       0.00|       0.00
%             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0|         0|      -3.0|         0|       0.0|      -0.3|      -0.3|      +0.2|          0|          0
1          500|   64|  10924.34|  7.25|      1509|         0|        34|         0|      1830|         0|        10|    655536|    655619|    665776|       0.00|       0.00
2          500|   64|  10927.52|  6.50|      1684|         0|        34|         0|      1813|         0|        10|    655726|    655803|    668527|       0.00|       0.00
%             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0|         0|      -0.9|         0|       0.0|      +0.0|      +0.0|      +0.4|          0|          0
1           50|  256|  10407.35|  9.38|      1112|         0|        32|         0|      1681|         0|         9|    624516|    624581|    639159|       0.00|       0.00
2           50|  256|  10422.60|  9.45|      1108|         0|        33|         0|      1716|         0|         9|    625432|    625498|    641343|       0.00|       0.00
%             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1|         0|      +2.1|         0|       0.0|      +0.1|      +0.1|      +0.3|          0|          0
1          100|  256|  10404.30|  8.99|      1159|         0|        34|         0|      1830|         0|         9|    624335|    624400|    637357|       0.00|       0.00
2          100|  256|  10442.84|  9.57|      1093|         0|        35|         0|      1848|         0|         9|    626650|    626714|    639017|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9|         0|      +1.0|         0|       0.0|      +0.4|      +0.4|      +0.3|          0|          0
1          250|  256|  10427.71|  9.10|      1157|         0|        32|         0|      1698|         0|         9|    625738|    625822|    641221|       0.00|       0.00
2          250|  256|  10464.71|  9.69|      1085|         0|        32|         0|      1698|         0|         9|    627960|    628040|    640680|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0|         0|       0.0|         0|       0.0|      +0.4|      +0.4|      -0.1|          0|          0
1          500|  256|  10442.18|  8.65|      1217|         0|        34|         0|      1830|         0|        10|    626608|    626675|    640081|       0.00|       0.00
2          500|  256|  10463.74|  9.63|      1089|         0|        34|         0|      1813|         0|         9|    627904|    627967|    643152|       0.00|       0.00
%             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0|         0|      -0.9|         0|     -10.0|      +0.2|      +0.2|      +0.5|          0|          0
1           50|  512|   8505.70|  7.20|      1191|         0|        33|         0|      1716|         0|         7|    510418|    510486|    524904|       0.00|       0.00
2           50|  512|   8563.34|  7.24|      1185|         0|        32|         0|      1664|         0|         8|    513877|    513940|    526294|       0.00|       0.00
%             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0|         0|      -3.0|         0|     +14.3|      +0.7|      +0.7|      +0.3|          0|          0
1          100|  512|   8509.97|  7.00|      1226|         0|        34|         0|      1830|         0|         7|    510674|    510756|    525205|       0.00|       0.00
2          100|  512|   8555.22|  7.52|      1144|         0|        34|         0|      1830|         0|         7|    513389|    513471|    526609|       0.00|       0.00
%             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0|         0|       0.0|         0|       0.0|      +0.5|      +0.5|      +0.3|          0|          0
1          250|  512|   8507.35|  7.21|      1183|         0|        32|         0|      1681|         0|         7|    510518|    510579|    524294|       0.00|       0.00
2          250|  512|   8578.13|  7.01|      1223|         0|        33|         0|      1716|         0|         8|    514765|    514829|    526481|       0.00|       0.00
%             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1|         0|      +2.1|         0|     +14.3|      +0.8|      +0.8|      +0.4|          0|          0
1          500|  512|   8521.68|  6.54|      1302|         0|        34|         0|      1813|         0|         8|    511375|    511443|    523357|       0.00|       0.00
2          500|  512|   8573.17|  6.94|      1235|         0|        34|         0|      1830|         0|         7|    514467|    514533|    526131|       0.00|       0.00
%             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0|         0|      +0.9|         0|     -12.5|      +0.6|      +0.6|      +0.5|          0|          0
1           50| 1024|   8150.01|  7.69|      1060|         0|        32|         0|      1681|         0|         7|    489075|    489155|    503136|       0.00|       0.00
2           50| 1024|   8129.35|  7.65|      1062|         0|        34|         0|      1804|         0|         7|    487841|    487915|    502061|       0.00|       0.00
%             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2|         0|      +7.3|         0|       0.0|      -0.3|      -0.3|      -0.2|          0|          0
1          100| 1024|   8166.06|  7.53|      1083|         0|        34|         0|      1804|         0|         7|    490039|    490104|    502990|       0.00|       0.00
2          100| 1024|   8118.23|  7.77|      1044|         0|        33|         0|      1725|         0|         7|    487171|    487236|    501939|       0.00|       0.00
%             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9|         0|      -4.4|         0|       0.0|      -0.6|      -0.6|      -0.2|          0|          0
1          250| 1024|   8175.35|  7.58|      1079|         0|        35|         0|      1848|         0|         7|    490597|    490657|    503641|       0.00|       0.00
2          250| 1024|   8146.63|  7.59|      1072|         0|        34|         0|      1813|         0|         7|    488874|    488939|    502010|       0.00|       0.00
%             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9|         0|      -1.9|         0|       0.0|      -0.4|      -0.4|      -0.3|          0|          0
1          500| 1024|   8182.51|  7.61|      1075|         0|        32|         0|      1698|         0|         7|    491028|    491095|    504222|       0.00|       0.00
2          500| 1024|   8157.39|  7.58|      1075|         0|        33|         0|      1716|         0|         7|    489521|    489593|    502368|       0.00|       0.00
%             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1|         0|      +1.1|         0|       0.0|      -0.3|      -0.3|      -0.4|          0|          0

Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.host_guest.txt
2 - avg-fixed.netperf.host_guest.txt

======
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    715.76| 28.44|        24|         0|        31|         0|      1629|         0|        36|   3206862|   1733540|   3245171|       0.00|       0.00
2            1|   64|    714.82| 28.44|        25|         0|        31|         0|      1612|         0|        36|   3204256|   1709508|   3242450|       0.00|       0.00
%             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0|         0|      -1.0|         0|       0.0|      -0.1|      -1.4|      -0.1|          0|          0
1            2|   64|   1449.31| 41.23|        35|         0|        33|         0|      1744|         0|        66|   2764635|   1667527|   2815591|       0.00|       0.00
2            2|   64|   1442.92| 41.32|        34|         0|        32|         0|      1709|         0|        66|   2751461|   1659498|   2799277|       0.00|       0.00
%             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1            4|   64|   2671.25| 48.95|        54|         0|        30|         0|      1577|         0|        47|    914085|    289771|    947054|       0.00|       0.00
2            4|   64|   2652.68| 48.20|        54|         0|        30|         0|      1594|         0|        46|    882300|    293273|    914249|       0.00|       0.00
%             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0|         0|      +1.1|         0|      -2.1|      -3.5|      +1.2|      -3.5|          0|          0
1            1|  256|   1478.61| 29.34|        50|         0|        32|         0|      1709|         0|        64|   3502384|   1541681|   3545371|       0.00|       0.00
2            1|  256|   1481.38| 29.27|        50|         0|        32|         0|      1726|         0|        64|   3485623|   1526105|   3528343|       0.00|       0.00
%             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0|         0|      +1.0|         0|       0.0|      -0.5|      -1.0|      -0.5|          0|          0
1            2|  256|   3095.93| 42.31|        72|         0|        31|         0|      1612|         0|       123|   1521019|    466426|   1637722|       0.00|       0.00
2            2|  256|   3098.16| 42.03|        73|         0|        30|         0|      1594|         0|       123|   1519458|    472632|   1632462|       0.00|       0.00
%             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2|         0|      -1.1|         0|       0.0|      -0.1|      +1.3|      -0.3|          0|          0
1            4|  256|   6553.60| 38.44|       170|         0|        33|         0|      1761|         0|        38|   1343193|    237604|   1402385|       0.00|       0.00
2            4|  256|   6649.37| 38.34|       172|         0|        32|         0|      1726|         0|        32|   1343770|    236320|   1400875|       0.00|       0.00
%             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0|         0|      -2.0|         0|     -15.8|      +0.0|      -0.5|      -0.1|          0|          0
1            1|  512|   1818.40| 29.28|        61|         0|        31|         0|      1646|         0|        76|   3326887|   1491961|   3371189|       0.00|       0.00
2            1|  512|   1822.74| 29.45|        61|         0|        30|         0|      1577|         0|        76|   3309934|   1470715|   3352921|       0.00|       0.00
%             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2|         0|      -4.2|         0|       0.0|      -0.5|      -1.4|      -0.5|          0|          0
1            2|  512|   3931.72| 43.51|        89|         0|        33|         0|      1761|         0|       137|   1148331|    301194|   1270351|       0.00|       0.00
2            2|  512|   3926.16| 43.45|        90|         0|        32|         0|      1726|         0|       135|   1166280|    306110|   1288869|       0.00|       0.00
%             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0|         0|      -2.0|         0|      -1.5|      +1.6|      +1.6|      +1.5|          0|          0
1            4|  512|   8634.49| 40.69|       211|         0|        31|         0|      1612|         0|        53|   1869998|    353856|   1948593|       0.00|       0.00
2            4|  512|   9137.12| 40.77|       223|         0|        30|         0|      1594|         0|        48|   1879574|    362851|   1959044|       0.00|       0.00
%             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2|         0|      -1.1|         0|      -9.4|      +0.5|      +2.5|      +0.5|          0|          0
1            1| 1024|   2042.67| 29.50|        68|         0|        33|         0|      1778|         0|        84|   3087032|   1309160|   3130736|       0.00|       0.00
2            1| 1024|   2033.24| 29.51|        68|         0|        32|         0|      1726|         0|        83|   3082898|   1295936|   3125229|       0.00|       0.00
%             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0|         0|      -2.9|         0|      -1.2|      -0.1|      -1.0|      -0.2|          0|          0
1            2| 1024|   7703.26| 39.58|       194|         0|        32|         0|      1673|         0|        65|   2522172|    663302|   2684835|       0.00|       0.00
2            2| 1024|   7657.00| 39.27|       194|         0|        30|         0|      1594|         0|        62|   2607281|    698035|   2755198|       0.00|       0.00
%             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2|         0|      -4.7|         0|      -4.6|      +3.4|      +5.2|      +2.6|          0|          0
1            4| 1024|  11814.84| 39.38|       299|         0|        32|         0|      1700|         0|        28|   2510318|    576071|   2596631|       0.00|       0.00
2            4| 1024|  12871.76| 38.93|       330|         0|        32|         0|      1726|         0|        25|   2232437|    632785|   2316035|       0.00|       0.00
%             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0|         0|      +1.5|         0|     -10.7|     -11.1|      +9.8|     -10.8|          0|          0
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    706.98| 26.25|        26|         0|        31|         0|      1656|         0|        80|   2630204|   4185330|   2692397|       0.00|       0.00
2            1|   64|    767.45| 27.78|        27|         0|        30|         0|      1594|         0|        85|   2626694|   4317273|   2688799|       0.00|       0.00
%             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2|         0|      -3.7|         0|      +6.2|      -0.1|      +3.2|      -0.1|          0|          0
1            2|   64|   2644.57| 42.60|        61|         0|        32|         0|      1700|         0|     34863|     49057|     16791|    204805|       0.00|       0.00
2            2|   64|   2721.19| 42.69|        63|         0|        32|         0|      1709|         0|     42365|     52739|     20094|    215895|       0.00|       0.00
%             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0|         0|      +0.5|         0|     +21.5|      +7.5|     +19.7|      +5.4|          0|          0
1            4|   64|   2756.40| 42.69|        64|         0|        32|         0|      1708|         0|     52043|     47976|     12694|    220732|       0.00|       0.00
2            4|   64|   2709.19| 42.79|        62|         0|        30|         0|      1594|         0|     50144|     47275|     12743|    218074|       0.00|       0.00
%             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2|         0|      -6.7|         0|      -3.6|      -1.5|      +0.4|      -1.2|          0|          0
1            1|  256|   3785.63| 32.34|       116|         0|        33|         0|      1734|         0|     36642|     58940|      1187|    157652|       0.00|       0.00
2            1|  256|   3593.17| 32.90|       108|         0|        32|         0|      1726|         0|     33179|     54980|      1130|    150168|       0.00|       0.00
%             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0|         0|      -0.5|         0|      -9.5|      -6.7|      -4.8|      -4.7|          0|          0
1            2|  256|   8316.26| 40.45|       205|         0|        32|         0|      1673|         0|     28302|    114835|      1655|    251137|       0.00|       0.00
2            2|  256|   8390.41| 41.17|       203|         0|        30|         0|      1594|         0|     31222|    115372|      1696|    257151|       0.00|       0.00
%             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2|         0|      -4.7|         0|     +10.3|      +0.5|      +2.5|      +2.4|          0|          0
1            4|  256|   9672.49| 42.40|       227|         0|        33|         0|      1761|         0|     30045|    132027|      3059|    283491|       0.00|       0.00
2            4|  256|   9741.71| 42.57|       228|         0|        32|         0|      1726|         6|     28875|    132125|      1999|    283036|       0.00|       0.00
%             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0|         0|      -2.0|         0|      -3.9|      +0.1|     -34.7|      -0.2|          0|          0
1            1|  512|   6052.41| 32.23|       187|         0|        31|         0|      1612|         0|     44681|     86084|      2592|    187465|       0.00|       0.00
2            1|  512|   6267.37| 32.51|       192|         0|        30|         0|      1577|         0|     45278|     88449|      2708|    191374|       0.00|       0.00
%             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2|         0|      -2.2|         0|      +1.3|      +2.7|      +4.5|      +2.1|          0|          0
1            2|  512|  12714.52| 38.27|       331|         0|        33|         0|      1744|         0|     24843|    169170|      2381|    292210|       0.00|       0.00
2            2|  512|  11341.65| 37.30|       303|         0|        32|         0|      1726|        14|     27393|    152465|      2980|    270740|       0.00|       0.00
%             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0|         0|      -1.0|         0|     +10.3|      -9.9|     +25.2|      -7.3|          0|          0
1            4|  512|  16127.03| 40.80|       394|         0|        31|         0|      1612|         0|     15063|    210628|      2234|    344124|       0.00|       0.00
2            4|  512|  14709.72| 39.94|       367|         0|        31|         0|      1612|         0|     17823|    193670|      4457|    324744|       0.00|       0.00
%             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0|         0|       0.0|         0|     +18.3|      -8.1|     +99.5|      -5.6|          0|          0
1            1| 1024|   6436.88| 30.31|       212|         0|        33|         0|      1778|         0|     47820|     83940|      2007|    172847|       0.00|       0.00
2            1| 1024|   6495.48| 30.51|       212|         0|        32|         0|      1726|         0|     47771|     85436|      2038|    175027|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0|         0|      -2.9|         0|      -0.1|      +1.8|      +1.5|      +1.3|          0|          0
1            2| 1024|  12245.34| 33.44|       365|         0|        31|         0|      1629|         0|     26499|    162142|      2909|    255717|       0.00|       0.00
2            2| 1024|  12913.57| 33.93|       380|         0|        31|         0|      1612|         0|     24196|    171386|      3671|    266270|       0.00|       0.00
%             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0|         0|      -1.0|         0|      -8.7|      +5.7|     +26.2|      +4.1|          0|          0
1            4| 1024|  16428.48| 34.81|       471|         0|        32|         0|      1709|         0|     12496|    215031|      6557|    318039|       0.00|       0.00
2            4| 1024|  17294.33| 35.05|       492|         0|        32|         0|      1709|         0|      9444|    224694|      4095|    328098|       0.00|       0.00
%             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0|         0|       0.0|         0|     -24.4|      +4.5|     -37.5|      +3.2|          0|          0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  14626.43| 14.93|       980|         0|        32|         0|      1681|         0|        13|    877664|    877731|    902624|       0.00|       0.00
2           50|   64|  14639.95| 13.10|      1123|         0|        32|         0|      1681|         0|        13|    878475|    878538|    897278|       0.00|       0.00
%             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0|         0|       0.0|         0|       0.0|      +0.1|      +0.1|      -0.6|          0|          0
1          100|   64|  14697.50| 14.31|      1029|         0|        34|         0|      1830|         0|        13|    881930|    881991|    903837|       0.00|       0.00
2          100|   64|  14692.35| 13.08|      1128|         0|        34|         0|      1830|         0|        13|    881619|    881679|    898952|       0.00|       0.00
%             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0|         0|       0.0|         0|       0.0|      -0.0|      -0.0|      -0.5|          0|          0
1          250|   64|  14683.40| 14.34|      1027|         0|        32|         0|      1698|         0|        13|    881083|    881147|    903071|       0.00|       0.00
2          250|   64|  14677.06| 12.85|      1144|         0|        32|         0|      1698|         0|        13|    880702|    880762|    898190|       0.00|       0.00
%             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0|         0|       0.0|         0|       0.0|      -0.0|      -0.0|      -0.5|          0|          0
1          500|   64|  14698.45| 14.39|      1025|         0|        34|         0|      1830|         0|        13|    881985|    882060|    905537|       0.00|       0.00
2          500|   64|  14626.17| 13.54|      1083|         0|        34|         0|      1830|         0|        13|    877648|    877726|    896427|       0.00|       0.00
%             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0|         0|       0.0|         0|       0.0|      -0.5|      -0.5|      -1.0|          0|          0
1           50|  256|  14386.52| 14.14|      1018|         0|        33|         0|      1716|         0|        13|    863266|    863336|    887195|       0.00|       0.00
2           50|  256|  14359.26| 14.22|      1016|         0|        32|         0|      1681|         0|        13|    861630|    861698|    882444|       0.00|       0.00
%             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.2|      -0.2|      -0.5|          0|          0
1          100|  256|  14364.89| 14.25|      1008|         0|        34|         0|      1813|         0|        13|    861969|    862033|    886679|       0.00|       0.00
2          100|  256|  14393.82| 14.53|       999|         0|        34|         0|      1813|         0|        13|    863704|    863765|    884487|       0.00|       0.00
%             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0|         0|       0.0|         0|       0.0|      +0.2|      +0.2|      -0.2|          0|          0
1          250|  256|  14387.95| 14.05|      1024|         0|        32|         0|      1681|         0|        13|    863352|    863429|    887505|       0.00|       0.00
2          250|  256|  14344.74| 14.62|       987|         0|        32|         0|      1698|         0|        13|    860760|    860843|    882154|       0.00|       0.00
%             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0|         0|      +1.0|         0|       0.0|      -0.3|      -0.3|      -0.6|          0|          0
1          500|  256|  14365.89| 14.19|      1013|         0|        33|         0|      1769|         0|        13|    862029|    862099|    885852|       0.00|       0.00
2          500|  256|  14329.59| 14.62|       990|         0|        34|         0|      1830|         0|        13|    859852|    861233|    880082|       0.00|       0.00
%             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0|         0|      +3.4|         0|       0.0|      -0.3|      -0.1|      -0.7|          0|          0
1           50|  512|  14273.07| 14.12|      1010|         0|        33|         0|      1742|         0|        13|    856460|    856524|    880356|       0.00|       0.00
2           50|  512|  14222.08| 14.72|       966|         0|        32|         0|      1698|         0|        12|    853398|    853468|    875621|       0.00|       0.00
%             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0|         0|      -2.5|         0|      -7.7|      -0.4|      -0.4|      -0.5|          0|          0
1          100|  512|  14236.34| 14.51|       982|         0|        34|         0|      1804|         0|        13|    854258|    854335|    876662|       0.00|       0.00
2          100|  512|  14147.89| 14.85|       953|         0|        34|         0|      1813|         0|        13|    848947|    849031|    871773|       0.00|       0.00
%             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0|         0|      +0.5|         0|       0.0|      -0.6|      -0.6|      -0.6|          0|          0
1          250|  512|  14260.79| 14.53|       983|         0|        33|         0|      1760|         0|        13|    855721|    855791|    879099|       0.00|       0.00
2          250|  512|  14190.68| 14.87|       954|         0|        32|         0|      1698|         0|        13|    851517|    851580|    874119|       0.00|       0.00
%             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0|         0|      -3.5|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1          500|  512|  14273.71| 14.18|      1008|         0|        33|         0|      1769|         0|        13|    856499|    856562|    879984|       0.00|       0.00
2          500|  512|  14213.87| 14.72|       967|         0|        34|         0|      1830|         0|        13|    852907|    852976|    875328|       0.00|       0.00
%             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0|         0|      +3.4|         0|       0.0|      -0.4|      -0.4|      -0.5|          0|          0
1           50| 1024|  14049.77| 13.89|      1024|         0|        32|         0|      1708|         0|        12|    843061|    843134|    863697|       0.00|       0.00
2           50| 1024|  13980.25| 14.38|       980|         0|        32|         0|      1698|         0|        12|    838890|    838973|    858350|       0.00|       0.00
%             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0|         0|      -0.6|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1          100| 1024|  14025.66| 13.32|      1066|         0|        33|         0|      1769|         0|        13|    841616|    841685|    864460|       0.00|       0.00
2          100| 1024|  13993.30| 14.82|       946|         0|        34|         0|      1830|         0|        12|    839676|    839735|    859288|       0.00|       0.00
%             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0|         0|      +3.4|         0|      -7.7|      -0.2|      -0.2|      -0.6|          0|          0
1          250| 1024|  14060.15| 13.90|      1023|         0|        35|         0|      1848|         0|        12|    843688|    843752|    864475|       0.00|       0.00
2          250| 1024|  13982.47| 15.34|       913|         0|        34|         0|      1830|         0|        12|    839026|    839089|    859914|       0.00|       0.00
%             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9|         0|      -1.0|         0|       0.0|      -0.6|      -0.6|      -0.5|          0|          0
1          500| 1024|  14051.14| 13.80|      1024|         0|        33|         0|      1716|         0|        13|    843144|    843212|    862981|       0.00|       0.00
2          500| 1024|  14013.95| 15.27|       918|         0|        32|         0|      1681|         0|        13|    840915|    840983|    862635|       0.00|       0.00
%             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.3|      -0.3|      -0.0|          0|          0


[-- Attachment #3: Type: text/plain, Size: 183 bytes --]

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

[-- Attachment #1: Type: text/plain, Size: 3122 bytes --]

On 19/12/11 10:19, Amos Kong wrote:
> On 12/12/11 13:12, Rusty Russell wrote:
>> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com> wrote:
>>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>>
>>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>>> parallel, from an external box to the guest. TCP_STREAM results were
>>>>> noisy. This could be due to buffering done by TCP, where packet size
>>>>> varies even as message size is constant.
>>>>>
>>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>>> regressed consistently, between 7 and 35%
>>>>>
>>>>> The "fix" applied was simply this:
>>>>
>>>> What machine& processor was this ?
>>>
>>> pined guest memory to numa node 1
>>
>> Please try this patch. How much does the branch cost us?
>>
>> (Compiles, untested).
>>
>> Thanks,
>> Rusty.
>>
>> From: Rusty Russell<rusty@rustcorp.com.au>
>> Subject: virtio: harsher barriers for virtio-mmio.
>>
>> We were cheating with our barriers; using the smp ones rather than the
>> real device ones. That was fine, until virtio-mmio came along, which
>> could be talking to a real device (a non-SMP CPU).
>>
>> Unfortunately, just putting back the real barriers (reverting
>> d57ed95d) causes a performance regression on virtio-pci. In
>> particular, Amos reports netbench's TCP_RR over virtio_net CPU
>> utilization increased up to 35% while throughput went down by up to
>> 14%.
>>
>> By comparison, this branch costs us???
>>
>> Reference: https://lkml.org/lkml/2011/12/11/22
>>
>> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
>> ---
>> drivers/lguest/lguest_device.c | 10 ++++++----
>> drivers/s390/kvm/kvm_virtio.c | 2 +-
>> drivers/virtio/virtio_mmio.c | 7 ++++---
>> drivers/virtio/virtio_pci.c | 4 ++--
>> drivers/virtio/virtio_ring.c | 34 +++++++++++++++++++++-------------
>> include/linux/virtio_ring.h | 1 +
>> tools/virtio/linux/virtio.h | 1 +
>> tools/virtio/virtio_test.c | 3 ++-
>> 8 files changed, 38 insertions(+), 24 deletions(-)
>
> Hi all,
>
> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more
> precision.
>
> Thanks, Amos
>
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
>
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt
>
> ======
> TCP_STREAM
> sessions| size|throughput| cpu| normalize| #tx-pkts| #rx-pkts| #tx-byts|
> #rx-byts| #re-trans| #tx-intr| #rx-intr| #io_exit| #irq_inj|#tpkt/#exit|
> #rpkt/#irq
> 1 1| 64| 1073.54| 10.50| 102| 0| 31| 0| 1612| 0| 16| 487641| 489753|
> 504764| 0.00| 0.00
> 2 1| 64| 1079.44| 10.29| 104| 0| 30| 0| 1594| 0| 17| 487156| 488828|
> 504411| 0.00| 0.00
> % | 0.0| +0.5| -2.0| +2.0| 0| -3.2| 0| -1.1| 0| +6.2| -0.1| -0.2| -0.1|

The format is broken in webpage, attached the result file.
it's also available here: http://amosk.info/download/rusty-fix-perf.txt

[-- Attachment #2: rusty-fix-perf.txt --]
[-- Type: text/plain, Size: 42872 bytes --]

--------- compare results -----------
Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.exhost_guest.txt
2 - avg-fixed.netperf.exhost_guest.txt

======
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1073.54| 10.50|       102|         0|        31|         0|      1612|         0|        16|    487641|    489753|    504764|       0.00|       0.00
2            1|   64|   1079.44| 10.29|       104|         0|        30|         0|      1594|         0|        17|    487156|    488828|    504411|       0.00|       0.00
%             |  0.0|      +0.5|  -2.0|      +2.0|         0|      -3.2|         0|      -1.1|         0|      +6.2|      -0.1|      -0.2|      -0.1|          0|          0
1            2|   64|   2141.12| 15.72|       136|         0|        33|         0|      1744|         0|        34|    873777|    972303|    928926|       0.00|       0.00
2            2|   64|   2140.88| 15.64|       137|         0|        33|         0|      1744|         0|        34|    926588|    942841|    974095|       0.00|       0.00
%             |  0.0|      -0.0|  -0.5|      +0.7|         0|       0.0|         0|       0.0|         0|       0.0|      +6.0|      -3.0|      +4.9|          0|          0
1            4|   64|   4076.80| 19.82|       205|         0|        30|         0|      1577|         0|        67|   1422282|   1166425|   1539219|       0.00|       0.00
2            4|   64|   4094.32| 20.70|       197|         0|        31|         0|      1612|         0|        68|   1704330|   1314077|   1833394|       0.00|       0.00
%             |  0.0|      +0.4|  +4.4|      -3.9|         0|      +3.3|         0|      +2.2|         0|      +1.5|     +19.8|     +12.7|     +19.1|          0|          0
1            1|  256|   2867.48| 13.44|       213|         0|        32|         0|      1726|         0|        14|    666430|    694922|    690730|       0.00|       0.00
2            1|  256|   2874.20| 12.71|       226|         0|        32|         0|      1709|         0|        14|    697960|    740407|    721807|       0.00|       0.00
%             |  0.0|      +0.2|  -5.4|      +6.1|         0|       0.0|         0|      -1.0|         0|       0.0|      +4.7|      +6.5|      +4.5|          0|          0
1            2|  256|   5642.82| 17.61|       320|         0|        30|         0|      1594|         0|        30|   1226861|   1236081|   1268562|       0.00|       0.00
2            2|  256|   5661.06| 17.41|       326|         0|        30|         0|      1594|         0|        29|   1175696|   1143490|   1221528|       0.00|       0.00
%             |  0.0|      +0.3|  -1.1|      +1.9|         0|       0.0|         0|       0.0|         0|      -3.3|      -4.2|      -7.5|      -3.7|          0|          0
1            4|  256|   9404.27| 23.55|       399|         0|        33|         0|      1744|         0|        37|   1692245|    659975|   1765103|       0.00|       0.00
2            4|  256|   8761.11| 23.18|       376|         0|        32|         0|      1726|         0|        36|   1699382|    418992|   1870804|       0.00|       0.00
%             |  0.0|      -6.8|  -1.6|      -5.8|         0|      -3.0|         0|      -1.0|         0|      -2.7|      +0.4|     -36.5|      +6.0|          0|          0
1            1|  512|   3803.66| 14.20|       267|         0|        30|         0|      1594|         0|        14|    693992|    750078|    721107|       0.00|       0.00
2            1|  512|   3838.02| 15.47|       248|         0|        31|         0|      1612|         0|        15|    811709|    773505|    838788|       0.00|       0.00
%             |  0.0|      +0.9|  +8.9|      -7.1|         0|      +3.3|         0|      +1.1|         0|      +7.1|     +17.0|      +3.1|     +16.3|          0|          0
1            2|  512|   8606.11| 19.34|       444|         0|        32|         0|      1709|         0|        29|   1264624|    647652|   1309740|       0.00|       0.00
2            2|  512|   8127.80| 18.93|       428|         0|        32|         0|      1726|         0|        28|   1216606|   1179269|   1266260|       0.00|       0.00
%             |  0.0|      -5.6|  -2.1|      -3.6|         0|       0.0|         0|      +1.0|         0|      -3.4|      -3.8|     +82.1|      -3.3|          0|          0
1            4|  512|   9409.41| 22.88|       413|         0|        30|         0|      1577|         0|        35|   1592587|   1072862|   1746787|       0.00|       0.00
2            4|  512|   9217.34| 23.05|       400|         0|        30|         0|      1594|         0|        34|   1596515|    513742|   1831538|       0.00|       0.00
%             |  0.0|      -2.0|  +0.7|      -3.1|         0|       0.0|         0|      +1.1|         0|      -2.9|      +0.2|     -52.1|      +4.9|          0|          0
1            1| 1024|   4389.75| 14.62|       303|         0|        32|         0|      1726|         0|        14|    663417|    611555|    689028|       0.00|       0.00
2            1| 1024|   4390.44| 13.66|       321|         0|        32|         0|      1726|         0|        13|    609836|    556090|    634511|       0.00|       0.00
%             |  0.0|      +0.0|  -6.6|      +5.9|         0|       0.0|         0|       0.0|         0|      -7.1|      -8.1|      -9.1|      -7.9|          0|          0
1            2| 1024|   9046.18| 19.63|       460|         0|        30|         0|      1594|         0|        28|   1206853|   1054653|   1256203|       0.00|       0.00
2            2| 1024|   9027.54| 20.15|       447|         0|        30|         0|      1577|         0|        27|   1179555|    884408|   1226843|       0.00|       0.00
%             |  0.0|      -0.2|  +2.6|      -2.8|         0|       0.0|         0|      -1.1|         0|      -3.6|      -2.3|     -16.1|      -2.3|          0|          0
1            4| 1024|   9410.41| 22.01|       427|         0|        32|         0|      1726|         0|        36|   1734433|    887190|   1852486|       0.00|       0.00
2            4| 1024|   9409.11| 23.18|       405|         0|        32|         0|      1726|         0|        34|   1695359|    625268|   1803599|       0.00|       0.00
%             |  0.0|      -0.0|  +5.3|      -5.2|         0|       0.0|         0|       0.0|         0|      -5.6|      -2.3|     -29.5|      -2.6|          0|          0
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|   1056.13| 17.51|        59|         0|        30|         0|      1594|         3|        86|    476098|   4780186|    538158|       0.00|       0.00
2            1|   64|   1065.51| 17.64|        60|         0|        30|         0|      1594|         3|        87|    476836|   4823602|    538860|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|      +1.7|         0|       0.0|         0|       0.0|       0.0|      +1.2|      +0.2|      +0.9|      +0.1|          0|          0
1            2|   64|   2472.80| 32.18|        76|         0|        32|         0|      1709|         2|       199|    479119|   2089931|    600119|       0.00|       0.00
2            2|   64|   2468.76| 32.28|        76|         0|        33|         0|      1744|         1|       199|    479372|   2179342|    600328|       0.00|       0.00
%             |  0.0|      -0.2|  +0.3|       0.0|         0|      +3.1|         0|      +2.0|     -50.0|       0.0|      +0.1|      +4.3|      +0.0|          0|          0
1            4|   64|   2453.22| 32.20|        75|         0|        31|         0|      1612|         2|       198|    479223|   2175801|    600081|       0.00|       0.00
2            4|   64|   2465.68| 32.30|        76|         0|        30|         0|      1594|         3|       200|    479404|   2108293|    600252|       0.00|       0.00
%             |  0.0|      +0.5|  +0.3|      +1.3|         0|      -3.2|         0|      -1.1|     +50.0|      +1.0|      +0.0|      -3.1|      +0.0|          0|          0
1            1|  256|   3608.37| 24.59|       146|         0|        32|         0|      1726|         2|       277|     88346|    108310|    150637|       0.00|       0.00
2            1|  256|   3611.12| 24.95|       144|         0|        32|         0|      1709|         0|       281|     83319|     64354|    145511|       0.00|       0.00
%             |  0.0|      +0.1|  +1.5|      -1.4|         0|       0.0|         0|      -1.0|    -100.0|      +1.4|      -5.7|     -40.6|      -3.4|          0|          0
1            2|  256|   7373.32| 26.83|       274|         0|        31|         0|      1612|         0|      4185|    278932|    264031|    377267|       0.00|       0.00
2            2|  256|   7460.03| 26.93|       276|         0|        30|         0|      1594|         1|      5913|    270840|    246165|    371999|       0.00|       0.00
%             |  0.0|      +1.2|  +0.4|      +0.7|         0|      -3.2|         0|      -1.1|         0|     +41.3|      -2.9|      -6.8|      -1.4|          0|          0
1            4|  256|   9322.66| 29.73|       313|         0|        32|         0|      1709|       848|      3041|    363114|    329308|    482586|       0.00|       0.00
2            4|  256|   9365.52| 29.59|       316|         0|        33|         0|      1744|       414|      2884|    371485|    339191|    491366|       0.00|       0.00
%             |  0.0|      +0.5|  -0.5|      +1.0|         0|      +3.1|         0|      +2.0|     -51.2|      -5.2|      +2.3|      +3.0|      +1.8|          0|          0
1            1|  512|   9360.25| 17.20|       543|         0|        30|         0|      1577|         0|       387|    438621|    408062|    499989|       0.00|       0.00
2            1|  512|   8899.37| 18.17|       495|         0|        30|         0|      1594|         0|      5049|    393341|    371545|    459397|       0.00|       0.00
%             |  0.0|      -4.9|  +5.6|      -8.8|         0|       0.0|         0|      +1.1|         0|   +1204.7|     -10.3|      -8.9|      -8.1|          0|          0
1            2|  512|   8713.11| 20.77|       420|         0|        32|         0|      1726|         0|     11227|    296157|    246495|    371632|       0.00|       0.00
2            2|  512|   8686.31| 20.72|       421|         0|        32|         0|      1709|         0|     11002|    296608|    246597|    371797|       0.00|       0.00
%             |  0.0|      -0.3|  -0.2|      +0.2|         0|       0.0|         0|      -1.0|         0|      -2.0|      +0.2|      +0.0|      +0.0|          0|          0
1            4|  512|   8780.40| 22.72|       386|         0|        30|         0|      1594|         0|     10337|    303619|    249822|    385457|       0.00|       0.00
2            4|  512|   8858.51| 22.66|       390|         0|        30|         0|      1577|         0|      9968|    301847|    246543|    382121|       0.00|       0.00
%             |  0.0|      +0.9|  -0.3|      +1.0|         0|       0.0|         0|      -1.1|         0|      -3.6|      -0.6|      -1.3|      -0.9|          0|          0
1            1| 1024|   9387.09| 16.43|       571|         0|        32|         0|      1726|         0|        84|    452351|    429483|    504022|       0.00|       0.00
2            1| 1024|   9388.90| 16.51|       568|         0|        32|         0|      1726|         2|        76|    446994|    422489|    499025|       0.00|       0.00
%             |  0.0|      +0.0|  +0.5|      -0.5|         0|       0.0|         0|       0.0|         0|      -9.5|      -1.2|      -1.6|      -1.0|          0|          0
1            2| 1024|   9361.43| 17.26|       542|         0|        30|         0|      1594|         5|       647|    388626|    358303|    442125|       0.00|       0.00
2            2| 1024|   9364.78| 17.25|       542|         0|        31|         0|      1612|        35|       554|    389326|    360169|    442343|       0.00|       0.00
%             |  0.0|      +0.0|  -0.1|       0.0|         0|      +3.3|         0|      +1.1|    +600.0|     -14.4|      +0.2|      +0.5|      +0.0|          0|          0
1            4| 1024|   9381.57| 18.35|       510|         0|        32|         0|      1726|      1189|       648|    405900|    376269|    459845|       0.00|       0.00
2            4| 1024|   9393.48| 18.27|       513|         0|        32|         0|      1726|      1754|       610|    402132|    373768|    455629|       0.00|       0.00
%             |  0.0|      +0.1|  -0.4|      +0.6|         0|       0.0|         0|       0.0|     +47.5|      -5.9|      -0.9|      -0.7|      -0.9|          0|          0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  10903.70|  8.03|      1382|         0|        32|         0|      1698|         0|         9|    654298|    654362|    665532|       0.00|       0.00
2           50|   64|  10880.50|  7.40|      1488|         0|        32|         0|      1698|         0|        10|    652907|    652973|    667042|       0.00|       0.00
%             |  0.0|      -0.2|  -7.8|      +7.7|         0|       0.0|         0|       0.0|         0|     +11.1|      -0.2|      -0.2|      +0.2|          0|          0
1          100|   64|  10953.00|  8.40|      1332|         0|        34|         0|      1830|         0|        10|    657257|    657325|    667088|       0.00|       0.00
2          100|   64|  10882.78|  7.67|      1439|         0|        34|         0|      1830|         0|         9|    653044|    653110|    668883|       0.00|       0.00
%             |  0.0|      -0.6|  -8.7|      +8.0|         0|       0.0|         0|       0.0|         0|     -10.0|      -0.6|      -0.6|      +0.3|          0|          0
1          250|   64|  10902.66|  8.57|      1300|         0|        33|         0|      1716|         0|        10|    654235|    654308|    665518|       0.00|       0.00
2          250|   64|  10866.79|  7.58|      1449|         0|        32|         0|      1664|         0|        10|    652083|    652155|    666869|       0.00|       0.00
%             |  0.0|      -0.3| -11.6|     +11.5|         0|      -3.0|         0|      -3.0|         0|       0.0|      -0.3|      -0.3|      +0.2|          0|          0
1          500|   64|  10924.34|  7.25|      1509|         0|        34|         0|      1830|         0|        10|    655536|    655619|    665776|       0.00|       0.00
2          500|   64|  10927.52|  6.50|      1684|         0|        34|         0|      1813|         0|        10|    655726|    655803|    668527|       0.00|       0.00
%             |  0.0|      +0.0| -10.3|     +11.6|         0|       0.0|         0|      -0.9|         0|       0.0|      +0.0|      +0.0|      +0.4|          0|          0
1           50|  256|  10407.35|  9.38|      1112|         0|        32|         0|      1681|         0|         9|    624516|    624581|    639159|       0.00|       0.00
2           50|  256|  10422.60|  9.45|      1108|         0|        33|         0|      1716|         0|         9|    625432|    625498|    641343|       0.00|       0.00
%             |  0.0|      +0.1|  +0.7|      -0.4|         0|      +3.1|         0|      +2.1|         0|       0.0|      +0.1|      +0.1|      +0.3|          0|          0
1          100|  256|  10404.30|  8.99|      1159|         0|        34|         0|      1830|         0|         9|    624335|    624400|    637357|       0.00|       0.00
2          100|  256|  10442.84|  9.57|      1093|         0|        35|         0|      1848|         0|         9|    626650|    626714|    639017|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -5.7|         0|      +2.9|         0|      +1.0|         0|       0.0|      +0.4|      +0.4|      +0.3|          0|          0
1          250|  256|  10427.71|  9.10|      1157|         0|        32|         0|      1698|         0|         9|    625738|    625822|    641221|       0.00|       0.00
2          250|  256|  10464.71|  9.69|      1085|         0|        32|         0|      1698|         0|         9|    627960|    628040|    640680|       0.00|       0.00
%             |  0.0|      +0.4|  +6.5|      -6.2|         0|       0.0|         0|       0.0|         0|       0.0|      +0.4|      +0.4|      -0.1|          0|          0
1          500|  256|  10442.18|  8.65|      1217|         0|        34|         0|      1830|         0|        10|    626608|    626675|    640081|       0.00|       0.00
2          500|  256|  10463.74|  9.63|      1089|         0|        34|         0|      1813|         0|         9|    627904|    627967|    643152|       0.00|       0.00
%             |  0.0|      +0.2| +11.3|     -10.5|         0|       0.0|         0|      -0.9|         0|     -10.0|      +0.2|      +0.2|      +0.5|          0|          0
1           50|  512|   8505.70|  7.20|      1191|         0|        33|         0|      1716|         0|         7|    510418|    510486|    524904|       0.00|       0.00
2           50|  512|   8563.34|  7.24|      1185|         0|        32|         0|      1664|         0|         8|    513877|    513940|    526294|       0.00|       0.00
%             |  0.0|      +0.7|  +0.6|      -0.5|         0|      -3.0|         0|      -3.0|         0|     +14.3|      +0.7|      +0.7|      +0.3|          0|          0
1          100|  512|   8509.97|  7.00|      1226|         0|        34|         0|      1830|         0|         7|    510674|    510756|    525205|       0.00|       0.00
2          100|  512|   8555.22|  7.52|      1144|         0|        34|         0|      1830|         0|         7|    513389|    513471|    526609|       0.00|       0.00
%             |  0.0|      +0.5|  +7.4|      -6.7|         0|       0.0|         0|       0.0|         0|       0.0|      +0.5|      +0.5|      +0.3|          0|          0
1          250|  512|   8507.35|  7.21|      1183|         0|        32|         0|      1681|         0|         7|    510518|    510579|    524294|       0.00|       0.00
2          250|  512|   8578.13|  7.01|      1223|         0|        33|         0|      1716|         0|         8|    514765|    514829|    526481|       0.00|       0.00
%             |  0.0|      +0.8|  -2.8|      +3.4|         0|      +3.1|         0|      +2.1|         0|     +14.3|      +0.8|      +0.8|      +0.4|          0|          0
1          500|  512|   8521.68|  6.54|      1302|         0|        34|         0|      1813|         0|         8|    511375|    511443|    523357|       0.00|       0.00
2          500|  512|   8573.17|  6.94|      1235|         0|        34|         0|      1830|         0|         7|    514467|    514533|    526131|       0.00|       0.00
%             |  0.0|      +0.6|  +6.1|      -5.1|         0|       0.0|         0|      +0.9|         0|     -12.5|      +0.6|      +0.6|      +0.5|          0|          0
1           50| 1024|   8150.01|  7.69|      1060|         0|        32|         0|      1681|         0|         7|    489075|    489155|    503136|       0.00|       0.00
2           50| 1024|   8129.35|  7.65|      1062|         0|        34|         0|      1804|         0|         7|    487841|    487915|    502061|       0.00|       0.00
%             |  0.0|      -0.3|  -0.5|      +0.2|         0|      +6.2|         0|      +7.3|         0|       0.0|      -0.3|      -0.3|      -0.2|          0|          0
1          100| 1024|   8166.06|  7.53|      1083|         0|        34|         0|      1804|         0|         7|    490039|    490104|    502990|       0.00|       0.00
2          100| 1024|   8118.23|  7.77|      1044|         0|        33|         0|      1725|         0|         7|    487171|    487236|    501939|       0.00|       0.00
%             |  0.0|      -0.6|  +3.2|      -3.6|         0|      -2.9|         0|      -4.4|         0|       0.0|      -0.6|      -0.6|      -0.2|          0|          0
1          250| 1024|   8175.35|  7.58|      1079|         0|        35|         0|      1848|         0|         7|    490597|    490657|    503641|       0.00|       0.00
2          250| 1024|   8146.63|  7.59|      1072|         0|        34|         0|      1813|         0|         7|    488874|    488939|    502010|       0.00|       0.00
%             |  0.0|      -0.4|  +0.1|      -0.6|         0|      -2.9|         0|      -1.9|         0|       0.0|      -0.4|      -0.4|      -0.3|          0|          0
1          500| 1024|   8182.51|  7.61|      1075|         0|        32|         0|      1698|         0|         7|    491028|    491095|    504222|       0.00|       0.00
2          500| 1024|   8157.39|  7.58|      1075|         0|        33|         0|      1716|         0|         7|    489521|    489593|    502368|       0.00|       0.00
%             |  0.0|      -0.3|  -0.4|       0.0|         0|      +3.1|         0|      +1.1|         0|       0.0|      -0.3|      -0.3|      -0.4|          0|          0

Mon Dec 19 09:51:09 2011

1 - avg-old.netperf.host_guest.txt
2 - avg-fixed.netperf.host_guest.txt

======
TCP_STREAM
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    715.76| 28.44|        24|         0|        31|         0|      1629|         0|        36|   3206862|   1733540|   3245171|       0.00|       0.00
2            1|   64|    714.82| 28.44|        25|         0|        31|         0|      1612|         0|        36|   3204256|   1709508|   3242450|       0.00|       0.00
%             |  0.0|      -0.1|   0.0|      +4.2|         0|       0.0|         0|      -1.0|         0|       0.0|      -0.1|      -1.4|      -0.1|          0|          0
1            2|   64|   1449.31| 41.23|        35|         0|        33|         0|      1744|         0|        66|   2764635|   1667527|   2815591|       0.00|       0.00
2            2|   64|   1442.92| 41.32|        34|         0|        32|         0|      1709|         0|        66|   2751461|   1659498|   2799277|       0.00|       0.00
%             |  0.0|      -0.4|  +0.2|      -2.9|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1            4|   64|   2671.25| 48.95|        54|         0|        30|         0|      1577|         0|        47|    914085|    289771|    947054|       0.00|       0.00
2            4|   64|   2652.68| 48.20|        54|         0|        30|         0|      1594|         0|        46|    882300|    293273|    914249|       0.00|       0.00
%             |  0.0|      -0.7|  -1.5|       0.0|         0|       0.0|         0|      +1.1|         0|      -2.1|      -3.5|      +1.2|      -3.5|          0|          0
1            1|  256|   1478.61| 29.34|        50|         0|        32|         0|      1709|         0|        64|   3502384|   1541681|   3545371|       0.00|       0.00
2            1|  256|   1481.38| 29.27|        50|         0|        32|         0|      1726|         0|        64|   3485623|   1526105|   3528343|       0.00|       0.00
%             |  0.0|      +0.2|  -0.2|       0.0|         0|       0.0|         0|      +1.0|         0|       0.0|      -0.5|      -1.0|      -0.5|          0|          0
1            2|  256|   3095.93| 42.31|        72|         0|        31|         0|      1612|         0|       123|   1521019|    466426|   1637722|       0.00|       0.00
2            2|  256|   3098.16| 42.03|        73|         0|        30|         0|      1594|         0|       123|   1519458|    472632|   1632462|       0.00|       0.00
%             |  0.0|      +0.1|  -0.7|      +1.4|         0|      -3.2|         0|      -1.1|         0|       0.0|      -0.1|      +1.3|      -0.3|          0|          0
1            4|  256|   6553.60| 38.44|       170|         0|        33|         0|      1761|         0|        38|   1343193|    237604|   1402385|       0.00|       0.00
2            4|  256|   6649.37| 38.34|       172|         0|        32|         0|      1726|         0|        32|   1343770|    236320|   1400875|       0.00|       0.00
%             |  0.0|      +1.5|  -0.3|      +1.2|         0|      -3.0|         0|      -2.0|         0|     -15.8|      +0.0|      -0.5|      -0.1|          0|          0
1            1|  512|   1818.40| 29.28|        61|         0|        31|         0|      1646|         0|        76|   3326887|   1491961|   3371189|       0.00|       0.00
2            1|  512|   1822.74| 29.45|        61|         0|        30|         0|      1577|         0|        76|   3309934|   1470715|   3352921|       0.00|       0.00
%             |  0.0|      +0.2|  +0.6|       0.0|         0|      -3.2|         0|      -4.2|         0|       0.0|      -0.5|      -1.4|      -0.5|          0|          0
1            2|  512|   3931.72| 43.51|        89|         0|        33|         0|      1761|         0|       137|   1148331|    301194|   1270351|       0.00|       0.00
2            2|  512|   3926.16| 43.45|        90|         0|        32|         0|      1726|         0|       135|   1166280|    306110|   1288869|       0.00|       0.00
%             |  0.0|      -0.1|  -0.1|      +1.1|         0|      -3.0|         0|      -2.0|         0|      -1.5|      +1.6|      +1.6|      +1.5|          0|          0
1            4|  512|   8634.49| 40.69|       211|         0|        31|         0|      1612|         0|        53|   1869998|    353856|   1948593|       0.00|       0.00
2            4|  512|   9137.12| 40.77|       223|         0|        30|         0|      1594|         0|        48|   1879574|    362851|   1959044|       0.00|       0.00
%             |  0.0|      +5.8|  +0.2|      +5.7|         0|      -3.2|         0|      -1.1|         0|      -9.4|      +0.5|      +2.5|      +0.5|          0|          0
1            1| 1024|   2042.67| 29.50|        68|         0|        33|         0|      1778|         0|        84|   3087032|   1309160|   3130736|       0.00|       0.00
2            1| 1024|   2033.24| 29.51|        68|         0|        32|         0|      1726|         0|        83|   3082898|   1295936|   3125229|       0.00|       0.00
%             |  0.0|      -0.5|  +0.0|       0.0|         0|      -3.0|         0|      -2.9|         0|      -1.2|      -0.1|      -1.0|      -0.2|          0|          0
1            2| 1024|   7703.26| 39.58|       194|         0|        32|         0|      1673|         0|        65|   2522172|    663302|   2684835|       0.00|       0.00
2            2| 1024|   7657.00| 39.27|       194|         0|        30|         0|      1594|         0|        62|   2607281|    698035|   2755198|       0.00|       0.00
%             |  0.0|      -0.6|  -0.8|       0.0|         0|      -6.2|         0|      -4.7|         0|      -4.6|      +3.4|      +5.2|      +2.6|          0|          0
1            4| 1024|  11814.84| 39.38|       299|         0|        32|         0|      1700|         0|        28|   2510318|    576071|   2596631|       0.00|       0.00
2            4| 1024|  12871.76| 38.93|       330|         0|        32|         0|      1726|         0|        25|   2232437|    632785|   2316035|       0.00|       0.00
%             |  0.0|      +8.9|  -1.1|     +10.4|         0|       0.0|         0|      +1.5|         0|     -10.7|     -11.1|      +9.8|     -10.8|          0|          0
TCP_MAERTS
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1            1|   64|    706.98| 26.25|        26|         0|        31|         0|      1656|         0|        80|   2630204|   4185330|   2692397|       0.00|       0.00
2            1|   64|    767.45| 27.78|        27|         0|        30|         0|      1594|         0|        85|   2626694|   4317273|   2688799|       0.00|       0.00
%             |  0.0|      +8.6|  +5.8|      +3.8|         0|      -3.2|         0|      -3.7|         0|      +6.2|      -0.1|      +3.2|      -0.1|          0|          0
1            2|   64|   2644.57| 42.60|        61|         0|        32|         0|      1700|         0|     34863|     49057|     16791|    204805|       0.00|       0.00
2            2|   64|   2721.19| 42.69|        63|         0|        32|         0|      1709|         0|     42365|     52739|     20094|    215895|       0.00|       0.00
%             |  0.0|      +2.9|  +0.2|      +3.3|         0|       0.0|         0|      +0.5|         0|     +21.5|      +7.5|     +19.7|      +5.4|          0|          0
1            4|   64|   2756.40| 42.69|        64|         0|        32|         0|      1708|         0|     52043|     47976|     12694|    220732|       0.00|       0.00
2            4|   64|   2709.19| 42.79|        62|         0|        30|         0|      1594|         0|     50144|     47275|     12743|    218074|       0.00|       0.00
%             |  0.0|      -1.7|  +0.2|      -3.1|         0|      -6.2|         0|      -6.7|         0|      -3.6|      -1.5|      +0.4|      -1.2|          0|          0
1            1|  256|   3785.63| 32.34|       116|         0|        33|         0|      1734|         0|     36642|     58940|      1187|    157652|       0.00|       0.00
2            1|  256|   3593.17| 32.90|       108|         0|        32|         0|      1726|         0|     33179|     54980|      1130|    150168|       0.00|       0.00
%             |  0.0|      -5.1|  +1.7|      -6.9|         0|      -3.0|         0|      -0.5|         0|      -9.5|      -6.7|      -4.8|      -4.7|          0|          0
1            2|  256|   8316.26| 40.45|       205|         0|        32|         0|      1673|         0|     28302|    114835|      1655|    251137|       0.00|       0.00
2            2|  256|   8390.41| 41.17|       203|         0|        30|         0|      1594|         0|     31222|    115372|      1696|    257151|       0.00|       0.00
%             |  0.0|      +0.9|  +1.8|      -1.0|         0|      -6.2|         0|      -4.7|         0|     +10.3|      +0.5|      +2.5|      +2.4|          0|          0
1            4|  256|   9672.49| 42.40|       227|         0|        33|         0|      1761|         0|     30045|    132027|      3059|    283491|       0.00|       0.00
2            4|  256|   9741.71| 42.57|       228|         0|        32|         0|      1726|         6|     28875|    132125|      1999|    283036|       0.00|       0.00
%             |  0.0|      +0.7|  +0.4|      +0.4|         0|      -3.0|         0|      -2.0|         0|      -3.9|      +0.1|     -34.7|      -0.2|          0|          0
1            1|  512|   6052.41| 32.23|       187|         0|        31|         0|      1612|         0|     44681|     86084|      2592|    187465|       0.00|       0.00
2            1|  512|   6267.37| 32.51|       192|         0|        30|         0|      1577|         0|     45278|     88449|      2708|    191374|       0.00|       0.00
%             |  0.0|      +3.6|  +0.9|      +2.7|         0|      -3.2|         0|      -2.2|         0|      +1.3|      +2.7|      +4.5|      +2.1|          0|          0
1            2|  512|  12714.52| 38.27|       331|         0|        33|         0|      1744|         0|     24843|    169170|      2381|    292210|       0.00|       0.00
2            2|  512|  11341.65| 37.30|       303|         0|        32|         0|      1726|        14|     27393|    152465|      2980|    270740|       0.00|       0.00
%             |  0.0|     -10.8|  -2.5|      -8.5|         0|      -3.0|         0|      -1.0|         0|     +10.3|      -9.9|     +25.2|      -7.3|          0|          0
1            4|  512|  16127.03| 40.80|       394|         0|        31|         0|      1612|         0|     15063|    210628|      2234|    344124|       0.00|       0.00
2            4|  512|  14709.72| 39.94|       367|         0|        31|         0|      1612|         0|     17823|    193670|      4457|    324744|       0.00|       0.00
%             |  0.0|      -8.8|  -2.1|      -6.9|         0|       0.0|         0|       0.0|         0|     +18.3|      -8.1|     +99.5|      -5.6|          0|          0
1            1| 1024|   6436.88| 30.31|       212|         0|        33|         0|      1778|         0|     47820|     83940|      2007|    172847|       0.00|       0.00
2            1| 1024|   6495.48| 30.51|       212|         0|        32|         0|      1726|         0|     47771|     85436|      2038|    175027|       0.00|       0.00
%             |  0.0|      +0.9|  +0.7|       0.0|         0|      -3.0|         0|      -2.9|         0|      -0.1|      +1.8|      +1.5|      +1.3|          0|          0
1            2| 1024|  12245.34| 33.44|       365|         0|        31|         0|      1629|         0|     26499|    162142|      2909|    255717|       0.00|       0.00
2            2| 1024|  12913.57| 33.93|       380|         0|        31|         0|      1612|         0|     24196|    171386|      3671|    266270|       0.00|       0.00
%             |  0.0|      +5.5|  +1.5|      +4.1|         0|       0.0|         0|      -1.0|         0|      -8.7|      +5.7|     +26.2|      +4.1|          0|          0
1            4| 1024|  16428.48| 34.81|       471|         0|        32|         0|      1709|         0|     12496|    215031|      6557|    318039|       0.00|       0.00
2            4| 1024|  17294.33| 35.05|       492|         0|        32|         0|      1709|         0|      9444|    224694|      4095|    328098|       0.00|       0.00
%             |  0.0|      +5.3|  +0.7|      +4.5|         0|       0.0|         0|       0.0|         0|     -24.4|      +4.5|     -37.5|      +3.2|          0|          0
TCP_RR
      sessions| size|throughput|   cpu| normalize|  #tx-pkts|  #rx-pkts|  #tx-byts|  #rx-byts| #re-trans|  #tx-intr|  #rx-intr|  #io_exit|  #irq_inj|#tpkt/#exit| #rpkt/#irq
1           50|   64|  14626.43| 14.93|       980|         0|        32|         0|      1681|         0|        13|    877664|    877731|    902624|       0.00|       0.00
2           50|   64|  14639.95| 13.10|      1123|         0|        32|         0|      1681|         0|        13|    878475|    878538|    897278|       0.00|       0.00
%             |  0.0|      +0.1| -12.3|     +14.6|         0|       0.0|         0|       0.0|         0|       0.0|      +0.1|      +0.1|      -0.6|          0|          0
1          100|   64|  14697.50| 14.31|      1029|         0|        34|         0|      1830|         0|        13|    881930|    881991|    903837|       0.00|       0.00
2          100|   64|  14692.35| 13.08|      1128|         0|        34|         0|      1830|         0|        13|    881619|    881679|    898952|       0.00|       0.00
%             |  0.0|      -0.0|  -8.6|      +9.6|         0|       0.0|         0|       0.0|         0|       0.0|      -0.0|      -0.0|      -0.5|          0|          0
1          250|   64|  14683.40| 14.34|      1027|         0|        32|         0|      1698|         0|        13|    881083|    881147|    903071|       0.00|       0.00
2          250|   64|  14677.06| 12.85|      1144|         0|        32|         0|      1698|         0|        13|    880702|    880762|    898190|       0.00|       0.00
%             |  0.0|      -0.0| -10.4|     +11.4|         0|       0.0|         0|       0.0|         0|       0.0|      -0.0|      -0.0|      -0.5|          0|          0
1          500|   64|  14698.45| 14.39|      1025|         0|        34|         0|      1830|         0|        13|    881985|    882060|    905537|       0.00|       0.00
2          500|   64|  14626.17| 13.54|      1083|         0|        34|         0|      1830|         0|        13|    877648|    877726|    896427|       0.00|       0.00
%             |  0.0|      -0.5|  -5.9|      +5.7|         0|       0.0|         0|       0.0|         0|       0.0|      -0.5|      -0.5|      -1.0|          0|          0
1           50|  256|  14386.52| 14.14|      1018|         0|        33|         0|      1716|         0|        13|    863266|    863336|    887195|       0.00|       0.00
2           50|  256|  14359.26| 14.22|      1016|         0|        32|         0|      1681|         0|        13|    861630|    861698|    882444|       0.00|       0.00
%             |  0.0|      -0.2|  +0.6|      -0.2|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.2|      -0.2|      -0.5|          0|          0
1          100|  256|  14364.89| 14.25|      1008|         0|        34|         0|      1813|         0|        13|    861969|    862033|    886679|       0.00|       0.00
2          100|  256|  14393.82| 14.53|       999|         0|        34|         0|      1813|         0|        13|    863704|    863765|    884487|       0.00|       0.00
%             |  0.0|      +0.2|  +2.0|      -0.9|         0|       0.0|         0|       0.0|         0|       0.0|      +0.2|      +0.2|      -0.2|          0|          0
1          250|  256|  14387.95| 14.05|      1024|         0|        32|         0|      1681|         0|        13|    863352|    863429|    887505|       0.00|       0.00
2          250|  256|  14344.74| 14.62|       987|         0|        32|         0|      1698|         0|        13|    860760|    860843|    882154|       0.00|       0.00
%             |  0.0|      -0.3|  +4.1|      -3.6|         0|       0.0|         0|      +1.0|         0|       0.0|      -0.3|      -0.3|      -0.6|          0|          0
1          500|  256|  14365.89| 14.19|      1013|         0|        33|         0|      1769|         0|        13|    862029|    862099|    885852|       0.00|       0.00
2          500|  256|  14329.59| 14.62|       990|         0|        34|         0|      1830|         0|        13|    859852|    861233|    880082|       0.00|       0.00
%             |  0.0|      -0.3|  +3.0|      -2.3|         0|      +3.0|         0|      +3.4|         0|       0.0|      -0.3|      -0.1|      -0.7|          0|          0
1           50|  512|  14273.07| 14.12|      1010|         0|        33|         0|      1742|         0|        13|    856460|    856524|    880356|       0.00|       0.00
2           50|  512|  14222.08| 14.72|       966|         0|        32|         0|      1698|         0|        12|    853398|    853468|    875621|       0.00|       0.00
%             |  0.0|      -0.4|  +4.2|      -4.4|         0|      -3.0|         0|      -2.5|         0|      -7.7|      -0.4|      -0.4|      -0.5|          0|          0
1          100|  512|  14236.34| 14.51|       982|         0|        34|         0|      1804|         0|        13|    854258|    854335|    876662|       0.00|       0.00
2          100|  512|  14147.89| 14.85|       953|         0|        34|         0|      1813|         0|        13|    848947|    849031|    871773|       0.00|       0.00
%             |  0.0|      -0.6|  +2.3|      -3.0|         0|       0.0|         0|      +0.5|         0|       0.0|      -0.6|      -0.6|      -0.6|          0|          0
1          250|  512|  14260.79| 14.53|       983|         0|        33|         0|      1760|         0|        13|    855721|    855791|    879099|       0.00|       0.00
2          250|  512|  14190.68| 14.87|       954|         0|        32|         0|      1698|         0|        13|    851517|    851580|    874119|       0.00|       0.00
%             |  0.0|      -0.5|  +2.3|      -3.0|         0|      -3.0|         0|      -3.5|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1          500|  512|  14273.71| 14.18|      1008|         0|        33|         0|      1769|         0|        13|    856499|    856562|    879984|       0.00|       0.00
2          500|  512|  14213.87| 14.72|       967|         0|        34|         0|      1830|         0|        13|    852907|    852976|    875328|       0.00|       0.00
%             |  0.0|      -0.4|  +3.8|      -4.1|         0|      +3.0|         0|      +3.4|         0|       0.0|      -0.4|      -0.4|      -0.5|          0|          0
1           50| 1024|  14049.77| 13.89|      1024|         0|        32|         0|      1708|         0|        12|    843061|    843134|    863697|       0.00|       0.00
2           50| 1024|  13980.25| 14.38|       980|         0|        32|         0|      1698|         0|        12|    838890|    838973|    858350|       0.00|       0.00
%             |  0.0|      -0.5|  +3.5|      -4.3|         0|       0.0|         0|      -0.6|         0|       0.0|      -0.5|      -0.5|      -0.6|          0|          0
1          100| 1024|  14025.66| 13.32|      1066|         0|        33|         0|      1769|         0|        13|    841616|    841685|    864460|       0.00|       0.00
2          100| 1024|  13993.30| 14.82|       946|         0|        34|         0|      1830|         0|        12|    839676|    839735|    859288|       0.00|       0.00
%             |  0.0|      -0.2| +11.3|     -11.3|         0|      +3.0|         0|      +3.4|         0|      -7.7|      -0.2|      -0.2|      -0.6|          0|          0
1          250| 1024|  14060.15| 13.90|      1023|         0|        35|         0|      1848|         0|        12|    843688|    843752|    864475|       0.00|       0.00
2          250| 1024|  13982.47| 15.34|       913|         0|        34|         0|      1830|         0|        12|    839026|    839089|    859914|       0.00|       0.00
%             |  0.0|      -0.6| +10.4|     -10.8|         0|      -2.9|         0|      -1.0|         0|       0.0|      -0.6|      -0.6|      -0.5|          0|          0
1          500| 1024|  14051.14| 13.80|      1024|         0|        33|         0|      1716|         0|        13|    843144|    843212|    862981|       0.00|       0.00
2          500| 1024|  14013.95| 15.27|       918|         0|        32|         0|      1681|         0|        13|    840915|    840983|    862635|       0.00|       0.00
%             |  0.0|      -0.3| +10.7|     -10.4|         0|      -3.0|         0|      -2.0|         0|       0.0|      -0.3|      -0.3|      -0.0|          0|          0

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Amos Kong]

On 19/12/11 10:19, Amos Kong wrote:
> On 12/12/11 13:12, Rusty Russell wrote:
>> On Mon, 12 Dec 2011 11:06:53 +0800, Amos Kong<akong@redhat.com> wrote:
>>> On 12/12/11 06:27, Benjamin Herrenschmidt wrote:
>>>> On Sun, 2011-12-11 at 14:25 +0200, Michael S. Tsirkin wrote:
>>>>
>>>>> Forwarding some results by Amos, who run multiple netperf streams in
>>>>> parallel, from an external box to the guest. TCP_STREAM results were
>>>>> noisy. This could be due to buffering done by TCP, where packet size
>>>>> varies even as message size is constant.
>>>>>
>>>>> TCP_RR results were consistent. In this benchmark, after switching
>>>>> to mandatory barriers, CPU utilization increased by up to 35% while
>>>>> throughput went down by up to 14%. the normalized throughput/cpu
>>>>> regressed consistently, between 7 and 35%
>>>>>
>>>>> The "fix" applied was simply this:
>>>>
>>>> What machine& processor was this ?
>>>
>>> pined guest memory to numa node 1
>>
>> Please try this patch. How much does the branch cost us?
>>
>> (Compiles, untested).
>>
>> Thanks,
>> Rusty.
>>
>> From: Rusty Russell<rusty@rustcorp.com.au>
>> Subject: virtio: harsher barriers for virtio-mmio.
>>
>> We were cheating with our barriers; using the smp ones rather than the
>> real device ones. That was fine, until virtio-mmio came along, which
>> could be talking to a real device (a non-SMP CPU).
>>
>> Unfortunately, just putting back the real barriers (reverting
>> d57ed95d) causes a performance regression on virtio-pci. In
>> particular, Amos reports netbench's TCP_RR over virtio_net CPU
>> utilization increased up to 35% while throughput went down by up to
>> 14%.
>>
>> By comparison, this branch costs us???
>>
>> Reference: https://lkml.org/lkml/2011/12/11/22
>>
>> Signed-off-by: Rusty Russell<rusty@rustcorp.com.au>
>> ---
>> drivers/lguest/lguest_device.c | 10 ++++++----
>> drivers/s390/kvm/kvm_virtio.c | 2 +-
>> drivers/virtio/virtio_mmio.c | 7 ++++---
>> drivers/virtio/virtio_pci.c | 4 ++--
>> drivers/virtio/virtio_ring.c | 34 +++++++++++++++++++++-------------
>> include/linux/virtio_ring.h | 1 +
>> tools/virtio/linux/virtio.h | 1 +
>> tools/virtio/virtio_test.c | 3 ++-
>> 8 files changed, 38 insertions(+), 24 deletions(-)
>
> Hi all,
>
> I tested with the same environment and scenarios.
> tested one scenarios for three times and compute the average for more
> precision.
>
> Thanks, Amos
>
> --------- compare results -----------
> Mon Dec 19 09:51:09 2011
>
> 1 - avg-old.netperf.exhost_guest.txt
> 2 - avg-fixed.netperf.exhost_guest.txt
>
> ======
> TCP_STREAM
> sessions| size|throughput| cpu| normalize| #tx-pkts| #rx-pkts| #tx-byts|
> #rx-byts| #re-trans| #tx-intr| #rx-intr| #io_exit| #irq_inj|#tpkt/#exit|
> #rpkt/#irq
> 1 1| 64| 1073.54| 10.50| 102| 0| 31| 0| 1612| 0| 16| 487641| 489753|
> 504764| 0.00| 0.00
> 2 1| 64| 1079.44| 10.29| 104| 0| 30| 0| 1594| 0| 17| 487156| 488828|
> 504411| 0.00| 0.00
> % | 0.0| +0.5| -2.0| +2.0| 0| -3.2| 0| -1.1| 0| +6.2| -0.1| -0.2| -0.1|

The format is broken in webpage, attached the result file.
it's also available here: http://amosk.info/download/rusty-fix-perf.txt
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: rusty-fix-perf.txt
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20111219/5da85248/attachment-0001.txt>

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 19 Dec 2011 10:50:13 +0800, Amos Kong <akong@redhat.com> wrote:
> The format is broken in webpage, attached the result file.
> it's also available here: http://amosk.info/download/rusty-fix-perf.txt

Thanks Amos.

Linus, please apply.  Fixes virtio-mmio without breaking virtio_pci
performance.

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch is in the noise.

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 19 Dec 2011 10:50:13 +0800, Amos Kong <akong@redhat.com> wrote:
> The format is broken in webpage, attached the result file.
> it's also available here: http://amosk.info/download/rusty-fix-perf.txt

Thanks Amos.

Linus, please apply.  Fixes virtio-mmio without breaking virtio_pci
performance.

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch is in the noise.

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Rusty Russell]

On Mon, 19 Dec 2011 10:50:13 +0800, Amos Kong <akong@redhat.com> wrote:
> The format is broken in webpage, attached the result file.
> it's also available here: http://amosk.info/download/rusty-fix-perf.txt

Thanks Amos.

Linus, please apply.  Fixes virtio-mmio without breaking virtio_pci
performance.

Thanks,
Rusty.

From: Rusty Russell <rusty@rustcorp.com.au>
Subject: virtio: harsher barriers for virtio-mmio.

We were cheating with our barriers; using the smp ones rather than the
real device ones.  That was fine, until virtio-mmio came along, which
could be talking to a real device (a non-SMP CPU).

Unfortunately, just putting back the real barriers (reverting
d57ed95d) causes a performance regression on virtio-pci.  In
particular, Amos reports netbench's TCP_RR over virtio_net CPU
utilization increased up to 35% while throughput went down by up to
14%.

By comparison, this branch is in the noise.

Reference: https://lkml.org/lkml/2011/12/11/22

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 drivers/lguest/lguest_device.c |   10 ++++++----
 drivers/s390/kvm/kvm_virtio.c  |    2 +-
 drivers/virtio/virtio_mmio.c   |    7 ++++---
 drivers/virtio/virtio_pci.c    |    4 ++--
 drivers/virtio/virtio_ring.c   |   34 +++++++++++++++++++++-------------
 include/linux/virtio_ring.h    |    1 +
 tools/virtio/linux/virtio.h    |    1 +
 tools/virtio/virtio_test.c     |    3 ++-
 8 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -291,11 +291,13 @@ static struct virtqueue *lg_find_vq(stru
 	}
 
 	/*
-	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
-	 * and we've got a pointer to its pages.
+	 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
+	 * and we've got a pointer to its pages.  Note that we set weak_barriers
+	 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
+	 * barriers.
 	 */
-	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN,
-				 vdev, lvq->pages, lg_notify, callback, name);
+	vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
+				 true, lvq->pages, lg_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(str
 		goto out;
 
 	vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
-				 vdev, (void *) config->address,
+				 vdev, true, (void *) config->address,
 				 kvm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -309,9 +309,10 @@ static struct virtqueue *vm_setup_vq(str
 	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
 			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
 
-	/* Create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN,
-				 vdev, info->queue, vm_notify, callback, name);
+	/* Create the vring: no weak barriers, the other side is could
+	 * be an independent "device". */
+	vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+				 false, info->queue, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct
 		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
 	/* create the vring */
-	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
-				 vdev, info->queue, vp_notify, callback, name);
+	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
+				 true, info->queue, vp_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
 #ifdef CONFIG_SMP
 /* Where possible, use SMP barriers which are more lightweight than mandatory
  * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
+ * through relaxed memory I/O windows (which virtio-pci does not use). */
+#define virtio_mb(vq) \
+	do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
+#define virtio_rmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
+#define virtio_wmb(vq) \
+	do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
 #else
 /* We must force memory ordering even if guest is UP since host could be
  * running on another CPU, but SMP barriers are defined to barrier() in that
  * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
+#define virtio_mb(vq) mb()
+#define virtio_rmb(vq) rmb()
+#define virtio_wmb(vq) wmb()
 #endif
 
 #ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
 	/* Actual memory layout for this queue */
 	struct vring vring;
 
+	/* Can we use weak barriers? */
+	bool weak_barriers;
+
 	/* Other side has made a mess, don't try any more. */
 	bool broken;
 
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_v
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueu
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct 
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *),
 				      void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(un
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
 	vq->notify = notify;
+	vq->weak_barriers = weak_barriers;
 	vq->broken = false;
 	vq->last_used_idx = 0;
 	vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct
 struct virtqueue *vring_new_virtqueue(unsigned int num,
 				      unsigned int vring_align,
 				      struct virtio_device *vdev,
+				      bool weak_barriers,
 				      void *pages,
 				      void (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info
 	assert(r >= 0);
 	memset(info->ring, 0, vring_size(num, 4096));
 	vring_init(&info->vring, num, info->ring, 4096);
-	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
+	info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
+				       true, info->ring,
 				       vq_notify, vq_callback, "test");
 	assert(info->vq);
 	info->vq->priv = info;

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Sat, Dec 3, 2011 at 1:09 AM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
>
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

I agree.

MST wanted to give this a try this week. If all goes well and there's
no unreasonable regression, we'd just switch to mandatory barriers.

Ohad.

Re: [RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Sat, Dec 3, 2011 at 1:09 AM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
>
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

I agree.

MST wanted to give this a try this week. If all goes well and there's
no unreasonable regression, we'd just switch to mandatory barriers.

Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

On Sat, Dec 3, 2011 at 1:09 AM, Benjamin Herrenschmidt
<benh@kernel.crashing.org> wrote:
> Have you measured the impact of using normal barriers (non-SMP ones)
> like we use on normal HW drivers unconditionally ?
>
> IE. If the difference is small enough I'd say just go for it and avoid
> the bloat.

I agree.

MST wanted to give this a try this week. If all goes well and there's
no unreasonable regression, we'd just switch to mandatory barriers.

Ohad.

[RFC] virtio: use mandatory barriers for remote processor vdevs

[Ohad Ben-Cohen]

Virtio is using memory barriers to control the ordering of
references to the vrings on SMP systems. When the guest is compiled
with SMP support, virtio is only using SMP barriers in order to
avoid incurring the overhead involved with mandatory barriers.

Lately, though, virtio is being increasingly used with inter-processor
communication scenarios too, which involve running two (separate)
instances of operating systems on two (separate) processors, each of
which might either be UP or SMP.

To control the ordering of memory references when the vrings are shared
between two external processors, we must always use mandatory barriers.

A trivial, albeit sub-optimal, solution would be to simply revert
commit d57ed95 "virtio: use smp_XX barriers on SMP". Obviously, though,
that's going to have a negative impact on performance of SMP-based
virtualization use cases.

A different approach, as demonstrated by this patch, would pick the type
of memory barriers, in run time, according to the requirements of the
virtio device. This way, both SMP virtualization scenarios and inter-
processor communication use cases would run correctly, without making
any performance compromises (except for those incurred by an additional
branch or level of indirection).

This patch introduces VIRTIO_RING_F_REMOTEPROC, a new virtio transport
feature, which should be used by virtio devices that run on remote
processors. The CONFIG_SMP variant of virtio_{mb, rmb, wmb} is then changed
to use SMP barriers only if VIRTIO_RING_F_REMOTEPROC was absent.

Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
---
Alternatively, we can also introduce some kind of virtio_mb_ops and set it
according to the nature of the vdev with handlers that just do the right
thing, instead of introducting that branch.

Though I also wonder how big really is the perforamnce gain of d57ed95 ?

 drivers/virtio/virtio_ring.c |   78 +++++++++++++++++++++++++++++-------------
 include/linux/virtio_ring.h  |    6 +++
 2 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c7a2c20..cf66a2d 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -23,24 +23,6 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
-/* virtio guest is communicating with a virtual "device" that actually runs on
- * a host processor.  Memory barriers are used to control SMP effects. */
-#ifdef CONFIG_SMP
-/* Where possible, use SMP barriers which are more lightweight than mandatory
- * barriers, because mandatory barriers control MMIO effects on accesses
- * through relaxed memory I/O windows (which virtio does not use). */
-#define virtio_mb() smp_mb()
-#define virtio_rmb() smp_rmb()
-#define virtio_wmb() smp_wmb()
-#else
-/* We must force memory ordering even if guest is UP since host could be
- * running on another CPU, but SMP barriers are defined to barrier() in that
- * configuration. So fall back to mandatory barriers instead. */
-#define virtio_mb() mb()
-#define virtio_rmb() rmb()
-#define virtio_wmb() wmb()
-#endif
-
 #ifdef DEBUG
 /* For development, we want to crash whenever the ring is screwed. */
 #define BAD_RING(_vq, fmt, args...)				\
@@ -86,6 +68,9 @@ struct vring_virtqueue
 	/* Host publishes avail event idx */
 	bool event;
 
+	/* Host runs on a remote processor */
+	bool rproc;
+
 	/* Number of free buffers */
 	unsigned int num_free;
 	/* Head of free buffer list. */
@@ -108,6 +93,48 @@ struct vring_virtqueue
 	void *data[];
 };
 
+/*
+ * virtio guest is communicating with a virtual "device" that may either run
+ * on the host processor, or on an external processor. The former requires
+ * memory barriers in order to control SMP effects, but the latter must
+ * use mandatory barriers.
+ */
+#ifdef CONFIG_SMP
+/* Where possible, use SMP barriers which are more lightweight than mandatory
+ * barriers, because mandatory barriers control MMIO effects on accesses
+ * through relaxed memory I/O windows. */
+static inline void virtio_mb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		mb();
+	else
+		smp_mb();
+}
+
+static inline void virtio_rmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		rmb();
+	else
+		smp_rmb();
+}
+
+static inline void virtio_wmb(struct vring_virtqueue *vq)
+{
+	if (vq->rproc)
+		wmb();
+	else
+		smp_wmb();
+}
+#else
+/* We must force memory ordering even if guest is UP since host could be
+ * running on another CPU, but SMP barriers are defined to barrier() in that
+ * configuration. So fall back to mandatory barriers instead. */
+static inline void virtio_mb(struct vring_virtqueue *vq) { mb(); }
+static inline void virtio_rmb(struct vring_virtqueue *vq) { rmb(); }
+static inline void virtio_wmb(struct vring_virtqueue *vq) { wmb(); }
+#endif
+
 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 
 /* Set up an indirect table of descriptors and add it to the queue. */
@@ -245,14 +272,14 @@ void virtqueue_kick(struct virtqueue *_vq)
 	START_USE(vq);
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
-	virtio_wmb();
+	virtio_wmb(vq);
 
 	old = vq->vring.avail->idx;
 	new = vq->vring.avail->idx = old + vq->num_added;
 	vq->num_added = 0;
 
 	/* Need to update avail index before checking if we should notify */
-	virtio_mb();
+	virtio_mb(vq);
 
 	if (vq->event ?
 	    vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +341,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	}
 
 	/* Only get used array entries after they have been exposed by host. */
-	virtio_rmb();
+	virtio_rmb(vq);
 
 	i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
 	*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +364,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	 * the read in the next get_buf call. */
 	if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vring_used_event(&vq->vring) = vq->last_used_idx;
-		virtio_mb();
+		virtio_mb(vq);
 	}
 
 	END_USE(vq);
@@ -366,7 +393,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
 	 * entry. Always do both to keep code simple. */
 	vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 	vring_used_event(&vq->vring) = vq->last_used_idx;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely(more_used(vq))) {
 		END_USE(vq);
 		return false;
@@ -393,7 +420,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
 	vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
-	virtio_mb();
+	virtio_mb(vq);
 	if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -486,6 +513,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+	vq->rproc = virtio_has_feature(vdev, VIRTIO_RING_F_REMOTEPROC);
 
 	/* No callback?  Tell other side not to bother us. */
 	if (!callback)
@@ -522,6 +550,8 @@ void vring_transport_features(struct virtio_device *vdev)
 			break;
 		case VIRTIO_RING_F_EVENT_IDX:
 			break;
+		case VIRTIO_RING_F_REMOTEPROC:
+			break;
 		default:
 			/* We don't understand this bit. */
 			clear_bit(i, vdev->features);
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 36be0f6..9839593 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -58,6 +58,12 @@
  * at the end of the used ring. Guest should ignore the used->flags field. */
 #define VIRTIO_RING_F_EVENT_IDX		29
 
+/*
+ * The device we're talking to resides on a remote processor, so we must always
+ * use mandatory memory barriers.
+ */
+#define VIRTIO_RING_F_REMOTEPROC	30
+
 /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
 struct vring_desc {
 	/* Address (guest-physical). */
-- 
1.7.5.4