From: Rusty Russell <rusty@rustcorp.com.au>
To: virtualization@lists.linux-foundation.org
Subject: [PATCH 04/22] virtio_ring: inline internal vring functions more aggressively.
Date: Mon, 18 Mar 2013 20:04:35 +1030 [thread overview]
Message-ID: <1363599293-23065-5-git-send-email-rusty@rustcorp.com.au> (raw)
In-Reply-To: <1363599293-23065-1-git-send-email-rusty@rustcorp.com.au>
We use inline and get gcc to do the right thing inlining the
"indirect" traversal functions. This also means we don't need to
clean the sgs.
for i in `seq 50`; do /usr/bin/time -f 'Wall time:%e' ./vringh_test --indirect --eventidx --parallel --fast-vringh; done 2>&1 | stats --trim-outliers:
Before:
Using CPUS 0 and 3
Guest: notified 0, pinged 39014-39063(39062)
Host: notified 39014-39063(39062), pinged 0
Wall time:1.900000-2.350000(1.921875)
After:
Using CPUS 0 and 3
Guest: notified 0, pinged 39062-39063(39063)
Host: notified 39062-39063(39063), pinged 0
Wall time:1.760000-2.220000(1.789167)
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
drivers/virtio/virtio_ring.c | 91 ++++++++++++++++++++++++++----------------
1 file changed, 56 insertions(+), 35 deletions(-)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c537385..a78ad45 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -98,13 +98,31 @@ struct vring_virtqueue
#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
+static inline struct scatterlist *sg_next_chained(struct scatterlist *sg,
+ unsigned int *count)
+{
+ return sg_next(sg);
+}
+
+static inline struct scatterlist *sg_next_arr(struct scatterlist *sg,
+ unsigned int *count)
+{
+ if (--(*count) == 0)
+ return NULL;
+ return sg + 1;
+}
+
/* Set up an indirect table of descriptors and add it to the queue. */
-static int vring_add_indirect(struct vring_virtqueue *vq,
- struct scatterlist *sgs[],
- unsigned int total_sg,
- unsigned int out_sgs,
- unsigned int in_sgs,
- gfp_t gfp)
+static inline int vring_add_indirect(struct vring_virtqueue *vq,
+ struct scatterlist *sgs[],
+ struct scatterlist *(*next)
+ (struct scatterlist *, unsigned int *),
+ unsigned int total_sg,
+ unsigned int total_out,
+ unsigned int total_in,
+ unsigned int out_sgs,
+ unsigned int in_sgs,
+ gfp_t gfp)
{
struct vring_desc *desc;
unsigned head;
@@ -125,7 +143,7 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
/* Transfer entries from the sg lists into the indirect page */
i = 0;
for (n = 0; n < out_sgs; n++) {
- for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+ for (sg = sgs[n]; sg; sg = next(sg, &total_out)) {
desc[i].flags = VRING_DESC_F_NEXT;
desc[i].addr = sg_phys(sg);
desc[i].len = sg->length;
@@ -134,7 +152,7 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
}
}
for (; n < (out_sgs + in_sgs); n++) {
- for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+ for (sg = sgs[n]; sg; sg = next(sg, &total_in)) {
desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
desc[i].addr = sg_phys(sg);
desc[i].len = sg->length;
@@ -163,17 +181,20 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
return head;
}
-static int virtqueue_add(struct virtqueue *_vq,
- struct scatterlist *sgs[],
- unsigned int total_sg,
- unsigned int out_sgs,
- unsigned int in_sgs,
- void *data,
- gfp_t gfp)
+static inline int virtqueue_add(struct virtqueue *_vq,
+ struct scatterlist *sgs[],
+ struct scatterlist *(*next)
+ (struct scatterlist *, unsigned int *),
+ unsigned int total_out,
+ unsigned int total_in,
+ unsigned int out_sgs,
+ unsigned int in_sgs,
+ void *data,
+ gfp_t gfp)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct scatterlist *sg;
- unsigned int i, n, avail, uninitialized_var(prev);
+ unsigned int i, n, avail, uninitialized_var(prev), total_sg;
int head;
START_USE(vq);
@@ -193,11 +214,14 @@ static int virtqueue_add(struct virtqueue *_vq,
}
#endif
+ total_sg = total_in + total_out;
+
/* If the host supports indirect descriptor tables, and we have multiple
* buffers, then go indirect. FIXME: tune this threshold */
if (vq->indirect && total_sg > 1 && vq->vq.num_free) {
- head = vring_add_indirect(vq, sgs, total_sg, out_sgs, in_sgs,
- gfp);
+ head = vring_add_indirect(vq, sgs, next, total_sg, total_out,
+ total_in,
+ out_sgs, in_sgs, gfp);
if (likely(head >= 0))
goto add_head;
}
@@ -222,7 +246,7 @@ static int virtqueue_add(struct virtqueue *_vq,
head = i = vq->free_head;
for (n = 0; n < out_sgs; n++) {
- for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+ for (sg = sgs[n]; sg; sg = next(sg, &total_out)) {
vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
vq->vring.desc[i].addr = sg_phys(sg);
vq->vring.desc[i].len = sg->length;
@@ -231,7 +255,7 @@ static int virtqueue_add(struct virtqueue *_vq,
}
}
for (; n < (out_sgs + in_sgs); n++) {
- for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+ for (sg = sgs[n]; sg; sg = next(sg, &total_in)) {
vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
vq->vring.desc[i].addr = sg_phys(sg);
vq->vring.desc[i].len = sg->length;
@@ -293,21 +317,12 @@ int virtqueue_add_buf(struct virtqueue *_vq,
gfp_t gfp)
{
struct scatterlist *sgs[2];
- unsigned int i;
sgs[0] = sg;
sgs[1] = sg + out;
- /* Workaround until callers pass well-formed sgs. */
- for (i = 0; i < out + in; i++)
- sg_unmark_end(sg + i);
-
- sg_mark_end(sg + out + in - 1);
- if (out && in)
- sg_mark_end(sg + out - 1);
-
- return virtqueue_add(_vq, sgs, out+in, out ? 1 : 0, in ? 1 : 0,
- data, gfp);
+ return virtqueue_add(_vq, sgs, sg_next_arr,
+ out, in, out ? 1 : 0, in ? 1 : 0, data, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_buf);
@@ -332,15 +347,21 @@ int virtqueue_add_sgs(struct virtqueue *_vq,
void *data,
gfp_t gfp)
{
- unsigned int i, total_sg;
+ unsigned int i, total_out, total_in;
/* Count them first. */
- for (i = total_sg = 0; i < out_sgs + in_sgs; i++) {
+ for (i = total_out = total_in = 0; i < out_sgs; i++) {
+ struct scatterlist *sg;
+ for (sg = sgs[i]; sg; sg = sg_next(sg))
+ total_out++;
+ }
+ for (; i < out_sgs + in_sgs; i++) {
struct scatterlist *sg;
for (sg = sgs[i]; sg; sg = sg_next(sg))
- total_sg++;
+ total_in++;
}
- return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, data, gfp);
+ return virtqueue_add(_vq, sgs, sg_next_chained,
+ total_out, total_in, out_sgs, in_sgs, data, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
--
1.7.10.4
next prev parent reply other threads:[~2013-03-18 9:34 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2013-03-18 9:34 [PATCH 00/22] virtqueue_add_sgs, virtqueue_add_outbuf, virtqueue_add_inbuf Rusty Russell
2013-03-18 9:34 ` [PATCH 01/22] scatterlist: introduce sg_unmark_end Rusty Russell
2013-03-18 9:34 ` [PATCH 02/22] virtio_ring: virtqueue_add_sgs, to add multiple sgs Rusty Russell
2013-03-18 9:34 ` [PATCH 03/22] virtio_ring: don't count elements twice for add_buf path Rusty Russell
2013-03-18 9:34 ` Rusty Russell [this message]
2013-03-18 9:34 ` [PATCH 05/22] virtio_ring: virtqueue_add_outbuf / virtqueue_add_inbuf Rusty Russell
2013-03-18 9:34 ` [PATCH 06/22] tools/virtio: make vringh_test use inbuf/outbuf Rusty Russell
2013-03-18 9:34 ` [PATCH 07/22] virtio-blk: reorganize virtblk_add_req Rusty Russell
2013-03-18 9:34 ` [PATCH 08/22] virtio-blk: use virtqueue_add_sgs on bio path Rusty Russell
2013-03-18 9:34 ` [PATCH 09/22] virtio-blk: use virtqueue_add_sgs on req path Rusty Russell
2013-03-18 9:34 ` [PATCH 10/22] virtio_blk: remove nents member Rusty Russell
2013-03-18 9:34 ` [PATCH 11/22] virtio-scsi: use virtqueue_add_sgs for command buffers Rusty Russell
2013-03-19 6:51 ` Asias He
2013-03-19 9:48 ` [PATCH V2 " Wanlong Gao
2013-03-19 10:35 ` Paolo Bonzini
2013-03-20 0:35 ` Rusty Russell
2013-03-20 4:43 ` Asias He
2013-03-18 9:34 ` [PATCH 12/22] virtio_scsi: use virtqueue_add_inbuf() for virtscsi_kick_event Rusty Russell
2013-03-20 1:19 ` Asias He
2013-03-18 9:34 ` [PATCH 13/22] virtio_net: use virtqueue_add_sgs[] for command buffers Rusty Russell
2013-03-18 11:10 ` Michael S. Tsirkin
2013-03-18 23:47 ` Rusty Russell
2013-03-19 0:00 ` Wanlong Gao
2013-03-18 9:34 ` [PATCH 14/22] virtio_net: use simplified virtqueue accessors Rusty Russell
2013-03-20 1:20 ` Asias He
2013-03-18 9:34 ` [PATCH 15/22] virtio_rng: " Rusty Russell
2013-03-20 1:18 ` Asias He
2013-03-18 9:34 ` [PATCH 16/22] virtio_console: " Rusty Russell
2013-03-18 10:10 ` Amit Shah
2013-03-18 9:34 ` [PATCH 17/22] caif_virtio: " Rusty Russell
2013-03-18 9:34 ` [PATCH 18/22] virtio_rpmsg_bus: " Rusty Russell
2013-03-20 10:08 ` Ohad Ben-Cohen
2013-03-20 23:28 ` Rusty Russell
2013-03-18 9:34 ` [PATCH 19/22] virtio_balloon: " Rusty Russell
2013-03-18 9:34 ` [PATCH 20/22] 9p/trans_virtio.c: use virtio_add_sgs[] Rusty Russell
2013-03-20 4:53 ` Rusty Russell
2013-03-18 9:34 ` [PATCH 21/22] tools/virtio: remove virtqueue_add_buf() from tests Rusty Russell
2013-03-18 9:34 ` [PATCH 22/22] virtio: remove virtqueue_add_buf() Rusty Russell
2013-03-20 5:20 ` [PATCH 00/22] virtqueue_add_sgs, virtqueue_add_outbuf, virtqueue_add_inbuf Rusty Russell
2013-03-20 6:01 ` Asias He
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1363599293-23065-5-git-send-email-rusty@rustcorp.com.au \
--to=rusty@rustcorp.com.au \
--cc=virtualization@lists.linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).