[Qemu-devel] [PATCH 0/3] coroutine: Fix qemu_coroutine_yield()

[Qemu-devel] [PATCH 0/3] coroutine: Fix qemu_coroutine_yield()

[Kevin Wolf]

Kevin Wolf (2):
  coroutine: Fix use after free with qemu_coroutine_yield()
  coroutine: Clean up qemu_coroutine_enter()

Stefan Hajnoczi (1):
  test-coroutine: Regression test for yield bug

 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 38 ++++++++++++++++----------------------
 tests/test-coroutine.c        | 26 ++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 22 deletions(-)

-- 
1.8.3.1

[Qemu-devel] [PATCH 1/3] coroutine: Fix use after free with qemu_coroutine_yield()

[Kevin Wolf]

Instead of using the same function for entering and exiting coroutines,
and hoping that it doesn't add any functionality that hurts with the
parameters used for exiting, we can just directly call into the real
task switch in qemu_coroutine_switch().

This fixes a use-after-free scenario where reentering a coroutine that
has yielded still accesses the old parent coroutine (which may have
meanwhile terminated) in the part of coroutine_swap() that follows
qemu_coroutine_switch().

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 qemu-coroutine.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 525247b..5019b81 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -148,5 +148,5 @@ void coroutine_fn qemu_coroutine_yield(void)
     }
 
     self->caller = NULL;
-    coroutine_swap(self, to);
+    qemu_coroutine_switch(self, to, COROUTINE_YIELD);
 }
-- 
1.8.3.1

Re: [Qemu-devel] [PATCH 1/3] coroutine: Fix use after free with qemu_coroutine_yield()

[Kevin Wolf]

Am 10.02.2015 um 11:41 hat Kevin Wolf geschrieben:
> Instead of using the same function for entering and exiting coroutines,
> and hoping that it doesn't add any functionality that hurts with the
> parameters used for exiting, we can just directly call into the real
> task switch in qemu_coroutine_switch().
> 
> This fixes a use-after-free scenario where reentering a coroutine that
> has yielded still accesses the old parent coroutine (which may have
> meanwhile terminated) in the part of coroutine_swap() that follows
> qemu_coroutine_switch().
> 
> Signed-off-by: Kevin Wolf <kwolf@redhat.com>

Cc: qemu-stable@nongnu.org

Thanks to Peter for noticing that I forgot this.

Kevin

[Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Kevin Wolf]

qemu_coroutine_enter() is now the only user of coroutine_swap(). Both
functions are short, so inline it.

Also, using COROUTINE_YIELD is now even more confusing because this code
is never called during qemu_coroutine_yield() any more. In fact, this
value is never read back, so we can just introduce a new COROUTINE_ENTER
which documents the purpose of the task switch better.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/include/block/coroutine_int.h b/include/block/coroutine_int.h
index f133d65..69b83db 100644
--- a/include/block/coroutine_int.h
+++ b/include/block/coroutine_int.h
@@ -29,6 +29,7 @@
 #include "block/coroutine.h"
 
 typedef enum {
+    COROUTINE_ENTER = 0,
     COROUTINE_YIELD = 1,
     COROUTINE_TERMINATE = 2,
 } CoroutineAction;
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 5019b81..c17a92b 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -99,29 +99,10 @@ static void coroutine_delete(Coroutine *co)
     qemu_coroutine_delete(co);
 }
 
-static void coroutine_swap(Coroutine *from, Coroutine *to)
-{
-    CoroutineAction ret;
-
-    ret = qemu_coroutine_switch(from, to, COROUTINE_YIELD);
-
-    qemu_co_queue_run_restart(to);
-
-    switch (ret) {
-    case COROUTINE_YIELD:
-        return;
-    case COROUTINE_TERMINATE:
-        trace_qemu_coroutine_terminate(to);
-        coroutine_delete(to);
-        return;
-    default:
-        abort();
-    }
-}
-
 void qemu_coroutine_enter(Coroutine *co, void *opaque)
 {
     Coroutine *self = qemu_coroutine_self();
+    CoroutineAction ret;
 
     trace_qemu_coroutine_enter(self, co, opaque);
 
@@ -132,7 +113,20 @@ void qemu_coroutine_enter(Coroutine *co, void *opaque)
 
     co->caller = self;
     co->entry_arg = opaque;
-    coroutine_swap(self, co);
+    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
+
+    qemu_co_queue_run_restart(co);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(co);
+        coroutine_delete(co);
+        return;
+    default:
+        abort();
+    }
 }
 
 void coroutine_fn qemu_coroutine_yield(void)
-- 
1.8.3.1

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Paolo Bonzini]

On 10/02/2015 11:41, Kevin Wolf wrote:
> +    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
> +
> +    qemu_co_queue_run_restart(co);
> +
> +    switch (ret) {
> +    case COROUTINE_YIELD:
> +        return;
> +    case COROUTINE_TERMINATE:
> +        trace_qemu_coroutine_terminate(co);
> +        coroutine_delete(co);
> +        return;
> +    default:

Say you have:

  co1                                     co2
------------------------------------------------------------------------
1 qemu_co_mutex_lock(&m);
2 qemu_coroutine_yield();
3                                         qemu_co_mutex_lock(&m);
4 qemu_co_mutex_unlock(&m);
5 qemu_coroutine_yield();

Then you have:

1 mutex->locked = true;

2 coroutine_swap(co1, leader, COROUTINE_YIELD);

3 while (mutex->locked) {
     qemu_co_queue_wait(&mutex->queue);
           '--> QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
                qemu_coroutine_yield();
                '--> coroutine_swap(co2, leader, COROUTINE_YIELD);
  }

4 mutex->locked = false;
  qemu_co_queue_next(&mutex->queue);
   '--> qemu_co_queue_do_restart(queue, true);
        '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
             QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);

5 coroutine_swap(co1, leader, COROUTINE_YIELD);

And co2 is never reentered until co1 terminates.  Right?

Paolo

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Kevin Wolf]

Am 10.02.2015 um 11:55 hat Paolo Bonzini geschrieben:
> 
> 
> On 10/02/2015 11:41, Kevin Wolf wrote:
> > +    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
> > +
> > +    qemu_co_queue_run_restart(co);
> > +
> > +    switch (ret) {
> > +    case COROUTINE_YIELD:
> > +        return;
> > +    case COROUTINE_TERMINATE:
> > +        trace_qemu_coroutine_terminate(co);
> > +        coroutine_delete(co);
> > +        return;
> > +    default:
> 
> Say you have:
> 
>   co1                                     co2
> ------------------------------------------------------------------------
> 1 qemu_co_mutex_lock(&m);
> 2 qemu_coroutine_yield();
> 3                                         qemu_co_mutex_lock(&m);
> 4 qemu_co_mutex_unlock(&m);
> 5 qemu_coroutine_yield();
> 
> Then you have:
> 
> 1 mutex->locked = true;
> 
> 2 coroutine_swap(co1, leader, COROUTINE_YIELD);
> 
> 3 while (mutex->locked) {
>      qemu_co_queue_wait(&mutex->queue);
>            '--> QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
>                 qemu_coroutine_yield();
>                 '--> coroutine_swap(co2, leader, COROUTINE_YIELD);
>   }
> 
> 4 mutex->locked = false;
>   qemu_co_queue_next(&mutex->queue);
>    '--> qemu_co_queue_do_restart(queue, true);
>         '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
>              QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
> 
> 5 coroutine_swap(co1, leader, COROUTINE_YIELD);
> 
> And co2 is never reentered until co1 terminates.  Right?

No, co2 will be reentered during the yield in line 5. However, it's not
the yielding coroutine that reenters it but the parent, which is resumed
at exactly the line of code that you quoted above.

This is actually how it always worked, even with the bug. The bug caused
it to access the queue of a random other coroutine, but that queue must
have always been empty because it was already processed when that other
coroutine yielded/terminated.

Kevin

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Paolo Bonzini]

On 10/02/2015 12:09, Kevin Wolf wrote:
>> > 
>> > 4 mutex->locked = false;
>> >   qemu_co_queue_next(&mutex->queue);
>> >    '--> qemu_co_queue_do_restart(queue, true);
>> >         '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
>> >              QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
>> > 
>> > 5 coroutine_swap(co1, leader, COROUTINE_YIELD);
>> > 
>> > And co2 is never reentered until co1 terminates.  Right?
> No, co2 will be reentered during the yield in line 5. However, it's not
> the yielding coroutine that reenters it but the parent, which is resumed
> at exactly the line of code that you quoted above.

So:

5 coroutine_swap(co1, leader, COROUTINE_YIELD);
  '--> jumps back to qemu_coroutine_switch
       '--> returns to qemu_coroutine_enter

  qemu_co_queue_run_restart(co);
  '--> QTAILQ_REMOVE(&co->co_queue_wakeup, next, co_queue_next);
       qemu_coroutine_enter(next, NULL);


Thanks for the explanation.  Series:

Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Stefan Hajnoczi]

[-- Attachment #1: Type: text/plain, Size: 1370 bytes --]

On Tue, Feb 10, 2015 at 11:41:27AM +0100, Kevin Wolf wrote:
> qemu_coroutine_enter() is now the only user of coroutine_swap(). Both
> functions are short, so inline it.
> 
> Also, using COROUTINE_YIELD is now even more confusing because this code
> is never called during qemu_coroutine_yield() any more. In fact, this
> value is never read back, so we can just introduce a new COROUTINE_ENTER
> which documents the purpose of the task switch better.
> 
> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
> ---
>  include/block/coroutine_int.h |  1 +
>  qemu-coroutine.c              | 36 +++++++++++++++---------------------
>  2 files changed, 16 insertions(+), 21 deletions(-)
> 
> diff --git a/include/block/coroutine_int.h b/include/block/coroutine_int.h
> index f133d65..69b83db 100644
> --- a/include/block/coroutine_int.h
> +++ b/include/block/coroutine_int.h
> @@ -29,6 +29,7 @@
>  #include "block/coroutine.h"
>  
>  typedef enum {
> +    COROUTINE_ENTER = 0,

This makes the ucontext code harder to understand because
CoroutineAction values are used with setjmp()/longjmp() in
qemu_coroutine_switch().

The longjmp() man page says:

  If longjmp() is invoked with a second argument of 0, 1 will be
  returned instead.

I haven't checked whether or not this causes problems, but the code
would be simpler if we avoided using 0.

[-- Attachment #2: Type: application/pgp-signature, Size: 473 bytes --]

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Kevin Wolf]

[-- Attachment #1: Type: text/plain, Size: 1604 bytes --]

Am 18.02.2015 um 14:50 hat Stefan Hajnoczi geschrieben:
> On Tue, Feb 10, 2015 at 11:41:27AM +0100, Kevin Wolf wrote:
> > qemu_coroutine_enter() is now the only user of coroutine_swap(). Both
> > functions are short, so inline it.
> > 
> > Also, using COROUTINE_YIELD is now even more confusing because this code
> > is never called during qemu_coroutine_yield() any more. In fact, this
> > value is never read back, so we can just introduce a new COROUTINE_ENTER
> > which documents the purpose of the task switch better.
> > 
> > Signed-off-by: Kevin Wolf <kwolf@redhat.com>
> > ---
> >  include/block/coroutine_int.h |  1 +
> >  qemu-coroutine.c              | 36 +++++++++++++++---------------------
> >  2 files changed, 16 insertions(+), 21 deletions(-)
> > 
> > diff --git a/include/block/coroutine_int.h b/include/block/coroutine_int.h
> > index f133d65..69b83db 100644
> > --- a/include/block/coroutine_int.h
> > +++ b/include/block/coroutine_int.h
> > @@ -29,6 +29,7 @@
> >  #include "block/coroutine.h"
> >  
> >  typedef enum {
> > +    COROUTINE_ENTER = 0,
> 
> This makes the ucontext code harder to understand because
> CoroutineAction values are used with setjmp()/longjmp() in
> qemu_coroutine_switch().
> 
> The longjmp() man page says:
> 
>   If longjmp() is invoked with a second argument of 0, 1 will be
>   returned instead.
> 
> I haven't checked whether or not this causes problems, but the code
> would be simpler if we avoided using 0.

It doesn't, the value is unused where we pass COROUTINE_ENTER. But I can
make it 3 instead.

Kevin

[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

[Stefan Hajnoczi]

[-- Attachment #1: Type: text/plain, Size: 1777 bytes --]

On Wed, Feb 18, 2015 at 03:20:26PM +0100, Kevin Wolf wrote:
> Am 18.02.2015 um 14:50 hat Stefan Hajnoczi geschrieben:
> > On Tue, Feb 10, 2015 at 11:41:27AM +0100, Kevin Wolf wrote:
> > > qemu_coroutine_enter() is now the only user of coroutine_swap(). Both
> > > functions are short, so inline it.
> > > 
> > > Also, using COROUTINE_YIELD is now even more confusing because this code
> > > is never called during qemu_coroutine_yield() any more. In fact, this
> > > value is never read back, so we can just introduce a new COROUTINE_ENTER
> > > which documents the purpose of the task switch better.
> > > 
> > > Signed-off-by: Kevin Wolf <kwolf@redhat.com>
> > > ---
> > >  include/block/coroutine_int.h |  1 +
> > >  qemu-coroutine.c              | 36 +++++++++++++++---------------------
> > >  2 files changed, 16 insertions(+), 21 deletions(-)
> > > 
> > > diff --git a/include/block/coroutine_int.h b/include/block/coroutine_int.h
> > > index f133d65..69b83db 100644
> > > --- a/include/block/coroutine_int.h
> > > +++ b/include/block/coroutine_int.h
> > > @@ -29,6 +29,7 @@
> > >  #include "block/coroutine.h"
> > >  
> > >  typedef enum {
> > > +    COROUTINE_ENTER = 0,
> > 
> > This makes the ucontext code harder to understand because
> > CoroutineAction values are used with setjmp()/longjmp() in
> > qemu_coroutine_switch().
> > 
> > The longjmp() man page says:
> > 
> >   If longjmp() is invoked with a second argument of 0, 1 will be
> >   returned instead.
> > 
> > I haven't checked whether or not this causes problems, but the code
> > would be simpler if we avoided using 0.
> 
> It doesn't, the value is unused where we pass COROUTINE_ENTER. But I can
> make it 3 instead.

Thanks, that would be good.

Stefan

[-- Attachment #2: Type: application/pgp-signature, Size: 473 bytes --]

[Qemu-devel] [PATCH 3/3] test-coroutine: Regression test for yield bug

[Kevin Wolf]

From: Stefan Hajnoczi <stefanha@redhat.com>

This adds a test for reentering a coroutine that previously yielded to a
coroutine that has meanwhile terminated.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-coroutine.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index 27d1b6f..b552d9f 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -13,6 +13,7 @@
 
 #include <glib.h>
 #include "block/coroutine.h"
+#include "block/coroutine_int.h"
 
 /*
  * Check that qemu_in_coroutine() works
@@ -122,6 +123,30 @@ static void test_yield(void)
     g_assert_cmpint(i, ==, 5); /* coroutine must yield 5 times */
 }
 
+static void coroutine_fn c2_fn(void *opaque)
+{
+    qemu_coroutine_yield();
+}
+
+static void coroutine_fn c1_fn(void *opaque)
+{
+    Coroutine *c2 = opaque;
+    qemu_coroutine_enter(c2, NULL);
+}
+
+static void test_co_queue(void)
+{
+    Coroutine *c1;
+    Coroutine *c2;
+
+    c1 = qemu_coroutine_create(c1_fn);
+    c2 = qemu_coroutine_create(c2_fn);
+
+    qemu_coroutine_enter(c1, c2);
+    memset(c1, 0xff, sizeof(Coroutine));
+    qemu_coroutine_enter(c2, NULL);
+}
+
 /*
  * Check that creation, enter, and return work
  */
@@ -343,6 +368,7 @@ static void perf_cost(void)
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/basic/co_queue", test_co_queue);
     g_test_add_func("/basic/lifecycle", test_lifecycle);
     g_test_add_func("/basic/yield", test_yield);
     g_test_add_func("/basic/nesting", test_nesting);
-- 
1.8.3.1

Re: [Qemu-devel] [PATCH 0/3] coroutine: Fix qemu_coroutine_yield()

[Kevin Wolf]

Am 10.02.2015 um 11:41 hat Kevin Wolf geschrieben:
> Kevin Wolf (2):
>   coroutine: Fix use after free with qemu_coroutine_yield()
>   coroutine: Clean up qemu_coroutine_enter()
> 
> Stefan Hajnoczi (1):
>   test-coroutine: Regression test for yield bug
> 
>  include/block/coroutine_int.h |  1 +
>  qemu-coroutine.c              | 38 ++++++++++++++++----------------------
>  tests/test-coroutine.c        | 26 ++++++++++++++++++++++++++
>  3 files changed, 43 insertions(+), 22 deletions(-)

Applied to the block branch.

Kevin