[Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[David Hildenbrand]

pause_all_cpus() is sometimes called from a VCPU thread (e.g. s390x
during special reset). It cannot deal with multiple VCPUs per Thread
(single threaded TCG) yet.

Booting an s390x guest with -smp 2 and single threaded TCG from disk
currently fails. The DIAG 308 will issue a pause_all_cpus() and wait
forever for the CPUs to actually stop. But it is waiting for itself.

So let's stop all VCPUs belonging to the current thread. Factor out
stopping of a VCPU.

Signed-off-by: David Hildenbrand <david@redhat.com>
---
 cpus.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/cpus.c b/cpus.c
index 114c29b6a0..3740c4db62 100644
--- a/cpus.c
+++ b/cpus.c
@@ -1057,13 +1057,22 @@ static void qemu_tcg_destroy_vcpu(CPUState *cpu)
 {
 }
 
+static void qemu_cpu_stop(CPUState *cpu, bool exit)
+{
+    g_assert(qemu_cpu_is_self(cpu));
+    cpu->stop = false;
+    cpu->stopped = true;
+    if (exit) {
+        cpu_exit(cpu);
+    }
+    qemu_cond_broadcast(&qemu_pause_cond);
+}
+
 static void qemu_wait_io_event_common(CPUState *cpu)
 {
     atomic_mb_set(&cpu->thread_kicked, false);
     if (cpu->stop) {
-        cpu->stop = false;
-        cpu->stopped = true;
-        qemu_cond_broadcast(&qemu_pause_cond);
+        qemu_cpu_stop(cpu, false);
     }
     process_queued_cpu_work(cpu);
 }
@@ -1610,12 +1619,12 @@ void pause_all_vcpus(void)
 
     qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false);
     CPU_FOREACH(cpu) {
-        cpu->stop = true;
-        qemu_cpu_kick(cpu);
-    }
-
-    if (qemu_in_vcpu_thread()) {
-        cpu_stop_current();
+        if (qemu_cpu_is_self(cpu)) {
+            qemu_cpu_stop(cpu, true);
+        } else {
+            cpu->stop = true;
+            qemu_cpu_kick(cpu);
+        }
     }
 
     while (!all_vcpus_paused()) {
@@ -1799,10 +1808,7 @@ void qemu_init_vcpu(CPUState *cpu)
 void cpu_stop_current(void)
 {
     if (current_cpu) {
-        current_cpu->stop = false;
-        current_cpu->stopped = true;
-        cpu_exit(current_cpu);
-        qemu_cond_broadcast(&qemu_pause_cond);
+        qemu_cpu_stop(current_cpu, true);
     }
 }
 
-- 
2.14.3

Re: [Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[Paolo Bonzini]

On 29/11/2017 20:12, David Hildenbrand wrote:
> pause_all_cpus() is sometimes called from a VCPU thread (e.g. s390x
> during special reset). It cannot deal with multiple VCPUs per Thread
> (single threaded TCG) yet.
> 
> Booting an s390x guest with -smp 2 and single threaded TCG from disk
> currently fails. The DIAG 308 will issue a pause_all_cpus() and wait
> forever for the CPUs to actually stop. But it is waiting for itself.
> 
> So let's stop all VCPUs belonging to the current thread. Factor out
> stopping of a VCPU.
> 
> Signed-off-by: David Hildenbrand <david@redhat.com>
> ---
>  cpus.c | 32 +++++++++++++++++++-------------
>  1 file changed, 19 insertions(+), 13 deletions(-)
> 
> diff --git a/cpus.c b/cpus.c
> index 114c29b6a0..3740c4db62 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -1057,13 +1057,22 @@ static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>  {
>  }
>  
> +static void qemu_cpu_stop(CPUState *cpu, bool exit)
> +{
> +    g_assert(qemu_cpu_is_self(cpu));
> +    cpu->stop = false;
> +    cpu->stopped = true;
> +    if (exit) {
> +        cpu_exit(cpu);
> +    }
> +    qemu_cond_broadcast(&qemu_pause_cond);
> +}
> +
>  static void qemu_wait_io_event_common(CPUState *cpu)
>  {
>      atomic_mb_set(&cpu->thread_kicked, false);
>      if (cpu->stop) {
> -        cpu->stop = false;
> -        cpu->stopped = true;
> -        qemu_cond_broadcast(&qemu_pause_cond);
> +        qemu_cpu_stop(cpu, false);
>      }
>      process_queued_cpu_work(cpu);
>  }
> @@ -1610,12 +1619,12 @@ void pause_all_vcpus(void)
>  
>      qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false);
>      CPU_FOREACH(cpu) {
> -        cpu->stop = true;
> -        qemu_cpu_kick(cpu);
> -    }
> -
> -    if (qemu_in_vcpu_thread()) {
> -        cpu_stop_current();
> +        if (qemu_cpu_is_self(cpu)) {
> +            qemu_cpu_stop(cpu, true);
> +        } else {
> +            cpu->stop = true;
> +            qemu_cpu_kick(cpu);
> +        }
>      }
>  
>      while (!all_vcpus_paused()) {
> @@ -1799,10 +1808,7 @@ void qemu_init_vcpu(CPUState *cpu)
>  void cpu_stop_current(void)
>  {
>      if (current_cpu) {
> -        current_cpu->stop = false;
> -        current_cpu->stopped = true;
> -        cpu_exit(current_cpu);
> -        qemu_cond_broadcast(&qemu_pause_cond);
> +        qemu_cpu_stop(current_cpu, true);
>      }
>  }

This function is not needed anymore, because vm_stop can just call
qemu_cpu_stop.  Does the following squash look fine to you?

diff --git a/cpus.c b/cpus.c
index 83700c1716..a65591c183 100644
--- a/cpus.c
+++ b/cpus.c
@@ -1057,14 +1057,11 @@ static void qemu_tcg_destroy_vcpu(CPUState *cpu)
 {
 }
 
-static void qemu_cpu_stop(CPUState *cpu, bool exit)
+static void qemu_cpu_stop(CPUState *cpu)
 {
     g_assert(qemu_cpu_is_self(cpu));
     cpu->stop = false;
     cpu->stopped = true;
-    if (exit) {
-        cpu_exit(cpu);
-    }
     qemu_cond_broadcast(&qemu_pause_cond);
 }
 
@@ -1072,7 +1069,7 @@ static void qemu_wait_io_event_common(CPUState *cpu)
 {
     atomic_mb_set(&cpu->thread_kicked, false);
     if (cpu->stop) {
-        qemu_cpu_stop(cpu, false);
+        qemu_cpu_stop(cpu);
     }
     process_queued_cpu_work(cpu);
 }
@@ -1620,7 +1617,8 @@ void pause_all_vcpus(void)
     qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false);
     CPU_FOREACH(cpu) {
         if (qemu_cpu_is_self(cpu)) {
-            qemu_cpu_stop(cpu, true);
+            qemu_cpu_stop(cpu);
+            cpu_exit(cpu);
         } else {
             cpu->stop = true;
             qemu_cpu_kick(cpu);
@@ -1802,13 +1800,6 @@ void qemu_init_vcpu(CPUState *cpu)
     }
 }
 
-void cpu_stop_current(void)
-{
-    if (current_cpu) {
-        qemu_cpu_stop(current_cpu, true);
-    }
-}
-
 int vm_stop(RunState state)
 {
     if (qemu_in_vcpu_thread()) {
@@ -1818,7 +1809,8 @@ int vm_stop(RunState state)
          * FIXME: should not return to device code in case
          * vm_stop() has been requested.
          */
-        cpu_stop_current();
+        qemu_cpu_stop(current_cpu);
+        cpu_exit(current_cpu);
         return 0;
     }
 

Re: [Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[David Hildenbrand]

>  int vm_stop(RunState state)
>  {
>      if (qemu_in_vcpu_thread()) {
> @@ -1818,7 +1809,8 @@ int vm_stop(RunState state)
>           * FIXME: should not return to device code in case
>           * vm_stop() has been requested.
>           */
> -        cpu_stop_current();
> +        qemu_cpu_stop(current_cpu);
> +        cpu_exit(current_cpu);

We're doing the cpu_exit() now after the broadcast, is this ok?

Also we drop the check for current_cpu, I assume this is also ok.

>          return 0;
>      }
> 

-- 

Thanks,

David / dhildenb

Re: [Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[David Hildenbrand]

> -void cpu_stop_current(void)
> -{
> -    if (current_cpu) {
> -        qemu_cpu_stop(current_cpu, true);
> -    }
> -}

Btw. this does not compile as this is used also in vl.c

> -
>  int vm_stop(RunState state)
>  {
>      if (qemu_in_vcpu_thread()) {
> @@ -1818,7 +1809,8 @@ int vm_stop(RunState state)
>           * FIXME: should not return to device code in case
>           * vm_stop() has been requested.
>           */
> -        cpu_stop_current();
> +        qemu_cpu_stop(current_cpu);
> +        cpu_exit(current_cpu);
>          return 0;
>      }
>  
> 


-- 

Thanks,

David / dhildenb

Re: [Qemu-devel] [PATCH v1] cpus: make pause_all_cpus() play with SMP on single threaded TCG

[Paolo Bonzini]

On 11/12/2017 17:44, David Hildenbrand wrote:
>> -void cpu_stop_current(void)
>> -{
>> -    if (current_cpu) {
>> -        qemu_cpu_stop(current_cpu, true);
>> -    }
>> -}
> Btw. this does not compile as this is used also in vl.c
> 

Doh, then I'm applying your patch untouched.

Paolo