[Qemu-devel] [patch uq/master 0/9] enable smp > 1 and related fixes

[Qemu-devel] [patch uq/master 0/9] enable smp > 1 and related fixes

[Marcelo Tosatti]

[Qemu-devel] [patch uq/master 1/9] kvm: set cpu_single_env around KVM_RUN ioctl

[Marcelo Tosatti]

[-- Attachment #1: cpu-single-env --]
[-- Type: text/plain, Size: 648 bytes --]

Zero cpu_single_env before leaving global lock protection, and
restore on return.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/kvm-all.c
===================================================================
--- qemu.orig/kvm-all.c
+++ qemu/kvm-all.c
@@ -846,9 +846,11 @@ int kvm_cpu_exec(CPUState *env)
         }
 
         kvm_arch_pre_run(env, run);
+        cpu_single_env = NULL;
         qemu_mutex_unlock_iothread();
         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
         qemu_mutex_lock_iothread();
+        cpu_single_env = env;
         kvm_arch_post_run(env, run);
 
         if (ret == -EINTR || ret == -EAGAIN) {

[Qemu-devel] [patch uq/master 2/9] make SIG_IPI to tcg vcpu thread reliable

[Marcelo Tosatti]

[-- Attachment #1: qemu-remove-signal-lock --]
[-- Type: text/plain, Size: 2515 bytes --]

Store tcg loop exit request on a global variable, and transfer it to
per-CPUState exit_request after assignment of cpu_single_env.

This makes exit request signal from robust. Drop the timedlock hack.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/cpu-exec.c
===================================================================
--- qemu.orig/cpu-exec.c
+++ qemu/cpu-exec.c  
@@ -213,6 +213,8 @@ static void cpu_handle_debug_exception(C
 
 /* main execution loop */
 
+volatile sig_atomic_t exit_request;
+
 int cpu_exec(CPUState *env1)
 {
     volatile host_reg_t saved_env_reg;
@@ -234,6 +236,11 @@ int cpu_exec(CPUState *env1)
     asm("");
     env = env1;
 
+    if (exit_request) {
+        env->exit_request = 1;
+        exit_request = 0;
+    }
+
 #if defined(TARGET_I386)
     if (!kvm_enabled()) {
         /* put eflags in CPU temporary format */
Index: qemu/cpus.c
===================================================================
--- qemu.orig/cpus.c
+++ qemu/cpus.c
@@ -472,6 +472,7 @@ static void cpu_signal(int sig)
 {
     if (cpu_single_env)
         cpu_exit(cpu_single_env);
+    exit_request = 1;
 }
 
 static void tcg_block_io_signals(void)
@@ -542,26 +543,20 @@ static void unblock_io_signals(void)
     pthread_sigmask(SIG_BLOCK, &set, NULL);
 }
 
-static void qemu_signal_lock(unsigned int msecs)
-{
-    qemu_mutex_lock(&qemu_fair_mutex);
-
-    while (qemu_mutex_trylock(&qemu_global_mutex)) {
-        qemu_thread_signal(tcg_cpu_thread, SIG_IPI);
-        if (!qemu_mutex_timedlock(&qemu_global_mutex, msecs))
-            break;
-    }
-    qemu_mutex_unlock(&qemu_fair_mutex);
-}
-
 void qemu_mutex_lock_iothread(void)
 {
     if (kvm_enabled()) {
         qemu_mutex_lock(&qemu_fair_mutex);
         qemu_mutex_lock(&qemu_global_mutex);
         qemu_mutex_unlock(&qemu_fair_mutex);
-    } else
-        qemu_signal_lock(100);
+    } else {
+        qemu_mutex_lock(&qemu_fair_mutex);
+        if (qemu_mutex_trylock(&qemu_global_mutex)) {
+            qemu_thread_signal(tcg_cpu_thread, SIG_IPI);
+            qemu_mutex_lock(&qemu_global_mutex);
+        }
+        qemu_mutex_unlock(&qemu_fair_mutex);
+    }
 }
 
 void qemu_mutex_unlock_iothread(void)
Index: qemu/exec-all.h
===================================================================
--- qemu.orig/exec-all.h
+++ qemu/exec-all.h
@@ -339,4 +339,7 @@ CPUDebugExcpHandler *cpu_set_debug_excp_
 /* vl.c */
 extern int singlestep;
 
+/* cpu-exec.c */
+extern volatile sig_atomic_t exit_request;
+
 #endif

[Qemu-devel] [patch uq/master 3/9] standardize on qemu_cpu_kick for signalling cpu thread(s)

[Marcelo Tosatti]

[-- Attachment #1: ipi-on-cpu-kick --]
[-- Type: text/plain, Size: 1267 bytes --]

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/cpus.c
===================================================================
--- qemu.orig/cpus.c
+++ qemu/cpus.c
@@ -454,8 +454,7 @@ void qemu_cpu_kick(void *_env)
 {
     CPUState *env = _env;
     qemu_cond_broadcast(env->halt_cond);
-    if (kvm_enabled())
-        qemu_thread_signal(env->thread, SIG_IPI);
+    qemu_thread_signal(env->thread, SIG_IPI);
 }
 
 int qemu_cpu_self(void *_env)
@@ -583,7 +582,6 @@ void pause_all_vcpus(void)
 
     while (penv) {
         penv->stop = 1;
-        qemu_thread_signal(penv->thread, SIG_IPI);
         qemu_cpu_kick(penv);
         penv = (CPUState *)penv->next_cpu;
     }
@@ -592,7 +590,7 @@ void pause_all_vcpus(void)
         qemu_cond_timedwait(&qemu_pause_cond, &qemu_global_mutex, 100);
         penv = first_cpu;
         while (penv) {
-            qemu_thread_signal(penv->thread, SIG_IPI);
+            qemu_cpu_kick(penv);
             penv = (CPUState *)penv->next_cpu;
         }
     }
@@ -605,7 +603,6 @@ void resume_all_vcpus(void)
     while (penv) {
         penv->stop = 0;
         penv->stopped = 0;
-        qemu_thread_signal(penv->thread, SIG_IPI);
         qemu_cpu_kick(penv);
         penv = (CPUState *)penv->next_cpu;
     }

[Qemu-devel] [patch uq/master 4/9] port qemu-kvm's on_vcpu code

[Marcelo Tosatti]

[-- Attachment #1: on-vcpu --]
[-- Type: text/plain, Size: 4143 bytes --]

run_on_cpu allows to execute work on a given CPUState context.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/cpu-all.h
===================================================================
--- qemu.orig/cpu-all.h
+++ qemu/cpu-all.h
@@ -818,6 +818,7 @@ void cpu_watchpoint_remove_all(CPUState 
 
 void cpu_single_step(CPUState *env, int enabled);
 void cpu_reset(CPUState *s);
+void run_on_cpu(CPUState *env, void (*func)(void *data), void *data);
 
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
 #define CPU_LOG_TB_IN_ASM  (1 << 1)
Index: qemu/cpu-defs.h
===================================================================
--- qemu.orig/cpu-defs.h
+++ qemu/cpu-defs.h
@@ -132,6 +132,7 @@ typedef struct icount_decr_u16 {
 
 struct kvm_run;
 struct KVMState;
+struct qemu_work_item;
 
 typedef struct CPUBreakpoint {
     target_ulong pc;
@@ -204,6 +205,7 @@ typedef struct CPUWatchpoint {
     uint32_t created;                                                   \
     struct QemuThread *thread;                                          \
     struct QemuCond *halt_cond;                                         \
+    struct qemu_work_item *queued_work_first, *queued_work_last;        \
     const char *cpu_model_str;                                          \
     struct KVMState *kvm_state;                                         \
     struct kvm_run *kvm_run;                                            \
Index: qemu/cpus.c
===================================================================
--- qemu.orig/cpus.c
+++ qemu/cpus.c
@@ -115,6 +115,8 @@ static int cpu_has_work(CPUState *env)
 {
     if (env->stop)
         return 1;
+    if (env->queued_work_first)
+        return 1;
     if (env->stopped || !vm_running)
         return 0;
     if (!env->halted)
@@ -252,6 +254,11 @@ int qemu_cpu_self(void *env)
     return 1;
 }
 
+void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
+{
+    func(data);
+}
+
 void resume_all_vcpus(void)
 {
 }
@@ -304,6 +311,7 @@ static QemuCond qemu_cpu_cond;
 /* system init */
 static QemuCond qemu_system_cond;
 static QemuCond qemu_pause_cond;
+static QemuCond qemu_work_cond;
 
 static void tcg_block_io_signals(void);
 static void kvm_block_io_signals(CPUState *env);
@@ -334,6 +342,50 @@ void qemu_main_loop_start(void)
     qemu_cond_broadcast(&qemu_system_cond);
 }
 
+void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
+{
+    struct qemu_work_item wi;
+
+    if (qemu_cpu_self(env)) {
+        func(data);
+        return;
+    }
+
+    wi.func = func;
+    wi.data = data;
+    if (!env->queued_work_first)
+        env->queued_work_first = &wi;
+    else
+        env->queued_work_last->next = &wi;
+    env->queued_work_last = &wi;
+    wi.next = NULL;
+    wi.done = false;
+
+    qemu_cpu_kick(env);
+    while (!wi.done) {
+        CPUState *self_env = cpu_single_env;
+
+        qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
+        cpu_single_env = self_env;
+    }
+}
+
+static void flush_queued_work(CPUState *env)
+{
+    struct qemu_work_item *wi;
+
+    if (!env->queued_work_first)
+        return;
+
+    while ((wi = env->queued_work_first)) {
+        env->queued_work_first = wi->next;
+        wi->func(wi->data);
+        wi->done = true;
+    }
+    env->queued_work_last = NULL;
+    qemu_cond_broadcast(&qemu_work_cond);
+}
+
 static void qemu_wait_io_event_common(CPUState *env)
 {
     if (env->stop) {
@@ -341,6 +393,7 @@ static void qemu_wait_io_event_common(CP
         env->stopped = 1;
         qemu_cond_signal(&qemu_pause_cond);
     }
+    flush_queued_work(env);
 }
 
 static void qemu_wait_io_event(CPUState *env)
Index: qemu/qemu-common.h
===================================================================
--- qemu.orig/qemu-common.h
+++ qemu/qemu-common.h
@@ -249,6 +249,14 @@ void qemu_notify_event(void);
 void qemu_cpu_kick(void *env);
 int qemu_cpu_self(void *env);
 
+/* work queue */
+struct qemu_work_item {
+    struct qemu_work_item *next;
+    void (*func)(void *data);
+    void *data;
+    int done;
+};
+
 #ifdef CONFIG_USER_ONLY
 #define qemu_init_vcpu(env) do { } while (0)
 #else

[Qemu-devel] [patch uq/master 5/9] kvm: synchronize state from cpu context

[Marcelo Tosatti]

[-- Attachment #1: cpu-sync-state-use-run-on-cpu --]
[-- Type: text/plain, Size: 980 bytes --]

From: Jan Kiszka <jan.kiszka@siemens.com>

It is not safe to retrieve the KVM internal state of a given cpu
while its potentially modifying it.

Queue the request to run on cpu context, similarly to qemu-kvm.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/kvm-all.c
===================================================================
--- qemu.orig/kvm-all.c
+++ qemu/kvm-all.c
@@ -796,14 +796,22 @@ void kvm_flush_coalesced_mmio_buffer(voi
 #endif
 }
 
-void kvm_cpu_synchronize_state(CPUState *env)
+static void do_kvm_cpu_synchronize_state(void *_env)
 {
+    CPUState *env = _env;
+
     if (!env->kvm_vcpu_dirty) {
         kvm_arch_get_registers(env);
         env->kvm_vcpu_dirty = 1;
     }
 }
 
+void kvm_cpu_synchronize_state(CPUState *env)
+{
+    if (!env->kvm_vcpu_dirty)
+        run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
+}
+
 void kvm_cpu_synchronize_post_reset(CPUState *env)
 {
     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);

[Qemu-devel] Re: [patch uq/master 5/9] kvm: synchronize state from cpu context

[Avi Kivity]

On 05/04/2010 03:45 PM, Marcelo Tosatti wrote:
> From: Jan Kiszka<jan.kiszka@siemens.com>
>
> It is not safe to retrieve the KVM internal state of a given cpu
> while its potentially modifying it.
>
> Queue the request to run on cpu context, similarly to qemu-kvm.
>    

Even better is to query the state in the cpu thread; but that can come 
later.

-- 
error compiling committee.c: too many arguments to function

[Qemu-devel] [patch uq/master 6/9] add cpu_is_stopped helper

[Marcelo Tosatti]

[-- Attachment #1: cpu-is-stopped --]
[-- Type: text/plain, Size: 819 bytes --]

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/cpu-all.h
===================================================================
--- qemu.orig/cpu-all.h
+++ qemu/cpu-all.h
@@ -818,6 +818,7 @@ void cpu_watchpoint_remove_all(CPUState 
 
 void cpu_single_step(CPUState *env, int enabled);
 void cpu_reset(CPUState *s);
+int cpu_is_stopped(CPUState *env);
 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data);
 
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
Index: qemu/cpus.c
===================================================================
--- qemu.orig/cpus.c
+++ qemu/cpus.c
@@ -91,6 +91,11 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
+int cpu_is_stopped(CPUState *env)
+{
+    return !vm_running || env->stopped;
+}
+
 static void do_vm_stop(int reason)
 {
     if (vm_running) {

[Qemu-devel] [patch uq/master 7/9] move stop/stopped CPU_COMMON fields after area zeroed by reset

[Marcelo Tosatti]

[-- Attachment #1: move-stopped --]
[-- Type: text/plain, Size: 1604 bytes --]

cpu_reset zeroes CPUState upto breakpoints member. Contents of
stop/stopped should not be zeroed on cpu_reset.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/cpu-defs.h
===================================================================
--- qemu.orig/cpu-defs.h
+++ qemu/cpu-defs.h
@@ -159,8 +159,6 @@ typedef struct CPUWatchpoint {
     target_ulong mem_io_vaddr; /* target virtual addr at which the      \
                                      memory was accessed */             \
     uint32_t halted; /* Nonzero if the CPU is in suspend state */       \
-    uint32_t stop;   /* Stop request */                                 \
-    uint32_t stopped; /* Artificially stopped */                        \
     uint32_t interrupt_request;                                         \
     volatile sig_atomic_t exit_request;                                 \
     CPU_COMMON_TLB                                                      \
@@ -203,6 +201,8 @@ typedef struct CPUWatchpoint {
     void *opaque;                                                       \
                                                                         \
     uint32_t created;                                                   \
+    uint32_t stop;   /* Stop request */                                 \
+    uint32_t stopped; /* Artificially stopped */                        \
     struct QemuThread *thread;                                          \
     struct QemuCond *halt_cond;                                         \
     struct qemu_work_item *queued_work_first, *queued_work_last;        \

[Qemu-devel] [patch uq/master 8/9] kvm: validate context for kvm cpu get/put operations

[Marcelo Tosatti]

[-- Attachment #1: assert-get-regs --]
[-- Type: text/plain, Size: 778 bytes --]

From: Jan Kiszka <jan.kiszka@siemens.com>

Validate that KVM vcpu state is only read/written from cpu thread itself
or that cpu is stopped.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/target-i386/kvm.c
===================================================================
--- qemu.orig/target-i386/kvm.c
+++ qemu/target-i386/kvm.c
@@ -949,6 +949,8 @@ int kvm_arch_put_registers(CPUState *env
 {
     int ret;
 
+    assert(cpu_is_stopped(env) || qemu_cpu_self(env));
+
     ret = kvm_getput_regs(env, 1);
     if (ret < 0)
         return ret;
@@ -991,6 +993,8 @@ int kvm_arch_get_registers(CPUState *env
 {
     int ret;
 
+    assert(cpu_is_stopped(env) || qemu_cpu_self(env));
+
     ret = kvm_getput_regs(env, 0);
     if (ret < 0)
         return ret;

[Qemu-devel] [patch uq/master 9/9] kvm: enable smp > 1

[Marcelo Tosatti]

[-- Attachment #1: enable-smp --]
[-- Type: text/plain, Size: 2883 bytes --]

Process INIT/SIPI requests and enable -smp > 1.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

Index: qemu/kvm-all.c
===================================================================
--- qemu.orig/kvm-all.c
+++ qemu/kvm-all.c
@@ -593,11 +593,6 @@ int kvm_init(int smp_cpus)
     int ret;
     int i;
 
-    if (smp_cpus > 1) {
-        fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
-        return -EINVAL;
-    }
-
     s = qemu_mallocz(sizeof(KVMState));
 
 #ifdef KVM_CAP_SET_GUEST_DEBUG
@@ -840,6 +835,11 @@ int kvm_cpu_exec(CPUState *env)
         }
 #endif
 
+        if (kvm_arch_process_irqchip_events(env)) {
+            ret = 0;
+            break;
+        }
+
         if (env->kvm_vcpu_dirty) {
             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
             env->kvm_vcpu_dirty = 0;
Index: qemu/kvm.h
===================================================================
--- qemu.orig/kvm.h
+++ qemu/kvm.h
@@ -90,6 +90,8 @@ int kvm_arch_handle_exit(CPUState *env, 
 
 int kvm_arch_pre_run(CPUState *env, struct kvm_run *run);
 
+int kvm_arch_process_irqchip_events(CPUState *env);
+
 int kvm_arch_get_registers(CPUState *env);
 
 /* state subset only touched by the VCPU itself during runtime */
Index: qemu/target-i386/kvm.c
===================================================================
--- qemu.orig/target-i386/kvm.c
+++ qemu/target-i386/kvm.c
@@ -1073,6 +1073,22 @@ int kvm_arch_post_run(CPUState *env, str
     return 0;
 }
 
+int kvm_arch_process_irqchip_events(CPUState *env)
+{
+    if (env->interrupt_request & CPU_INTERRUPT_INIT) {
+        kvm_cpu_synchronize_state(env);
+        do_cpu_init(env);
+        env->exception_index = EXCP_HALTED;
+    }
+
+    if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
+        kvm_cpu_synchronize_state(env);
+        do_cpu_sipi(env);
+    }
+
+    return env->halted;
+}
+
 static int kvm_handle_halt(CPUState *env)
 {
     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
Index: qemu/target-ppc/kvm.c
===================================================================
--- qemu.orig/target-ppc/kvm.c
+++ qemu/target-ppc/kvm.c
@@ -224,6 +224,11 @@ int kvm_arch_post_run(CPUState *env, str
     return 0;
 }
 
+int kvm_arch_process_irqchip_events(CPUState *env)
+{
+    return 0;
+}
+
 static int kvmppc_handle_halt(CPUState *env)
 {
     if (!(env->interrupt_request & CPU_INTERRUPT_HARD) && (msr_ee)) {
Index: qemu/target-s390x/kvm.c
===================================================================
--- qemu.orig/target-s390x/kvm.c
+++ qemu/target-s390x/kvm.c
@@ -175,6 +175,11 @@ int kvm_arch_post_run(CPUState *env, str
     return 0;
 }
 
+int kvm_arch_process_irqchip_events(CPUState *env)
+{
+    return 0;
+}
+
 static void kvm_s390_interrupt_internal(CPUState *env, int type, uint32_t parm,
                                         uint64_t parm64, int vm)
 {

[Qemu-devel] Re: [patch uq/master 0/9] enable smp > 1 and related fixes

[Anthony Liguori]

On 05/04/2010 07:45 AM, Marcelo Tosatti wrote:
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>    

How does this work without an in-kernel apic (or does uq/master already 
have an in-kernel apic)?

Regards,

Anthony Liguori

[Qemu-devel] Re: [patch uq/master 0/9] enable smp > 1 and related fixes

[Avi Kivity]

On 05/05/2010 09:24 PM, Anthony Liguori wrote:
> On 05/04/2010 07:45 AM, Marcelo Tosatti wrote:
>> -- 
>> To unsubscribe from this list: send the line "unsubscribe kvm" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
> How does this work without an in-kernel apic (or does uq/master 
> already have an in-kernel apic)?

An in-kernel apic isn't needed (qemu-kvm supports smp with userspace apic).

-- 
error compiling committee.c: too many arguments to function

[Qemu-devel] Re: [patch uq/master 0/9] enable smp > 1 and related fixes

[Avi Kivity]

On 05/04/2010 03:45 PM, Marcelo Tosatti wrote:

Applied all, thanks.

-- 
error compiling committee.c: too many arguments to function