[Qemu-devel] [PATCH 0/3] s390x: some further patches

[Qemu-devel] [PATCH 0/3] s390x: some further patches

[Cornelia Huck]

Here are some further s390x patches I had still lying around:
Improve interrupt injection for vcpu interrupts, move fpu regs
from the vmstate to be able to extend it with vector registers
in the future and enable migration of vcpu interrupt state.

Patchset is on top of the virtio patches I sent out on monday
(<1430753300-25739-1-git-send-email-cornelia.huck@de.ibm.com>).
Relevant git branch:

git://github.com/cohuck/qemu s390-next

I'd like to do a pull request for all of these patches by end of
the week.

David Hildenbrand (1):
  s390x: move fpu regs into a subsection of the vmstate

Jens Freimann (2):
  s390x/kvm: use ioctl KVM_S390_IRQ for vcpu interrupts
  s390x/kvm: migrate vcpu interrupt state

 target-s390x/cpu-qom.h |  3 ++
 target-s390x/cpu.c     |  1 +
 target-s390x/cpu.h     |  9 ++++++
 target-s390x/kvm.c     | 77 ++++++++++++++++++++++++++++++++++++++++++++++++--
 target-s390x/machine.c | 49 +++++++++++++++++++++++++++-----
 5 files changed, 130 insertions(+), 9 deletions(-)

-- 
2.4.0

[Qemu-devel] [PATCH 1/3] s390x/kvm: use ioctl KVM_S390_IRQ for vcpu interrupts

[Cornelia Huck]

From: Jens Freimann <jfrei@linux.vnet.ibm.com>

KVM_S390_INT uses only two parameter fields. This is not
enough to pass all required information for certain interrupts.

A new ioctl KVM_S390_IRQ is available which allows us to
inject all local interrupts as defined in the Principles of
Operation. It takes a struct kvm_s390_irq as a parameter
which can store interrupt payload data for all interrupts.

Let's use the new ioctl for injecting vcpu interrupts.

Tested-by: Thomas Huth <thuth@linux.vnet.ibm.com>
Reviewed-by: Thomas Huth <thuth@linux.vnet.ibm.com>
Reviewed-by: David Hildenbrand <dahi@linux.vnet.ibm.com>
Signed-off-by: Jens Freimann <jfrei@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 target-s390x/kvm.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 8e65e43..43ad009 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -124,6 +124,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 static int cap_sync_regs;
 static int cap_async_pf;
 static int cap_mem_op;
+static int cap_s390_irq;
 
 static void *legacy_s390_alloc(size_t size, uint64_t *align);
 
@@ -249,6 +250,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
     cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
+    cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
 
     kvm_s390_enable_cmma(s);
 
@@ -827,10 +829,9 @@ static int s390_kvm_irq_to_interrupt(struct kvm_s390_irq *irq,
     return r;
 }
 
-void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
+static void inject_vcpu_irq_legacy(CPUState *cs, struct kvm_s390_irq *irq)
 {
     struct kvm_s390_interrupt kvmint = {};
-    CPUState *cs = CPU(cpu);
     int r;
 
     r = s390_kvm_irq_to_interrupt(irq, &kvmint);
@@ -846,6 +847,23 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
     }
 }
 
+void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
+{
+    CPUState *cs = CPU(cpu);
+    int r;
+
+    if (cap_s390_irq) {
+        r = kvm_vcpu_ioctl(cs, KVM_S390_IRQ, irq);
+        if (!r) {
+            return;
+        }
+        error_report("KVM failed to inject interrupt %llx", irq->type);
+        exit(1);
+    }
+
+    inject_vcpu_irq_legacy(cs, irq);
+}
+
 static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
 {
     struct kvm_s390_interrupt kvmint = {};
-- 
2.4.0

[Qemu-devel] [PATCH 2/3] s390x: move fpu regs into a subsection of the vmstate

[Cornelia Huck]

From: David Hildenbrand <dahi@linux.vnet.ibm.com>

Let's move the floating point registers into a seperate subsection and
bump up the version id. This cleans up the current vmstate and will
allow for a future extension with vector registers in a compatible way.

This patch is based on a patch from Eric Farman.

Reviewed-by: Eric Farman <farman@linux.vnet.ibm.com>
Signed-off-by: David Hildenbrand <dahi@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 target-s390x/machine.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/target-s390x/machine.c b/target-s390x/machine.c
index bd4cea7..a034423 100644
--- a/target-s390x/machine.c
+++ b/target-s390x/machine.c
@@ -33,12 +33,11 @@ static int cpu_post_load(void *opaque, int version_id)
     return 0;
 }
 
-const VMStateDescription vmstate_s390_cpu = {
-    .name = "cpu",
-    .post_load = cpu_post_load,
-    .version_id = 2,
-    .minimum_version_id = 2,
-    .fields      = (VMStateField[]) {
+const VMStateDescription vmstate_fpu = {
+    .name = "cpu/fpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.fregs[0].ll, S390CPU),
         VMSTATE_UINT64(env.fregs[1].ll, S390CPU),
         VMSTATE_UINT64(env.fregs[2].ll, S390CPU),
@@ -55,11 +54,26 @@ const VMStateDescription vmstate_s390_cpu = {
         VMSTATE_UINT64(env.fregs[13].ll, S390CPU),
         VMSTATE_UINT64(env.fregs[14].ll, S390CPU),
         VMSTATE_UINT64(env.fregs[15].ll, S390CPU),
+        VMSTATE_UINT32(env.fpc, S390CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static inline bool fpu_needed(void *opaque)
+{
+    return true;
+}
+
+const VMStateDescription vmstate_s390_cpu = {
+    .name = "cpu",
+    .post_load = cpu_post_load,
+    .version_id = 3,
+    .minimum_version_id = 3,
+    .fields      = (VMStateField[]) {
         VMSTATE_UINT64_ARRAY(env.regs, S390CPU, 16),
         VMSTATE_UINT64(env.psw.mask, S390CPU),
         VMSTATE_UINT64(env.psw.addr, S390CPU),
         VMSTATE_UINT64(env.psa, S390CPU),
-        VMSTATE_UINT32(env.fpc, S390CPU),
         VMSTATE_UINT32(env.todpr, S390CPU),
         VMSTATE_UINT64(env.pfault_token, S390CPU),
         VMSTATE_UINT64(env.pfault_compare, S390CPU),
@@ -74,4 +88,12 @@ const VMStateDescription vmstate_s390_cpu = {
         VMSTATE_UINT8(env.sigp_order, S390CPU),
         VMSTATE_END_OF_LIST()
      },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_fpu,
+            .needed = fpu_needed,
+        } , {
+            /* empty */
+        }
+    },
 };
-- 
2.4.0

[Qemu-devel] [PATCH 3/3] s390x/kvm: migrate vcpu interrupt state

[Cornelia Huck]

From: Jens Freimann <jfrei@linux.vnet.ibm.com>

This patch adds support to migrate vcpu interrupts.
We use ioctl KVM_S390_GET_IRQ_STATE and _SET_IRQ_STATE
to get/set the complete interrupt state for a vcpu.

Reviewed-by: David Hildenbrand <dahi@linux.vnet.ibm.com>
Signed-off-by: Jens Freimann <jfrei@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 target-s390x/cpu-qom.h |  3 +++
 target-s390x/cpu.c     |  1 +
 target-s390x/cpu.h     |  9 +++++++++
 target-s390x/kvm.c     | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++
 target-s390x/machine.c | 15 +++++++++++++-
 5 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index 8b376df..936ae21 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -66,6 +66,9 @@ typedef struct S390CPU {
     /*< public >*/
 
     CPUS390XState env;
+    /* needed for live migration */
+    void *irqstate;
+    uint32_t irqstate_saved_size;
 } S390CPU;
 
 static inline S390CPU *s390_env_get_cpu(CPUS390XState *env)
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index e0537fa..d2f9836 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -213,6 +213,7 @@ static void s390_cpu_finalize(Object *obj)
     S390CPU *cpu = S390_CPU(obj);
 
     qemu_unregister_reset(s390_cpu_machine_reset_cb, cpu);
+    g_free(cpu->irqstate);
 #endif
 }
 
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index ba7d250..c557211 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -1079,6 +1079,8 @@ void kvm_s390_clear_cmma_callback(void *opaque);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_reset_vcpu(S390CPU *cpu);
 int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit);
+void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
+int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
@@ -1121,6 +1123,13 @@ static inline int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit,
 {
     return 0;
 }
+static inline void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu)
+{
+}
+static inline int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
+{
+    return 0;
+}
 #endif
 
 static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 43ad009..aba1265 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -109,6 +109,14 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_IO                         0x40
 
+#define NR_LOCAL_IRQS 32
+/*
+ * Needs to be big enough to contain max_cpus emergency signals
+ * and in addition NR_LOCAL_IRQS interrupts
+ */
+#define VCPU_IRQ_BUF_SIZE (sizeof(struct kvm_s390_irq) * \
+                           (max_cpus + NR_LOCAL_IRQS))
+
 static CPUWatchpoint hw_watchpoint;
 /*
  * We don't use a list because this structure is also used to transmit the
@@ -274,6 +282,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
 {
     S390CPU *cpu = S390_CPU(cs);
     kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
+    cpu->irqstate = g_malloc0(VCPU_IRQ_BUF_SIZE);
     return 0;
 }
 
@@ -2059,6 +2068,52 @@ int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
     return ret;
 }
 
+void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu)
+{
+    struct kvm_s390_irq_state irq_state;
+    CPUState *cs = CPU(cpu);
+    int32_t bytes;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_S390_IRQ_STATE)) {
+        return;
+    }
+
+    irq_state.buf = (uint64_t) cpu->irqstate;
+    irq_state.len = VCPU_IRQ_BUF_SIZE;
+
+    bytes = kvm_vcpu_ioctl(cs, KVM_S390_GET_IRQ_STATE, &irq_state);
+    if (bytes < 0) {
+        cpu->irqstate_saved_size = 0;
+        error_report("Migration of interrupt state failed");
+        return;
+    }
+
+    cpu->irqstate_saved_size = bytes;
+}
+
+int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    struct kvm_s390_irq_state irq_state;
+    int r;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_S390_IRQ_STATE)) {
+        return -ENOSYS;
+    }
+
+    if (cpu->irqstate_saved_size == 0) {
+        return 0;
+    }
+    irq_state.buf = (uint64_t) cpu->irqstate;
+    irq_state.len = cpu->irqstate_saved_size;
+
+    r = kvm_vcpu_ioctl(cs, KVM_S390_SET_IRQ_STATE, &irq_state);
+    if (r) {
+        error_report("Setting interrupt state failed %d", r);
+    }
+    return r;
+}
+
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
                               uint64_t address, uint32_t data)
 {
diff --git a/target-s390x/machine.c b/target-s390x/machine.c
index a034423..7853e3c 100644
--- a/target-s390x/machine.c
+++ b/target-s390x/machine.c
@@ -28,10 +28,19 @@ static int cpu_post_load(void *opaque, int version_id)
      */
     if (kvm_enabled()) {
         kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
+        return kvm_s390_vcpu_interrupt_post_load(cpu);
     }
 
     return 0;
 }
+static void cpu_pre_save(void *opaque)
+{
+    S390CPU *cpu = opaque;
+
+    if (kvm_enabled()) {
+        kvm_s390_vcpu_interrupt_pre_save(cpu);
+    }
+}
 
 const VMStateDescription vmstate_fpu = {
     .name = "cpu/fpu",
@@ -67,7 +76,8 @@ static inline bool fpu_needed(void *opaque)
 const VMStateDescription vmstate_s390_cpu = {
     .name = "cpu",
     .post_load = cpu_post_load,
-    .version_id = 3,
+    .pre_save = cpu_pre_save,
+    .version_id = 4,
     .minimum_version_id = 3,
     .fields      = (VMStateField[]) {
         VMSTATE_UINT64_ARRAY(env.regs, S390CPU, 16),
@@ -86,6 +96,9 @@ const VMStateDescription vmstate_s390_cpu = {
         VMSTATE_UINT64_ARRAY(env.cregs, S390CPU, 16),
         VMSTATE_UINT8(env.cpu_state, S390CPU),
         VMSTATE_UINT8(env.sigp_order, S390CPU),
+        VMSTATE_UINT32_V(irqstate_saved_size, S390CPU, 4),
+        VMSTATE_VBUFFER_UINT32(irqstate, S390CPU, 4, NULL, 0,
+                               irqstate_saved_size),
         VMSTATE_END_OF_LIST()
      },
     .subsections = (VMStateSubsection[]) {
-- 
2.4.0