[Qemu-devel] [PULL 03/22] s390x/async_pf: Check for apf extension and enable pfault

[Christian Borntraeger <borntraeger@de.ibm.com>]

From: Dominik Dingel <dingel@linux.vnet.ibm.com>

S390 can also use async page faults, to enhance guest scheduling.
In case of live migration we want to disable the feature and let
all pending request finish.

Signed-off-by: Dominik Dingel <dingel@linux.vnet.ibm.com>
Signed-off-by: Jens Freimann <jfrei@linux.vnet.ibm.com>
Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 hw/intc/s390_flic.c | 36 ++++++++++++++++++++++++++++++++++++
 target-s390x/cpu.c  |  6 ++++++
 target-s390x/cpu.h  |  4 ++++
 target-s390x/kvm.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 94 insertions(+)

diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
index 1a7876d..b2ef3e3 100644
--- a/hw/intc/s390_flic.c
+++ b/hw/intc/s390_flic.c
@@ -63,6 +63,34 @@ static int flic_get_all_irqs(KVMS390FLICState *flic,
     return rc == -1 ? -errno : rc;
 }
 
+static void flic_enable_pfault(KVMS390FLICState *flic)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_APF_ENABLE,
+    };
+    int rc;
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    if (rc) {
+        fprintf(stderr, "flic: couldn't enable pfault\n");
+    }
+}
+
+static void flic_disable_wait_pfault(KVMS390FLICState *flic)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_APF_DISABLE_WAIT,
+    };
+    int rc;
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    if (rc) {
+        fprintf(stderr, "flic: couldn't disable pfault\n");
+    }
+}
+
 /** flic_enqueue_irqs - returns 0 on success
  * @buf: pointer to buffer which is passed to kernel
  * @len: length of buffer
@@ -136,6 +164,8 @@ static void kvm_flic_save(QEMUFile *f, void *opaque)
     void *buf;
     int count;
 
+    flic_disable_wait_pfault((struct KVMS390FLICState *) opaque);
+
     buf = g_try_malloc0(len);
     if (!buf) {
         /* Storing FLIC_FAILED into the count field here will cause the
@@ -184,6 +214,8 @@ static int kvm_flic_load(QEMUFile *f, void *opaque, int version_id)
         goto out;
     }
 
+    flic_enable_pfault((struct KVMS390FLICState *) opaque);
+
     count = qemu_get_be64(f);
     len = count * sizeof(struct kvm_s390_irq);
     if (count == FLIC_FAILED) {
@@ -256,10 +288,14 @@ static void kvm_s390_flic_reset(DeviceState *dev)
         return;
     }
 
+    flic_disable_wait_pfault(flic);
+
     rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
     if (rc) {
         trace_flic_reset_failed(errno);
     }
+
+    flic_enable_pfault(flic);
 }
 
 static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index ff57b80..f1319e5 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -83,6 +83,7 @@ static void s390_cpu_reset(CPUState *s)
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
     CPUS390XState *env = &cpu->env;
 
+    env->pfault_token = -1UL;
     s390_del_running_cpu(cpu);
     scc->parent_reset(s);
 #if !defined(CONFIG_USER_ONLY)
@@ -105,6 +106,8 @@ static void s390_cpu_initial_reset(CPUState *s)
     /* architectured initial values for CR 0 and 14 */
     env->cregs[0] = CR0_RESET;
     env->cregs[14] = CR14_RESET;
+
+    env->pfault_token = -1UL;
 }
 
 /* CPUClass:reset() */
@@ -123,6 +126,9 @@ static void s390_cpu_full_reset(CPUState *s)
     /* architectured initial values for CR 0 and 14 */
     env->cregs[0] = CR0_RESET;
     env->cregs[14] = CR14_RESET;
+
+    env->pfault_token = -1UL;
+
     /* set halted to 1 to make sure we can add the cpu in
      * s390_ipl_cpu code, where CPUState::halted is set back to 0
      * after incrementing the cpu counter */
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 96c2b4a..b09ff92 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -121,6 +121,10 @@ typedef struct CPUS390XState {
     uint64_t cputm;
     uint32_t todpr;
 
+    uint64_t pfault_token;
+    uint64_t pfault_compare;
+    uint64_t pfault_select;
+
     CPU_COMMON
 
     /* reset does memset(0) up to here */
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index f60ccdc..9430a35 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -87,12 +87,14 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 };
 
 static int cap_sync_regs;
+static int cap_async_pf;
 
 static void *legacy_s390_alloc(size_t size);
 
 int kvm_arch_init(KVMState *s)
 {
     cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
+    cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
         phys_mem_set_alloc(legacy_s390_alloc);
@@ -178,6 +180,29 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         return ret;
     }
 
+    if (cap_async_pf) {
+        reg.id = KVM_REG_S390_PFTOKEN;
+        reg.addr = (__u64)&(env->pfault_token);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret < 0) {
+            return ret;
+        }
+
+        reg.id = KVM_REG_S390_PFCOMPARE;
+        reg.addr = (__u64)&(env->pfault_compare);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret < 0) {
+            return ret;
+        }
+
+        reg.id = KVM_REG_S390_PFSELECT;
+        reg.addr = (__u64)&(env->pfault_select);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
     if (cap_sync_regs &&
         cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
         cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
@@ -282,6 +307,29 @@ int kvm_arch_get_registers(CPUState *cs)
         return r;
     }
 
+    if (cap_async_pf) {
+        reg.id = KVM_REG_S390_PFTOKEN;
+        reg.addr = (__u64)&(env->pfault_token);
+        r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+        if (r < 0) {
+            return r;
+        }
+
+        reg.id = KVM_REG_S390_PFCOMPARE;
+        reg.addr = (__u64)&(env->pfault_compare);
+        r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+        if (r < 0) {
+            return r;
+        }
+
+        reg.id = KVM_REG_S390_PFSELECT;
+        reg.addr = (__u64)&(env->pfault_select);
+        r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+        if (r < 0) {
+            return r;
+        }
+    }
+
     return 0;
 }
 
-- 
1.8.4.2