[patch 3/5] KVM: hypercall batching (v2)

[Marcelo Tosatti <mtosatti@redhat.com>]

[-- Attachment #1: kvm-multicall --]
[-- Type: text/plain, Size: 10253 bytes --]

Batch pte updates and tlb flushes in lazy MMU mode.

v1->v2:
- report individual hypercall error code, have multicall return number of 
processed entries.
- cover entire multicall duration with slots_lock instead of 
acquiring/reacquiring.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Anthony Liguori <aliguori@us.ibm.com>

Index: kvm.paravirt2/arch/x86/kernel/kvm.c
===================================================================
--- kvm.paravirt2.orig/arch/x86/kernel/kvm.c
+++ kvm.paravirt2/arch/x86/kernel/kvm.c
@@ -25,6 +25,77 @@
 #include <linux/kvm_para.h>
 #include <linux/cpu.h>
 #include <linux/mm.h>
+#include <linux/hardirq.h>
+
+#define MAX_MULTICALL_NR (PAGE_SIZE / sizeof(struct kvm_multicall_entry))
+
+struct kvm_para_state {
+	struct kvm_multicall_entry queue[MAX_MULTICALL_NR];
+	int queue_index;
+	enum paravirt_lazy_mode mode;
+};
+
+static DEFINE_PER_CPU(struct kvm_para_state, para_state);
+
+static int can_defer_hypercall(struct kvm_para_state *state, unsigned int nr)
+{
+	if (state->mode == PARAVIRT_LAZY_MMU) {
+		switch (nr) {
+		case KVM_HYPERCALL_MMU_WRITE:
+		case KVM_HYPERCALL_FLUSH_TLB:
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static void hypercall_queue_flush(struct kvm_para_state *state)
+{
+	long ret;
+
+	if (state->queue_index) {
+		ret = kvm_hypercall2(KVM_HYPERCALL_MULTICALL,
+				     __pa(&state->queue), state->queue_index);
+		WARN_ON (ret != state->queue_index);
+		state->queue_index = 0;
+	}
+}
+
+static void kvm_hypercall_defer(struct kvm_para_state *state,
+				unsigned int nr,
+				unsigned long a0, unsigned long a1,
+				unsigned long a2, unsigned long a3)
+{
+	struct kvm_multicall_entry *entry;
+
+	BUG_ON(preemptible());
+
+	if (state->queue_index == MAX_MULTICALL_NR)
+		hypercall_queue_flush(state);
+
+	entry = &state->queue[state->queue_index++];
+	entry->nr = nr;
+	entry->a0 = a0;
+	entry->a1 = a1;
+	entry->a2 = a2;
+	entry->a3 = a3;
+}
+
+static long kvm_hypercall(unsigned int nr, unsigned long a0,
+			  unsigned long a1, unsigned long a2,
+			  unsigned long a3)
+{
+	struct kvm_para_state *state = &get_cpu_var(para_state);
+	long ret = 0;
+
+	if (can_defer_hypercall(state, nr))
+		kvm_hypercall_defer(state, nr, a0, a1, a2, a3);
+	else
+		ret = kvm_hypercall4(nr, a0, a1, a2, a3);
+
+	put_cpu_var(para_state);
+	return ret;
+}
 
 /*
  * No need for any "IO delay" on KVM
@@ -44,8 +115,8 @@ static void kvm_mmu_write(void *dest, co
 	if (size == 2)
 		a1 = *(u32 *)&p[4];
 #endif
-	kvm_hypercall3(KVM_HYPERCALL_MMU_WRITE, (unsigned long)__pa(dest), a0,
-			a1);
+	kvm_hypercall(KVM_HYPERCALL_MMU_WRITE, (unsigned long)__pa(dest), a0,
+			a1, 0);
 }
 
 /*
@@ -110,12 +181,31 @@ static void kvm_set_pud(pud_t *pudp, pud
 
 static void kvm_flush_tlb(void)
 {
-	kvm_hypercall0(KVM_HYPERCALL_FLUSH_TLB);
+	kvm_hypercall(KVM_HYPERCALL_FLUSH_TLB, 0, 0, 0, 0);
 }
 
 static void kvm_release_pt(u32 pfn)
 {
-	kvm_hypercall1(KVM_HYPERCALL_RELEASE_PT, pfn << PAGE_SHIFT);
+	kvm_hypercall(KVM_HYPERCALL_RELEASE_PT, pfn << PAGE_SHIFT, 0, 0, 0);
+}
+
+static void kvm_enter_lazy_mmu(void)
+{
+	struct kvm_para_state *state
+		= &per_cpu(para_state, smp_processor_id());
+
+	paravirt_enter_lazy_mmu();
+	state->mode = paravirt_get_lazy_mode();
+}
+
+static void kvm_leave_lazy_mmu(void)
+{
+	struct kvm_para_state *state
+		= &per_cpu(para_state, smp_processor_id());
+
+	hypercall_queue_flush(state);
+	paravirt_leave_lazy(paravirt_get_lazy_mode());
+	state->mode = paravirt_get_lazy_mode();
 }
 
 static void paravirt_ops_setup(void)
@@ -144,6 +234,11 @@ static void paravirt_ops_setup(void)
 		pv_mmu_ops.release_pt = kvm_release_pt;
 		pv_mmu_ops.release_pd = kvm_release_pt;
 	}
+
+	if (kvm_para_has_feature(KVM_FEATURE_MULTICALL)) {
+		pv_mmu_ops.lazy_mode.enter = kvm_enter_lazy_mmu;
+		pv_mmu_ops.lazy_mode.leave = kvm_leave_lazy_mmu;
+	}
 }
 
 void __init kvm_guest_init(void)
Index: kvm.paravirt2/arch/x86/kvm/x86.c
===================================================================
--- kvm.paravirt2.orig/arch/x86/kvm/x86.c
+++ kvm.paravirt2/arch/x86/kvm/x86.c
@@ -79,6 +79,8 @@ struct kvm_stats_debugfs_item debugfs_en
 	{ "fpu_reload", VCPU_STAT(fpu_reload) },
 	{ "insn_emulation", VCPU_STAT(insn_emulation) },
 	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
+	{ "multicall", VCPU_STAT(multicall) },
+	{ "multicall_nr", VCPU_STAT(multicall_nr) },
 	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
 	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
 	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
@@ -856,8 +858,10 @@ long kvm_arch_dev_ioctl(struct file *fil
 	}
 	case KVM_GET_PARA_FEATURES: {
 		__u32 para_features = KVM_PARA_FEATURES;
-		if (tdp_enabled)
+		if (tdp_enabled) {
 			para_features &= ~(1UL << KVM_FEATURE_MMU_WRITE);
+			para_features &= ~(1UL << KVM_FEATURE_MULTICALL);
+		}
 
 		r = -EFAULT;
 		if (copy_to_user(argp, &para_features, sizeof para_features))
@@ -1759,22 +1763,31 @@ mmio:
 	return X86EMUL_UNHANDLEABLE;
 }
 
-static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
+static int __emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 			       const void *val, int bytes)
 {
 	int ret;
 
-	down_read(&vcpu->kvm->slots_lock);
 	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
-	if (ret < 0) {
-		up_read(&vcpu->kvm->slots_lock);
+	if (ret < 0)
 		return 0;
-	}
+
 	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
-	up_read(&vcpu->kvm->slots_lock);
 	return 1;
 }
 
+static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
+			       const void *val, int bytes)
+{
+	int ret;
+
+	down_read(&vcpu->kvm->slots_lock);
+	ret = __emulator_write_phys(vcpu, gpa, val, bytes);
+	up_read(&vcpu->kvm->slots_lock);
+	return ret;
+}
+
+
 static int emulator_write_emulated_onepage(unsigned long addr,
 					   const void *val,
 					   unsigned int bytes,
@@ -2403,6 +2416,61 @@ static int kvm_hypercall_release_pt(stru
 	return 0;
 }
 
+static int dispatch_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,
+			      unsigned long a0, unsigned long a1,
+			      unsigned long a2, unsigned long a3)
+{
+	switch (nr) {
+	case KVM_HC_VAPIC_POLL_IRQ:
+		return 0;
+	case KVM_HYPERCALL_MMU_WRITE:
+		return kvm_hypercall_mmu_write(vcpu, a0, a1, a2);
+	case KVM_HYPERCALL_FLUSH_TLB:
+		return kvm_hypercall_flush_tlb(vcpu);
+	case KVM_HYPERCALL_RELEASE_PT:
+		return kvm_hypercall_release_pt(vcpu, a0);
+	}
+
+	return -KVM_ENOSYS;
+}
+
+static int kvm_hypercall_multicall(struct kvm_vcpu *vcpu, gpa_t addr, u32 nents)
+{
+	int i, nr_processed = 0;
+
+	++vcpu->stat.multicall;
+	vcpu->stat.multicall_nr += nents;
+
+	down_read(&vcpu->kvm->slots_lock);
+	for (i = 0; i < nents; i++) {
+		struct kvm_multicall_entry mc;
+		int ret;
+
+		ret = kvm_read_guest(vcpu->kvm, addr, &mc, sizeof(mc));
+		if (ret) {
+			up_read(&vcpu->kvm->slots_lock);
+			return nr_processed;
+		}
+
+		mc.error_code = dispatch_hypercall(vcpu, mc.nr, mc.a0, mc.a1,
+						   mc.a2, mc.a3);
+		if (mc.error_code) {
+			ret = kvm_write_guest(vcpu->kvm, addr, &mc,
+					      sizeof(mc));
+			if (ret) {
+				up_read(&vcpu->kvm->slots_lock);
+				return nr_processed;
+			}
+		} else
+			nr_processed++;
+
+		addr += sizeof(mc);
+	}
+	up_read(&vcpu->kvm->slots_lock);
+
+	return nr_processed;
+}
+
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
@@ -2423,23 +2491,11 @@ int kvm_emulate_hypercall(struct kvm_vcp
 		a3 &= 0xFFFFFFFF;
 	}
 
-	switch (nr) {
-	case KVM_HC_VAPIC_POLL_IRQ:
-		ret = 0;
-		break;
-	case KVM_HYPERCALL_MMU_WRITE:
-		ret = kvm_hypercall_mmu_write(vcpu, a0, a1, a2);
-		break;
-	case KVM_HYPERCALL_FLUSH_TLB:
-		ret = kvm_hypercall_flush_tlb(vcpu);
-		break;
-	case KVM_HYPERCALL_RELEASE_PT:
-		ret = kvm_hypercall_release_pt(vcpu, a0);
-		break;
-	default:
-		ret = -KVM_ENOSYS;
-		break;
-	}
+	if (nr == KVM_HYPERCALL_MULTICALL)
+		ret = kvm_hypercall_multicall(vcpu, a0, a1);
+	else
+		ret = dispatch_hypercall(vcpu, nr, a0, a1, a2, a3);
+
 	vcpu->arch.regs[VCPU_REGS_RAX] = ret;
 	kvm_x86_ops->decache_regs(vcpu);
 	return 0;
Index: kvm.paravirt2/include/asm-x86/kvm_host.h
===================================================================
--- kvm.paravirt2.orig/include/asm-x86/kvm_host.h
+++ kvm.paravirt2/include/asm-x86/kvm_host.h
@@ -327,6 +327,8 @@ struct kvm_vcpu_stat {
 	u32 fpu_reload;
 	u32 insn_emulation;
 	u32 insn_emulation_fail;
+	u32 multicall;
+	u32 multicall_nr;
 };
 
 struct descriptor_table {
Index: kvm.paravirt2/include/asm-x86/kvm_para.h
===================================================================
--- kvm.paravirt2.orig/include/asm-x86/kvm_para.h
+++ kvm.paravirt2/include/asm-x86/kvm_para.h
@@ -13,6 +13,7 @@
 #define KVM_FEATURE_CLOCKSOURCE		0
 #define KVM_FEATURE_NOP_IO_DELAY	1
 #define KVM_FEATURE_MMU_WRITE		2
+#define KVM_FEATURE_MULTICALL		3
 
 #define MSR_KVM_WALL_CLOCK  0x11
 #define MSR_KVM_SYSTEM_TIME 0x12
@@ -37,13 +38,22 @@ struct kvm_wall_clock {
 	uint32_t wc_nsec;
 } __attribute__((__packed__));
 
+struct kvm_multicall_entry {
+	u32 nr;
+	u32 error_code;
+	u64 a0;
+	u64 a1;
+	u64 a2;
+	u64 a3;
+};
 
 extern void kvmclock_init(void);
 
 
 #define KVM_PARA_FEATURES ((1UL << KVM_FEATURE_NOP_IO_DELAY)	|	\
 			   (1UL << KVM_FEATURE_CLOCKSOURCE)	|	\
-			   (1UL << KVM_FEATURE_MMU_WRITE))
+			   (1UL << KVM_FEATURE_MMU_WRITE)	|	\
+			   (1UL << KVM_FEATURE_MULTICALL))
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a
  * trap that we will then rewrite to the appropriate instruction.
Index: kvm.paravirt2/include/linux/kvm_para.h
===================================================================
--- kvm.paravirt2.orig/include/linux/kvm_para.h
+++ kvm.paravirt2/include/linux/kvm_para.h
@@ -13,11 +13,13 @@
 #define KVM_ENOSYS		1000
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
+#define KVM_EINVAL		EINVAL
 
 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HYPERCALL_MMU_WRITE		2
 #define KVM_HYPERCALL_FLUSH_TLB		3
 #define KVM_HYPERCALL_RELEASE_PT	4
+#define KVM_HYPERCALL_MULTICALL		5
 
 /*
  * hypercalls use architecture specific

-- 


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/