[PATCH 0/3] Usermode side patches for in-kernel APIC

[PATCH 0/3] Usermode side patches for in-kernel APIC

[Gregory Haskins]

Here is the patch series for the usermode side.  Note that the last patch is
pretty much experimental, and doesnt appear to be working the way I intended.
Please advise if you know of the right way to do that.  It doesnt seem to have
affected anything that I was able to test so far, so I don't know if it
matters or if there is a HALT race condition lurking in there until the signal
handling is fixed.

In any case, you can apply at least the first two and get "level-1" support
running (in kernel LAPIC, QEMU based 8259/IOAPIC).

Note that I abandoned my original patch that kicked off this whole party where
I majorly re-worked QEMU.  I figured if that new qemu-irq stuff from QEMU
upstream was coming soon we could just go the #if USE_KVM route for now.  We
can make things more elegant when the new infrastructure is in place somewhere
down the road.

As with the kernel side changes, comments/feedback/bug-reports/patches
welcome.

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 1/3] KVM: Updates for compiling in-kernel APIC support with external-modules

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 kernel/Kbuild                   |    2 +-
 kernel/external-module-compat.h |    7 +++++++
 2 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/kernel/Kbuild b/kernel/Kbuild
index e9bcda7..103a179 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -1,5 +1,5 @@
 EXTRA_CFLAGS := -I$(src)/include -include $(src)/external-module-compat.h
 obj-m := kvm.o kvm-intel.o kvm-amd.o
-kvm-objs := kvm_main.o mmu.o x86_emulate.o
+kvm-objs := kvm_main.o mmu.o x86_emulate.o userint.o kernint.o lapic.o
 kvm-intel-objs := vmx.o vmx-debug.o
 kvm-amd-objs := svm.o
diff --git a/kernel/external-module-compat.h b/kernel/external-module-compat.h
index 6ccaadd..2150ca9 100644
--- a/kernel/external-module-compat.h
+++ b/kernel/external-module-compat.h
@@ -135,6 +135,13 @@ static inline struct super_block *kvmfs_get_sb(
 
 #endif
 
+/*
+ * hrtimers are not supported in older kernels (not sure where this starts)
+ */
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,16)
+#define KVM_NO_HRTIMER 1
+#endif
+
 #include <linux/miscdevice.h>
 #ifndef KVM_MINOR
 #define KVM_MINOR 232


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 2/3] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/hw/apic.c  |   17 ++++++++++++++++-
 qemu/hw/pc.c    |   11 +++++++++++
 qemu/qemu-kvm.c |   12 +++++-------
 user/kvmctl.c   |   25 +++++++++++++++++++++++--
 user/kvmctl.h   |    5 +++--
 user/main.c     |    3 +--
 6 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/qemu/hw/apic.c b/qemu/hw/apic.c
index 0b73233..a26b96e 100644
--- a/qemu/hw/apic.c
+++ b/qemu/hw/apic.c
@@ -19,6 +19,11 @@
  */
 #include "vl.h"
 
+#ifdef USE_KVM
+#include "kvmctl.h"
+extern kvm_context_t kvm_context;
+#endif
+
 //#define DEBUG_APIC
 //#define DEBUG_IOAPIC
 
@@ -888,10 +893,20 @@ static void ioapic_service(IOAPICState *s)
                     vector = pic_read_irq(isa_pic);
                 else
                     vector = entry & 0xff;
-                
+             
+#ifdef USE_KVM
+		if (kvm_allowed) {
+		    kvm_apic_bus_deliver(kvm_context, dest, trig_mode,
+					 dest_mode, delivery_mode, vector);
+		} else {
+#endif
                 apic_get_delivery_bitmask(deliver_bitmask, dest, dest_mode);
                 apic_bus_deliver(deliver_bitmask, delivery_mode, 
                                  vector, polarity, trig_mode);
+#ifdef USE_KVM
+		}
+#endif
+
             }
         }
     }
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index eda49cf..ffcbfc6 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -91,6 +91,9 @@ int cpu_get_pic_interrupt(CPUState *env)
 {
     int intno;
 
+#ifdef USE_KVM
+    if (!kvm_allowed) {
+#endif
     intno = apic_get_interrupt(env);
     if (intno >= 0) {
         /* set irq request if a PIC irq is still pending */
@@ -98,10 +101,15 @@ int cpu_get_pic_interrupt(CPUState *env)
         pic_update_irq(isa_pic); 
         return intno;
     }
+
     /* read the irq from the PIC */
     if (!apic_accept_pic_intr(env))
         return -1;
 
+#ifdef USE_KVM
+    }
+#endif
+
     intno = pic_read_irq(isa_pic);
     return intno;
 }
@@ -483,6 +491,9 @@ static void pc_init1(int ram_size, int vga_ram_size, int boot_device,
         }
         register_savevm("cpu", i, 4, cpu_save, cpu_load, env);
         qemu_register_reset(main_cpu_reset, env);
+#ifdef USE_KVM
+	if (!kvm_allowed)
+#endif
         if (pci_enabled) {
             apic_init(env);
         }
diff --git a/qemu/qemu-kvm.c b/qemu/qemu-kvm.c
index 212570a..266c73d 100644
--- a/qemu/qemu-kvm.c
+++ b/qemu/qemu-kvm.c
@@ -227,9 +227,11 @@ static void load_regs(CPUState *env)
     sregs.cr3 = env->cr[3];
     sregs.cr4 = env->cr[4];
 
-    sregs.apic_base = cpu_get_apic_base(env);
+    /* These two are no longer used once the in-kernel APIC is enabled */
+    sregs.apic_base = 0;
+    sregs.cr8 = 0;
+
     sregs.efer = env->efer;
-    sregs.cr8 = cpu_get_apic_tpr(env);
 
     kvm_set_sregs(kvm_context, 0, &sregs);
 
@@ -412,7 +414,7 @@ static int try_push_interrupts(void *opaque)
         (env->eflags & IF_MASK)) {
             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
             // for now using cpu 0
-            kvm_inject_irq(kvm_context, 0, cpu_get_pic_interrupt(env));
+            kvm_inject_isa_irq(kvm_context, cpu_get_pic_interrupt(env));
     }
 
     return (env->interrupt_request & CPU_INTERRUPT_HARD) != 0;
@@ -425,16 +427,12 @@ static void post_kvm_run(void *opaque, struct kvm_run *kvm_run)
 
     env->eflags = (kvm_run->if_flag) ? env->eflags | IF_MASK:env->eflags & ~IF_MASK;
     env->ready_for_interrupt_injection = kvm_run->ready_for_interrupt_injection;
-    //cpu_set_apic_tpr(env, kvm_run->cr8);
-    cpu_set_apic_base(env, kvm_run->apic_base);
 }
 
 static void pre_kvm_run(void *opaque, struct kvm_run *kvm_run)
 {
     CPUState **envs = opaque, *env;
     env = envs[0];
-
-    kvm_run->cr8 = cpu_get_apic_tpr(env);
 }
 
 void kvm_load_registers(CPUState *env)
diff --git a/user/kvmctl.c b/user/kvmctl.c
index dcdffbf..e5f3e3e 100644
--- a/user/kvmctl.c
+++ b/user/kvmctl.c
@@ -233,6 +233,7 @@ int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
 	int fd = kvm->fd;
 	int zfd;
 	int r;
+	const int apic_level = 1;
 	struct kvm_memory_region low_memory = {
 		.slot = 3,
 		.memory_size = memory  < dosmem ? memory : dosmem,
@@ -282,6 +283,12 @@ int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
 	     MAP_PRIVATE|MAP_FIXED, zfd, 0);
 	close(zfd);
 
+	r = ioctl(fd, KVM_ENABLE_KERNEL_PIC, &apic_level);
+	if (r == -1) {
+		fprintf(stderr, "kvm_enable_kernel_pic: %m\n");
+		return -1;
+	}
+
 	r = ioctl(fd, KVM_CREATE_VCPU, 0);
 	if (r == -1) {
 		fprintf(stderr, "kvm_create_vcpu: %m\n");
@@ -772,12 +779,26 @@ more:
 	return r;
 }
 
-int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq)
 {
 	struct kvm_interrupt intr;
 
 	intr.irq = irq;
-	return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
+	return ioctl(kvm->vm_fd, KVM_ISA_INTERRUPT, &intr);
+}
+
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector)
+{
+	struct kvm_apic_msg msg;
+
+	msg.dest          = dest;
+	msg.trig_mode     = trig_mode;
+	msg.dest_mode     = dest_mode;
+	msg.delivery_mode = delivery_mode;
+	msg.vector        = vector;
+
+	return ioctl(kvm->vm_fd, KVM_APIC_MSG, &msg);
 }
 
 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
diff --git a/user/kvmctl.h b/user/kvmctl.h
index 27c010c..2fd5606 100644
--- a/user/kvmctl.h
+++ b/user/kvmctl.h
@@ -236,11 +236,12 @@ int kvm_set_msrs(kvm_context_t, int vcpu, struct kvm_msr_entry *msrs, int n);
  * This allows you to simulate an external vectored interrupt.
  *
  * \param kvm Pointer to the current kvm_context
- * \param vcpu Which virtual CPU should get dumped
  * \param irq Vector number
  * \return 0 on success
  */
-int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq);
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq);
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector);
 int kvm_guest_debug(kvm_context_t, int vcpu, struct kvm_debug_guest *dbg);
 
 /*!
diff --git a/user/main.c b/user/main.c
index 83fd752..34b3ba9 100644
--- a/user/main.c
+++ b/user/main.c
@@ -49,7 +49,7 @@ static int test_outb(void *opaque, uint16_t addr, uint8_t value)
     switch (addr) {
     case 0xff: // irq injector
 	printf("injecting interrupt 0x%x\n", value);
-	kvm_inject_irq(kvm, 0, value);
+	// kvm_inject_irq(kvm, 0, value);
 	break;
     case 0xf1: // serial
 	if (newline)
@@ -116,7 +116,6 @@ static struct kvm_callbacks test_callbacks = {
     .debug       = test_debug,
     .halt        = test_halt,
     .io_window = test_io_window,
-    .try_push_interrupts = test_try_push_interrupts,
     .post_kvm_run = test_post_kvm_run,
     .pre_kvm_run = test_pre_kvm_run,
 };


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 3/3] KVM-USER: Add support for in-kernel interrupts to wake-up QEMU

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/vl.c |   26 ++++++++++++++++++++++++++
 1 files changed, 26 insertions(+), 0 deletions(-)

diff --git a/qemu/vl.c b/qemu/vl.c
index 7df1c80..de4d342 100644
--- a/qemu/vl.c
+++ b/qemu/vl.c
@@ -1126,6 +1126,28 @@ void quit_timers(void)
 #endif
 }
 
+#ifdef USE_KVM
+
+static void kvm_interrupt_handler(int host_signum)
+{    
+    printf("kvm interrupt!\n");
+    CPUState *env = cpu_single_env;
+    if (env)
+	cpu_interrupt(env, CPU_INTERRUPT_EXIT);
+}
+
+void init_kvm_interrupts(void)
+{
+    struct sigaction act;
+    
+    act.sa_flags = 0;
+    act.sa_handler = kvm_interrupt_handler;
+    sigaction(33, &act, NULL);
+
+}
+
+#endif /* USE_KVM */
+
 /***********************************************************/
 /* character device */
 
@@ -7444,6 +7466,10 @@ int main(int argc, char **argv)
     init_timers();
     init_timer_alarm();
     qemu_aio_init();
+#ifdef USE_KVM
+    if (kvm_allowed)
+	init_kvm_interrupts();
+#endif
 
 #ifdef _WIN32
     socket_init();


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 0/3] Usermode side patches for in-kernel APIC

[Avi Kivity]

Gregory Haskins wrote:
> Here is the patch series for the usermode side.  Note that the last patch is
> pretty much experimental, and doesnt appear to be working the way I intended.
> Please advise if you know of the right way to do that.  It doesnt seem to have
> affected anything that I was able to test so far, so I don't know if it
> matters or if there is a HALT race condition lurking in there until the signal
> handling is fixed.
>
> In any case, you can apply at least the first two and get "level-1" support
> running (in kernel LAPIC, QEMU based 8259/IOAPIC).
>
> Note that I abandoned my original patch that kicked off this whole party where
> I majorly re-worked QEMU.  I figured if that new qemu-irq stuff from QEMU
> upstream was coming soon we could just go the #if USE_KVM route for now.  We
> can make things more elegant when the new infrastructure is in place somewhere
> down the road.
>
> As with the kernel side changes, comments/feedback/bug-reports/patches
> welcome.
>
>   

These are fine.  Of course, new userspace has to work with 2.6.22, so 
you need to check feature availability.

-- 
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 0/3] Usermode side patches for in-kernel APIC

[Gregory Haskins]

>>> On Mon, May 7, 2007 at  6:53 AM, in message <463F0544.5070609-atKUWr5tajBWk0Htik3J/w@public.gmane.org>,
Avi Kivity <avi-atKUWr5tajBWk0Htik3J/w@public.gmane.org> wrote: 
> Gregory Haskins wrote:
>> Here is the patch series for the usermode side.  Note that the last patch is
>> pretty much experimental, and doesnt appear to be working the way I 
> intended.
>> Please advise if you know of the right way to do that.  It doesnt seem to 
> have
>> affected anything that I was able to test so far, so I don't know if it
>> matters or if there is a HALT race condition lurking in there until the 
> signal
>> handling is fixed.
>>
>> In any case, you can apply at least the first two and get "level- 1" support
>> running (in kernel LAPIC, QEMU based 8259/IOAPIC).
>>
>> Note that I abandoned my original patch that kicked off this whole party 
> where
>> I majorly re- worked QEMU.  I figured if that new qemu- irq stuff from QEMU
>> upstream was coming soon we could just go the #if USE_KVM route for now.  We
>> can make things more elegant when the new infrastructure is in place 
> somewhere
>> down the road.
>>
>> As with the kernel side changes, comments/feedback/bug- reports/patches
>> welcome.
>>
>>   
> 
> These are fine.  Of course, new userspace has to work with 2.6.22, so 
> you need to check feature availability.


Agreed.  I also need to make the code be able to dynamically disable the new stuff.   Ill hopefully get an update for this soon.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 2/3] KVM: in-kernel-apic modification to QEMU

[Dor Laor]

> qemu/hw/apic.c  |   17 ++++++++++++++++-
> qemu/hw/pc.c    |   11 +++++++++++
> qemu/qemu-kvm.c |   12 +++++-------
> user/kvmctl.c   |   25 +++++++++++++++++++++++--
> user/kvmctl.h   |    5 +++--
> user/main.c     |    3 +--
> 6 files changed, 59 insertions(+), 14 deletions(-)
>
>diff --git a/qemu/hw/apic.c b/qemu/hw/apic.c
>@@ -888,10 +893,20 @@ static void ioapic_service(IOAPICState *s)
>                     vector = pic_read_irq(isa_pic);
>                 else
>                     vector = entry & 0xff;
>-
>+
>+#ifdef USE_KVM
>+		if (kvm_allowed) {

For having the option of using kvm with qemu-apic you can use the
apic_level variable below and add it to qemu's configuration params.
This will be valuable while testing your implementation.



>+		    kvm_apic_bus_deliver(kvm_context, dest, trig_mode,
>+					 dest_mode, delivery_mode,
vector);
>+		} else {
>+#endif
>                 apic_get_delivery_bitmask(deliver_bitmask, dest,
>dest_mode);
>                 apic_bus_deliver(deliver_bitmask, delivery_mode,
>                                  vector, polarity, trig_mode);
>+#ifdef USE_KVM
>+		}
>+#endif
>+
>             }
>         }
>     }


....

>diff --git a/user/kvmctl.c b/user/kvmctl.c
>index dcdffbf..e5f3e3e 100644
>--- a/user/kvmctl.c
>+++ b/user/kvmctl.c
>@@ -233,6 +233,7 @@ int kvm_create(kvm_context_t kvm, unsigned long
memory,
>void **vm_mem)
> 	int fd = kvm->fd;
> 	int zfd;
> 	int r;
>+	const int apic_level = 1;
> 	struct kvm_memory_region low_memory = {
> 		.slot = 3,
> 		.memory_size = memory  < dosmem ? memory : dosmem,
>@@ -282,6 +283,12 @@ int kvm_create(kvm_context_t kvm, unsigned long
>memory, void **vm_mem)
> 	     MAP_PRIVATE|MAP_FIXED, zfd, 0);
> 	close(zfd);
>
>+	r = ioctl(fd, KVM_ENABLE_KERNEL_PIC, &apic_level);
>+	if (r == -1) {
>+		fprintf(stderr, "kvm_enable_kernel_pic: %m\n");
>+		return -1;
>+	}
>+
> 	r = ioctl(fd, KVM_CREATE_VCPU, 0);
> 	if (r == -1) {
> 		fprintf(stderr, "kvm_create_vcpu: %m\n");
>@@ -772,12 +779,26 @@ more:
> 	return r;
> }
>
>-int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
>+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq)
> {
> 	struct kvm_interrupt intr;
>
> 	intr.irq = irq;
>-	return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
>+	return ioctl(kvm->vm_fd, KVM_ISA_INTERRUPT, &intr);
>+}

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 2/3] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

>>> On Wed, May 9, 2007 at  4:16 AM, in message
<64F9B87B6B770947A9F8391472E032160BBA5FCB-yEcIvxbTEBqsx+V+t5oei8rau4O3wl8o3fe8/T/H7NteoWH0uzbU5w@public.gmane.org>,
"Dor Laor" <dor.laor-atKUWr5tajBWk0Htik3J/w@public.gmane.org> wrote: 

> For having the option of using kvm with qemu- apic you can use the
> apic_level variable below and add it to qemu's configuration params.
> This will be valuable while testing your implementation.

Thats a good idea.  Thanks Dor!

-Greg



-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/