[PATCH 0/5] in-kernel APIC v4 (usermode side)

[PATCH 0/5] in-kernel APIC v4 (usermode side)

[Gregory Haskins]

Userspace additions to coordinate with v4 kernel patches.  Note that the
eventfd is supported in the user-library, but QEMU doesnt use it yet.

Regards,
-Greg

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 1/5] KVM: Updates for compiling in-kernel APIC support with external-modules

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 kernel/Kbuild |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/kernel/Kbuild b/kernel/Kbuild
index e9bcda7..103a179 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -1,5 +1,5 @@
 EXTRA_CFLAGS := -I$(src)/include -include $(src)/external-module-compat.h
 obj-m := kvm.o kvm-intel.o kvm-amd.o
-kvm-objs := kvm_main.o mmu.o x86_emulate.o
+kvm-objs := kvm_main.o mmu.o x86_emulate.o userint.o kernint.o lapic.o
 kvm-intel-objs := vmx.o vmx-debug.o
 kvm-amd-objs := svm.o


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 2/5] KVM-USER: Make the kvm_allowed flag always defined so we dont need #ifdefs

[Gregory Haskins]

Non-performance critical code is made more awkward by having to always define
both "#ifdef KVM" and "if (kvm_allowed)".  Define "kvm_allowed = 0" by
default.  Anthony Ligouri is credited with the idea.

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/qemu-kvm.c |    9 ++++++++-
 1 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/qemu/qemu-kvm.c b/qemu/qemu-kvm.c
index 212570a..d4419a3 100644
--- a/qemu/qemu-kvm.c
+++ b/qemu/qemu-kvm.c
@@ -3,6 +3,14 @@
 #include "config-host.h"
 
 #ifdef USE_KVM
+ #define KVM_ALLOWED_DEFAULT 1
+#else
+ #define KVM_ALLOWED_DEFAULT 0
+#endif
+
+int kvm_allowed = KVM_ALLOWED_DEFAULT;
+
+#ifdef USE_KVM
 
 #include "exec.h"
 
@@ -14,7 +22,6 @@
 
 extern void perror(const char *s);
 
-int kvm_allowed = 1;
 kvm_context_t kvm_context;
 static struct kvm_msr_list *kvm_msr_list;
 static int kvm_has_msr_star;


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 3/5] KVM-USER: Add ability to specify APIC emulation type from the command-line

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/qemu-kvm.c |    1 +
 qemu/vl.c       |    5 +++++
 qemu/vl.h       |    1 +
 3 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/qemu/qemu-kvm.c b/qemu/qemu-kvm.c
index d4419a3..faa4684 100644
--- a/qemu/qemu-kvm.c
+++ b/qemu/qemu-kvm.c
@@ -9,6 +9,7 @@
 #endif
 
 int kvm_allowed = KVM_ALLOWED_DEFAULT;
+int kvm_apic_level = 1;
 
 #ifdef USE_KVM
 
diff --git a/qemu/vl.c b/qemu/vl.c
index 7df1c80..986cea4 100644
--- a/qemu/vl.c
+++ b/qemu/vl.c
@@ -6531,6 +6531,7 @@ enum {
     QEMU_OPTION_vnc,
     QEMU_OPTION_no_acpi,
     QEMU_OPTION_no_kvm,
+    QEMU_OPTION_kvm_apic,
     QEMU_OPTION_no_reboot,
     QEMU_OPTION_daemonize,
     QEMU_OPTION_option_rom,
@@ -6600,6 +6601,7 @@ const QEMUOption qemu_options[] = {
 #endif
 #ifdef USE_KVM
     { "no-kvm", 0, QEMU_OPTION_no_kvm },
+    { "kvm_apic", HAS_ARG, QEMU_OPTION_kvm_apic },
 #endif
 #if defined(TARGET_PPC) || defined(TARGET_SPARC)
     { "g", 1, QEMU_OPTION_g },
@@ -7309,6 +7311,9 @@ int main(int argc, char **argv)
 	    case QEMU_OPTION_no_kvm:
 		kvm_allowed = 0;
 		break;
+	    case QEMU_OPTION_kvm_apic:
+		kvm_apic_level = atoi(optarg);
+		break;
 #endif
             case QEMU_OPTION_usb:
                 usb_enabled = 1;
diff --git a/qemu/vl.h b/qemu/vl.h
index debd17c..dec410e 100644
--- a/qemu/vl.h
+++ b/qemu/vl.h
@@ -158,6 +158,7 @@ extern int graphic_depth;
 extern const char *keyboard_layout;
 extern int kqemu_allowed;
 extern int kvm_allowed;
+extern int kvm_apic_level;
 extern int win2k_install_hack;
 extern int usb_enabled;
 extern int smp_cpus;


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/hw/apic.c  |   20 +++++++++++++++-----
 qemu/hw/pc.c    |   29 ++++++++++++++++-------------
 qemu/qemu-kvm.c |   49 +++++++++++++++++++++++++++++++++++++++----------
 qemu/qemu-kvm.h |    2 ++
 qemu/vl.h       |    7 ++++++-
 user/kvmctl.c   |   33 ++++++++++++++++++++++++++++++++-
 user/kvmctl.h   |   31 ++++++++++++++++++++++++++++++-
 user/main.c     |    2 +-
 8 files changed, 141 insertions(+), 32 deletions(-)

diff --git a/qemu/hw/apic.c b/qemu/hw/apic.c
index 0b73233..5665057 100644
--- a/qemu/hw/apic.c
+++ b/qemu/hw/apic.c
@@ -18,6 +18,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #include "vl.h"
+#include "qemu-kvm.h"
 
 //#define DEBUG_APIC
 //#define DEBUG_IOAPIC
@@ -87,6 +88,7 @@ typedef struct APICState {
 } APICState;
 
 struct IOAPICState {
+    CPUState *cpu_env;
     uint8_t id;
     uint8_t ioregsel;
 
@@ -888,10 +890,17 @@ static void ioapic_service(IOAPICState *s)
                     vector = pic_read_irq(isa_pic);
                 else
                     vector = entry & 0xff;
-                
-                apic_get_delivery_bitmask(deliver_bitmask, dest, dest_mode);
-                apic_bus_deliver(deliver_bitmask, delivery_mode, 
-                                 vector, polarity, trig_mode);
+             
+		if (kvm_allowed && kvm_apic_level) {
+		    ext_apic_bus_deliver(dest, trig_mode, dest_mode,
+					 delivery_mode, vector);
+		    cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+		} else {
+		    apic_get_delivery_bitmask(deliver_bitmask, dest,
+					      dest_mode);
+		    apic_bus_deliver(deliver_bitmask, delivery_mode, 
+				     vector, polarity, trig_mode);
+		}
             }
         }
     }
@@ -1045,7 +1054,7 @@ static CPUWriteMemoryFunc *ioapic_mem_write[3] = {
     ioapic_mem_writel,
 };
 
-IOAPICState *ioapic_init(void)
+IOAPICState *ioapic_init(CPUState *env)
 {
     IOAPICState *s;
     int io_memory;
@@ -1054,6 +1063,7 @@ IOAPICState *ioapic_init(void)
     if (!s)
         return NULL;
     ioapic_reset(s);
+    s->cpu_env = env;
     s->id = last_apic_id++;
 
     io_memory = cpu_register_io_memory(0, ioapic_mem_read, 
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index eda49cf..df51539 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -91,16 +91,19 @@ int cpu_get_pic_interrupt(CPUState *env)
 {
     int intno;
 
-    intno = apic_get_interrupt(env);
-    if (intno >= 0) {
-        /* set irq request if a PIC irq is still pending */
-        /* XXX: improve that */
-        pic_update_irq(isa_pic); 
-        return intno;
+    if (!use_kernel_apic()) {
+	intno = apic_get_interrupt(env);
+	if (intno >= 0) {
+	    /* set irq request if a PIC irq is still pending */
+	    /* XXX: improve that */
+	    pic_update_irq(isa_pic); 
+	    return intno;
+	}
+	
+	/* read the irq from the PIC */
+	if (!apic_accept_pic_intr(env))
+	    return -1;
     }
-    /* read the irq from the PIC */
-    if (!apic_accept_pic_intr(env))
-        return -1;
 
     intno = pic_read_irq(isa_pic);
     return intno;
@@ -483,9 +486,9 @@ static void pc_init1(int ram_size, int vga_ram_size, int boot_device,
         }
         register_savevm("cpu", i, 4, cpu_save, cpu_load, env);
         qemu_register_reset(main_cpu_reset, env);
-        if (pci_enabled) {
-            apic_init(env);
-        }
+	if (!use_kernel_apic() && pci_enabled) {
+	    apic_init(env);
+	}
     }
 
     /* allocate RAM */
@@ -671,7 +674,7 @@ static void pc_init1(int ram_size, int vga_ram_size, int boot_device,
     register_ioport_write(0x92, 1, 1, ioport92_write, NULL);
 
     if (pci_enabled) {
-        ioapic = ioapic_init();
+        ioapic = ioapic_init(env);
     }
     isa_pic = pic_init(pic_irq_request, first_cpu);
     pit = pit_init(0x40, 0);
diff --git a/qemu/qemu-kvm.c b/qemu/qemu-kvm.c
index faa4684..59e79bf 100644
--- a/qemu/qemu-kvm.c
+++ b/qemu/qemu-kvm.c
@@ -235,9 +235,16 @@ static void load_regs(CPUState *env)
     sregs.cr3 = env->cr[3];
     sregs.cr4 = env->cr[4];
 
-    sregs.apic_base = cpu_get_apic_base(env);
+    if (!kvm_apic_level) {
+	/* These two are no longer used once the in-kernel APIC is enabled */
+	sregs.apic_base = 0;
+	sregs.cr8 = 0;
+    } else {
+	sregs.apic_base = cpu_get_apic_base(env);
+	sregs.cr8 = cpu_get_apic_tpr(env);
+    }
+
     sregs.efer = env->efer;
-    sregs.cr8 = cpu_get_apic_tpr(env);
 
     kvm_set_sregs(kvm_context, 0, &sregs);
 
@@ -329,10 +336,12 @@ static void save_regs(CPUState *env)
     env->cr[3] = sregs.cr3;
     env->cr[4] = sregs.cr4;
 
-    cpu_set_apic_base(env, sregs.apic_base);
+    if (!kvm_apic_level) {
+	cpu_set_apic_base(env, sregs.apic_base);
+	//cpu_set_apic_tpr(env, sregs.cr8);
+    }
 
     env->efer = sregs.efer;
-    //cpu_set_apic_tpr(env, sregs.cr8);
 
 #define HFLAG_COPY_MASK ~( \
 			HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
@@ -418,9 +427,17 @@ static int try_push_interrupts(void *opaque)
     if (env->ready_for_interrupt_injection &&
         (env->interrupt_request & CPU_INTERRUPT_HARD) &&
         (env->eflags & IF_MASK)) {
+	    int irq = cpu_get_pic_interrupt(env);
+
+	    if (irq != -1) {
+		if (kvm_apic_level)
+		    kvm_inject_isa_irq(kvm_context, irq);
+		else
+		    // for now using cpu 0
+		    kvm_inject_irq(kvm_context, 0, irq);
+	    }
+	    
             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
-            // for now using cpu 0
-            kvm_inject_irq(kvm_context, 0, cpu_get_pic_interrupt(env));
     }
 
     return (env->interrupt_request & CPU_INTERRUPT_HARD) != 0;
@@ -433,8 +450,11 @@ static void post_kvm_run(void *opaque, struct kvm_run *kvm_run)
 
     env->eflags = (kvm_run->if_flag) ? env->eflags | IF_MASK:env->eflags & ~IF_MASK;
     env->ready_for_interrupt_injection = kvm_run->ready_for_interrupt_injection;
-    //cpu_set_apic_tpr(env, kvm_run->cr8);
-    cpu_set_apic_base(env, kvm_run->apic_base);
+
+    if (!kvm_apic_level) {
+	//cpu_set_apic_tpr(env, kvm_run->cr8);
+	cpu_set_apic_base(env, kvm_run->apic_base);
+    }
 }
 
 static void pre_kvm_run(void *opaque, struct kvm_run *kvm_run)
@@ -442,7 +462,15 @@ static void pre_kvm_run(void *opaque, struct kvm_run *kvm_run)
     CPUState **envs = opaque, *env;
     env = envs[0];
 
-    kvm_run->cr8 = cpu_get_apic_tpr(env);
+    if (!kvm_apic_level)
+	kvm_run->cr8 = cpu_get_apic_tpr(env);
+}
+
+int ext_apic_bus_deliver(int dest, int trig_mode, int dest_mode,
+			 int delivery_mode, int vector)
+{
+	return kvm_apic_bus_deliver(kvm_context, dest, trig_mode, dest_mode,
+				    delivery_mode, vector);
 }
 
 void kvm_load_registers(CPUState *env)
@@ -670,7 +698,8 @@ int kvm_qemu_create_context(void)
 {
     int i;
 
-    if (kvm_create(kvm_context, phys_ram_size, (void**)&phys_ram_base) < 0) {
+    if (kvm_create(kvm_context, phys_ram_size, kvm_apic_level,
+		   (void**)&phys_ram_base) < 0) {
 	kvm_qemu_destroy();
 	return -1;
     }
diff --git a/qemu/qemu-kvm.h b/qemu/qemu-kvm.h
index 8a1c25d..7bfdf62 100644
--- a/qemu/qemu-kvm.h
+++ b/qemu/qemu-kvm.h
@@ -6,6 +6,8 @@
 int kvm_qemu_init(void);
 int kvm_qemu_create_context(void);
 void kvm_qemu_destroy(void);
+int ext_apic_bus_deliver(int dest, int trig_mode, int dest_mode,
+			 int delivery_mode, int vector);
 void kvm_load_registers(CPUState *env);
 void kvm_save_registers(CPUState *env);
 int kvm_cpu_exec(CPUState *env);
diff --git a/qemu/vl.h b/qemu/vl.h
index dec410e..d532117 100644
--- a/qemu/vl.h
+++ b/qemu/vl.h
@@ -167,6 +167,11 @@ extern int semihosting_enabled;
 extern int autostart;
 extern int time_drift_fix;
 
+static inline int use_kernel_apic()
+{
+    return kvm_allowed && kvm_apic_level;
+}
+
 #define MAX_OPTION_ROMS 16
 extern const char *option_rom[MAX_OPTION_ROMS];
 extern int nb_option_roms;
@@ -1060,7 +1065,7 @@ typedef struct IOAPICState IOAPICState;
 int apic_init(CPUState *env);
 int apic_get_interrupt(CPUState *env);
 int apic_accept_pic_intr(CPUState *env);
-IOAPICState *ioapic_init(void);
+IOAPICState *ioapic_init(CPUState *env);
 void ioapic_set_irq(void *opaque, int vector, int level);
 
 /* i8254.c */
diff --git a/user/kvmctl.c b/user/kvmctl.c
index dcdffbf..bc6e238 100644
--- a/user/kvmctl.c
+++ b/user/kvmctl.c
@@ -225,7 +225,8 @@ void kvm_finalize(kvm_context_t kvm)
 	free(kvm);
 }
 
-int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
+int kvm_create(kvm_context_t kvm, unsigned long memory, int apic_level,
+	       void **vm_mem)
 {
 	unsigned long dosmem = 0xa0000;
 	unsigned long exmem = 0xc0000;
@@ -282,6 +283,14 @@ int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
 	     MAP_PRIVATE|MAP_FIXED, zfd, 0);
 	close(zfd);
 
+	if (apic_level) {
+		r = ioctl(fd, KVM_ENABLE_KERNEL_PIC, &apic_level);
+		if (r == -1) {
+			fprintf(stderr, "kvm_enable_kernel_pic: %m\n");
+			return -1;
+		}
+	}
+
 	r = ioctl(fd, KVM_CREATE_VCPU, 0);
 	if (r == -1) {
 		fprintf(stderr, "kvm_create_vcpu: %m\n");
@@ -780,6 +789,28 @@ int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 	return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 }
 
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq)
+{
+	struct kvm_interrupt intr;
+
+	intr.irq = irq;
+	return ioctl(kvm->vm_fd, KVM_ISA_INTERRUPT, &intr);
+}
+
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector)
+{
+	struct kvm_apic_msg msg;
+
+	msg.dest          = dest;
+	msg.trig_mode     = trig_mode;
+	msg.dest_mode     = dest_mode;
+	msg.delivery_mode = delivery_mode;
+	msg.vector        = vector;
+
+	return ioctl(kvm->vm_fd, KVM_APIC_MSG, &msg);
+}
+
 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
 {
 	return ioctl(kvm->vcpu_fd[vcpu], KVM_DEBUG_GUEST, dbg);
diff --git a/user/kvmctl.h b/user/kvmctl.h
index 27c010c..b775347 100644
--- a/user/kvmctl.h
+++ b/user/kvmctl.h
@@ -100,12 +100,14 @@ void kvm_finalize(kvm_context_t kvm);
  *
  * \param kvm Pointer to the current kvm_context
  * \param phys_mem_bytes The amount of physical ram you want the VM to have
+ * \param apic_level The APIC emulation level (0=QEMU, 1=KVM)
  * \param phys_mem This pointer will be set to point to the memory that
  * kvm_create allocates for physical RAM
  * \return 0 on success
  */
 int kvm_create(kvm_context_t kvm,
 	       unsigned long phys_mem_bytes,
+	       int apic_level,
 	       void **phys_mem);
 
 /*!
@@ -236,11 +238,38 @@ int kvm_set_msrs(kvm_context_t, int vcpu, struct kvm_msr_entry *msrs, int n);
  * This allows you to simulate an external vectored interrupt.
  *
  * \param kvm Pointer to the current kvm_context
- * \param vcpu Which virtual CPU should get dumped
+ * \param vcpu Which virtual CPU should handle interrupt
  * \param irq Vector number
  * \return 0 on success
  */
 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq);
+
+/*!
+ * \brief Simulate an external vectored interrupt to the ISA bus
+ *
+ * This allows you to simulate an external vectored interrupt.
+ *
+ * \param kvm Pointer to the current kvm_context
+ * \param irq Vector number
+ * \return 0 on success
+ */
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq);
+
+/*!
+ * \brief Simulate an external vectored interrupt to the APIC bus
+ *
+ * This allows you to simulate a vectored interrupt via the LAPIC mechanism.
+ *
+ * \param kvm Pointer to the current kvm_context
+ * \param dest Encoded destination
+ * \param trig_mode 0=edge-trigger, 1=level-trigger
+ * \param dest_mode Destination mode encoding
+ * \param delivery_mode Delivery_mode encoding
+ * \param vector The vector number
+ * \return 0 on success
+ */
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector);
 int kvm_guest_debug(kvm_context_t, int vcpu, struct kvm_debug_guest *dbg);
 
 /*!
diff --git a/user/main.c b/user/main.c
index 83fd752..add5eef 100644
--- a/user/main.c
+++ b/user/main.c
@@ -180,7 +180,7 @@ int main(int ac, char **av)
 	    fprintf(stderr, "kvm_init failed\n");
 	    return 1;
 	}
-	if (kvm_create(kvm, 128 * 1024 * 1024, &vm_mem) < 0) {
+	if (kvm_create(kvm, 128 * 1024 * 1024, 1, &vm_mem) < 0) {
 	    kvm_finalize(kvm);
 	    fprintf(stderr, "kvm_create failed\n");
 	    return 1;


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Anthony Liguori]

Gregory Haskins wrote:
> Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
> ---         
> +		if (kvm_allowed && kvm_apic_level) {
>   
use_kernel_apic()?
> +		    ext_apic_bus_deliver(dest, trig_mode, dest_mode,
> +					 delivery_mode, vector);
> +		    cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
> +		} else {
> +		    apic_get_delivery_bitmask(deliver_bitmask, dest,
> +					      dest_mode);
> +		    apic_bus_deliver(deliver_bitmask, delivery_mode, 
> +				     vector, polarity, trig_mode);
> +		}
>              }
>          }
>      }
>   

Regards,

Anthony Liguori

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

>>> On Tue, May 15, 2007 at 10:17 AM, in message <4649C0E9.4080401-rdkfGonbjUSkNkDKm+mE6A@public.gmane.org>,
Anthony Liguori <anthony-rdkfGonbjUSkNkDKm+mE6A@public.gmane.org> wrote: 
> Gregory Haskins wrote:
>> Signed- off- by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
>> ---          
>> +		if (kvm_allowed && kvm_apic_level) {
>>   
> use_kernel_apic()?

Oops...missed that one.  Thanks!

>> +		    ext_apic_bus_deliver(dest, trig_mode, dest_mode,
>> +					 delivery_mode, vector);
>> +		    cpu_interrupt(s- >cpu_env, CPU_INTERRUPT_HARD);
>> +		} else {
>> +		    apic_get_delivery_bitmask(deliver_bitmask, dest,
>> +					      dest_mode);
>> +		    apic_bus_deliver(deliver_bitmask, delivery_mode, 
>> +				     vector, polarity, trig_mode);
>> +		}
>>              }
>>          }
>>      }
>>   
> 
> Regards,
> 
> Anthony Liguori



-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Avi Kivity]

Gregory Haskins wrote:
> @@ -1045,7 +1054,7 @@ static CPUWriteMemoryFunc *ioapic_mem_write[3] = {
>      ioapic_mem_writel,
>  };
>  
> -IOAPICState *ioapic_init(void)
> +IOAPICState *ioapic_init(CPUState *env)
>  {
>      IOAPICState *s;
>      int io_memory;
> @@ -1054,6 +1063,7 @@ IOAPICState *ioapic_init(void)
>      if (!s)
>          return NULL;
>      ioapic_reset(s);
> +    s->cpu_env = env;
>   

Is there actually a 1:1 correspondence between ioapics and vcpus?




-- 
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

Re: [PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

>>> On Wed, May 16, 2007 at  7:51 AM, in message <464AF035.9040405-atKUWr5tajBWk0Htik3J/w@public.gmane.org>,
Avi Kivity <avi-atKUWr5tajBWk0Htik3J/w@public.gmane.org> wrote: 
> Gregory Haskins wrote:
>> @@ - 1045,7 +1054,7 @@ static CPUWriteMemoryFunc *ioapic_mem_write[3] = {
>>      ioapic_mem_writel,
>>  };
>>  
>> - IOAPICState *ioapic_init(void)
>> +IOAPICState *ioapic_init(CPUState *env)
>>  {
>>      IOAPICState *s;
>>      int io_memory;
>> @@ - 1054,6 +1063,7 @@ IOAPICState *ioapic_init(void)
>>      if (!s)
>>          return NULL;
>>      ioapic_reset(s);
>> +    s- >cpu_env = env;
>>   
> 
> Is there actually a 1:1 correspondence between ioapics and vcpus?
> 
> 
> 


While this would be highly unusual, there is nothing that prevents a system architect from doing so.  In the case of the QEMU/KVM model, however, this is not true.   There is one 1:1 relationship with the IOAPIC and the ISA bus, not per-vcpu.  It looks like I might have coded this incorrectly.   I will fix.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 5/5] KVM-USER: Add support for listening for kernel-based interrupts

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 user/kvmctl.c |   32 ++++++++++++++++++++++++++++++++
 user/kvmctl.h |    2 ++
 2 files changed, 34 insertions(+), 0 deletions(-)

diff --git a/user/kvmctl.c b/user/kvmctl.c
index bc6e238..ea86426 100644
--- a/user/kvmctl.c
+++ b/user/kvmctl.c
@@ -22,6 +22,8 @@
 #include <string.h>
 #include <errno.h>
 #include <sys/ioctl.h>
+#include <sys/syscall.h>
+
 #include "kvmctl.h"
 
 #define EXPECTED_KVM_API_VERSION 12
@@ -398,6 +400,36 @@ int kvm_get_dirty_pages(kvm_context_t kvm, int slot, void *buf)
 	return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 }
 
+#ifndef __NR_eventfd
+#if defined(__x86_64__)
+#define __NR_eventfd 283
+#elif defined(__i386__)
+#define __NR_eventfd 323
+#else
+#error Cannot detect your architecture!
+#endif
+#endif
+
+int kvm_vcpu_create_eventfd(kvm_context_t kvm, int vcpu)
+{
+	int r;
+        /*
+	 * Replace this once the updated glibc comes out
+	 */
+	int fd = syscall(__NR_eventfd, 0);
+
+	if (fd < 0)
+		return fd;
+
+	r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_EVENTFD, fd);
+	if (r < 0) {
+		close(fd);
+		return r;
+	}
+
+	return fd;
+}
+
 int kvm_get_mem_map(kvm_context_t kvm, int slot, void *buf)
 {
 #ifdef KVM_GET_MEM_MAP
diff --git a/user/kvmctl.h b/user/kvmctl.h
index b775347..07cbc08 100644
--- a/user/kvmctl.h
+++ b/user/kvmctl.h
@@ -337,6 +337,8 @@ void kvm_destroy_phys_mem(kvm_context_t, unsigned long phys_start,
 			  unsigned long len);
 int kvm_get_dirty_pages(kvm_context_t, int slot, void *buf);
 
+int kvm_vcpu_create_eventfd(kvm_context_t, int vcpu);
+
 
 /*
  * \brief Create a memory alias


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 0/5] in-kernel APIC v6 (usermode side)

[Gregory Haskins]

Changelog from v5:

1) Moved eventfd patch to the end of the series and dropped from the public
   release (for now...see the kernel-side comments for details)

Regards,
-Greg

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[PATCH 4/5] KVM: in-kernel-apic modification to QEMU

[Gregory Haskins]

Signed-off-by: Gregory Haskins <ghaskins-Et1tbQHTxzrQT0dZR+AlfA@public.gmane.org>
---

 qemu/hw/apic.c  |   20 +++++++++++++++-----
 qemu/hw/pc.c    |   29 ++++++++++++++++-------------
 qemu/qemu-kvm.c |   49 +++++++++++++++++++++++++++++++++++++++----------
 qemu/qemu-kvm.h |    2 ++
 qemu/vl.h       |   11 ++++++++++-
 user/kvmctl.c   |   33 ++++++++++++++++++++++++++++++++-
 user/kvmctl.h   |   31 ++++++++++++++++++++++++++++++-
 user/main.c     |    2 +-
 8 files changed, 145 insertions(+), 32 deletions(-)

diff --git a/qemu/hw/apic.c b/qemu/hw/apic.c
index 5704224..901f3b6 100644
--- a/qemu/hw/apic.c
+++ b/qemu/hw/apic.c
@@ -18,6 +18,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #include "vl.h"
+#include "qemu-kvm.h"
 
 //#define DEBUG_APIC
 //#define DEBUG_IOAPIC
@@ -87,6 +88,7 @@ typedef struct APICState {
 } APICState;
 
 struct IOAPICState {
+    CPUState *cpu_env;
     uint8_t id;
     uint8_t ioregsel;
 
@@ -895,10 +897,17 @@ static void ioapic_service(IOAPICState *s)
                     vector = pic_read_irq(isa_pic);
                 else
                     vector = entry & 0xff;
-                
-                apic_get_delivery_bitmask(deliver_bitmask, dest, dest_mode);
-                apic_bus_deliver(deliver_bitmask, delivery_mode, 
-                                 vector, polarity, trig_mode);
+             
+		if (use_kernel_apic()) {
+		    ext_apic_bus_deliver(dest, trig_mode, dest_mode,
+					 delivery_mode, vector);
+		    cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+		} else {
+		    apic_get_delivery_bitmask(deliver_bitmask, dest,
+					      dest_mode);
+		    apic_bus_deliver(deliver_bitmask, delivery_mode, 
+				     vector, polarity, trig_mode);
+		}
             }
         }
     }
@@ -1052,7 +1061,7 @@ static CPUWriteMemoryFunc *ioapic_mem_write[3] = {
     ioapic_mem_writel,
 };
 
-IOAPICState *ioapic_init(void)
+IOAPICState *ioapic_init(CPUState *env)
 {
     IOAPICState *s;
     int io_memory;
@@ -1061,6 +1070,7 @@ IOAPICState *ioapic_init(void)
     if (!s)
         return NULL;
     ioapic_reset(s);
+    s->cpu_env = env;
     s->id = last_apic_id++;
 
     io_memory = cpu_register_io_memory(0, ioapic_mem_read, 
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index eda49cf..df51539 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -91,16 +91,19 @@ int cpu_get_pic_interrupt(CPUState *env)
 {
     int intno;
 
-    intno = apic_get_interrupt(env);
-    if (intno >= 0) {
-        /* set irq request if a PIC irq is still pending */
-        /* XXX: improve that */
-        pic_update_irq(isa_pic); 
-        return intno;
+    if (!use_kernel_apic()) {
+	intno = apic_get_interrupt(env);
+	if (intno >= 0) {
+	    /* set irq request if a PIC irq is still pending */
+	    /* XXX: improve that */
+	    pic_update_irq(isa_pic); 
+	    return intno;
+	}
+	
+	/* read the irq from the PIC */
+	if (!apic_accept_pic_intr(env))
+	    return -1;
     }
-    /* read the irq from the PIC */
-    if (!apic_accept_pic_intr(env))
-        return -1;
 
     intno = pic_read_irq(isa_pic);
     return intno;
@@ -483,9 +486,9 @@ static void pc_init1(int ram_size, int vga_ram_size, int boot_device,
         }
         register_savevm("cpu", i, 4, cpu_save, cpu_load, env);
         qemu_register_reset(main_cpu_reset, env);
-        if (pci_enabled) {
-            apic_init(env);
-        }
+	if (!use_kernel_apic() && pci_enabled) {
+	    apic_init(env);
+	}
     }
 
     /* allocate RAM */
@@ -671,7 +674,7 @@ static void pc_init1(int ram_size, int vga_ram_size, int boot_device,
     register_ioport_write(0x92, 1, 1, ioport92_write, NULL);
 
     if (pci_enabled) {
-        ioapic = ioapic_init();
+        ioapic = ioapic_init(env);
     }
     isa_pic = pic_init(pic_irq_request, first_cpu);
     pit = pit_init(0x40, 0);
diff --git a/qemu/qemu-kvm.c b/qemu/qemu-kvm.c
index d184cbd..1192fbb 100644
--- a/qemu/qemu-kvm.c
+++ b/qemu/qemu-kvm.c
@@ -245,9 +245,16 @@ static void load_regs(CPUState *env)
     sregs.cr3 = env->cr[3];
     sregs.cr4 = env->cr[4];
 
-    sregs.apic_base = cpu_get_apic_base(env);
+    if (!kvm_apic.level) {
+	/* These two are no longer used once the in-kernel APIC is enabled */
+	sregs.apic_base = 0;
+	sregs.cr8 = 0;
+    } else {
+	sregs.apic_base = cpu_get_apic_base(env);
+	sregs.cr8 = cpu_get_apic_tpr(env);
+    }
+
     sregs.efer = env->efer;
-    sregs.cr8 = cpu_get_apic_tpr(env);
 
     kvm_set_sregs(kvm_context, 0, &sregs);
 
@@ -339,10 +346,12 @@ static void save_regs(CPUState *env)
     env->cr[3] = sregs.cr3;
     env->cr[4] = sregs.cr4;
 
-    cpu_set_apic_base(env, sregs.apic_base);
+    if (!kvm_apic.level) {
+	cpu_set_apic_base(env, sregs.apic_base);
+	//cpu_set_apic_tpr(env, sregs.cr8);
+    }
 
     env->efer = sregs.efer;
-    //cpu_set_apic_tpr(env, sregs.cr8);
 
 #define HFLAG_COPY_MASK ~( \
 			HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
@@ -428,9 +437,17 @@ static int try_push_interrupts(void *opaque)
     if (env->ready_for_interrupt_injection &&
         (env->interrupt_request & CPU_INTERRUPT_HARD) &&
         (env->eflags & IF_MASK)) {
+	    int irq = cpu_get_pic_interrupt(env);
+
+	    if (irq != -1) {
+		if (kvm_apic.level)
+		    kvm_inject_isa_irq(kvm_context, irq);
+		else
+		    // for now using cpu 0
+		    kvm_inject_irq(kvm_context, 0, irq);
+	    }
+	    
             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
-            // for now using cpu 0
-            kvm_inject_irq(kvm_context, 0, cpu_get_pic_interrupt(env));
     }
 
     return (env->interrupt_request & CPU_INTERRUPT_HARD) != 0;
@@ -445,8 +462,11 @@ static void post_kvm_run(void *opaque, int vcpu)
 	? env->eflags | IF_MASK : env->eflags & ~IF_MASK;
     env->ready_for_interrupt_injection
 	= kvm_is_ready_for_interrupt_injection(kvm_context, vcpu);
-    //cpu_set_apic_tpr(env, kvm_run->cr8);
-    cpu_set_apic_base(env, kvm_get_apic_base(kvm_context, vcpu));
+
+    if (!kvm_apic.level) {
+	//cpu_set_apic_tpr(env, kvm_run->cr8);
+	cpu_set_apic_base(env, kvm_get_apic_base(kvm_context, vcpu));
+    }
 }
 
 static void pre_kvm_run(void *opaque, int vcpu)
@@ -454,7 +474,15 @@ static void pre_kvm_run(void *opaque, int vcpu)
     CPUState **envs = opaque, *env;
     env = envs[0];
 
-    kvm_set_cr8(kvm_context, vcpu, cpu_get_apic_tpr(env));
+    if (!kvm_apic.level)
+	kvm_set_cr8(kvm_context, vcpu, cpu_get_apic_tpr(env));
+}
+
+int ext_apic_bus_deliver(int dest, int trig_mode, int dest_mode,
+			 int delivery_mode, int vector)
+{
+	return kvm_apic_bus_deliver(kvm_context, dest, trig_mode, dest_mode,
+				    delivery_mode, vector);
 }
 
 void kvm_load_registers(CPUState *env)
@@ -682,7 +710,8 @@ int kvm_qemu_create_context(void)
 {
     int i;
 
-    if (kvm_create(kvm_context, phys_ram_size, (void**)&phys_ram_base) < 0) {
+    if (kvm_create(kvm_context, phys_ram_size, kvm_apic.level,
+		   (void**)&phys_ram_base) < 0) {
 	kvm_qemu_destroy();
 	return -1;
     }
diff --git a/qemu/qemu-kvm.h b/qemu/qemu-kvm.h
index c4cb34e..2dd5e00 100644
--- a/qemu/qemu-kvm.h
+++ b/qemu/qemu-kvm.h
@@ -8,6 +8,8 @@
 int kvm_qemu_init(void);
 int kvm_qemu_create_context(void);
 void kvm_qemu_destroy(void);
+int ext_apic_bus_deliver(int dest, int trig_mode, int dest_mode,
+			 int delivery_mode, int vector);
 void kvm_load_registers(CPUState *env);
 void kvm_save_registers(CPUState *env);
 int kvm_cpu_exec(CPUState *env);
diff --git a/qemu/vl.h b/qemu/vl.h
index 4e93a81..514a86b 100644
--- a/qemu/vl.h
+++ b/qemu/vl.h
@@ -167,6 +167,15 @@ extern int semihosting_enabled;
 extern int autostart;
 extern int time_drift_fix;
 
+static inline int use_kernel_apic()
+{
+#ifdef USE_KVM
+    return kvm_allowed && kvm_qemu_get_apic_level();
+#else
+    return 0;
+#endif
+}
+
 #define MAX_OPTION_ROMS 16
 extern const char *option_rom[MAX_OPTION_ROMS];
 extern int nb_option_roms;
@@ -1060,7 +1069,7 @@ typedef struct IOAPICState IOAPICState;
 int apic_init(CPUState *env);
 int apic_get_interrupt(CPUState *env);
 int apic_accept_pic_intr(CPUState *env);
-IOAPICState *ioapic_init(void);
+IOAPICState *ioapic_init(CPUState *env);
 void ioapic_set_irq(void *opaque, int vector, int level);
 
 /* i8254.c */
diff --git a/user/kvmctl.c b/user/kvmctl.c
index 82d1926..70f9626 100644
--- a/user/kvmctl.c
+++ b/user/kvmctl.c
@@ -202,7 +202,8 @@ void kvm_finalize(kvm_context_t kvm)
 	free(kvm);
 }
 
-int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
+int kvm_create(kvm_context_t kvm, unsigned long memory, int apic_level,
+	       void **vm_mem)
 {
 	unsigned long dosmem = 0xa0000;
 	unsigned long exmem = 0xc0000;
@@ -259,6 +260,14 @@ int kvm_create(kvm_context_t kvm, unsigned long memory, void **vm_mem)
 	     MAP_PRIVATE|MAP_FIXED, zfd, 0);
 	close(zfd);
 
+	if (apic_level) {
+		r = ioctl(fd, KVM_ENABLE_KERNEL_PIC, &apic_level);
+		if (r == -1) {
+			fprintf(stderr, "kvm_enable_kernel_pic: %m\n");
+			return -1;
+		}
+	}
+
 	r = ioctl(fd, KVM_CREATE_VCPU, 0);
 	if (r == -1) {
 		fprintf(stderr, "kvm_create_vcpu: %m\n");
@@ -999,6 +1008,28 @@ int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 	return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 }
 
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq)
+{
+	struct kvm_interrupt intr;
+
+	intr.irq = irq;
+	return ioctl(kvm->vm_fd, KVM_ISA_INTERRUPT, &intr);
+}
+
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector)
+{
+	struct kvm_apic_msg msg;
+
+	msg.dest          = dest;
+	msg.trig_mode     = trig_mode;
+	msg.dest_mode     = dest_mode;
+	msg.delivery_mode = delivery_mode;
+	msg.vector        = vector;
+
+	return ioctl(kvm->vm_fd, KVM_APIC_MSG, &msg);
+}
+
 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
 {
 	return ioctl(kvm->vcpu_fd[vcpu], KVM_DEBUG_GUEST, dbg);
diff --git a/user/kvmctl.h b/user/kvmctl.h
index b14cb45..9aab269 100644
--- a/user/kvmctl.h
+++ b/user/kvmctl.h
@@ -100,12 +100,14 @@ void kvm_finalize(kvm_context_t kvm);
  *
  * \param kvm Pointer to the current kvm_context
  * \param phys_mem_bytes The amount of physical ram you want the VM to have
+ * \param apic_level The APIC emulation level (0=QEMU, 1=KVM)
  * \param phys_mem This pointer will be set to point to the memory that
  * kvm_create allocates for physical RAM
  * \return 0 on success
  */
 int kvm_create(kvm_context_t kvm,
 	       unsigned long phys_mem_bytes,
+	       int apic_level,
 	       void **phys_mem);
 
 /*!
@@ -280,11 +282,38 @@ int kvm_set_msrs(kvm_context_t, int vcpu, struct kvm_msr_entry *msrs, int n);
  * This allows you to simulate an external vectored interrupt.
  *
  * \param kvm Pointer to the current kvm_context
- * \param vcpu Which virtual CPU should get dumped
+ * \param vcpu Which virtual CPU should handle interrupt
  * \param irq Vector number
  * \return 0 on success
  */
 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq);
+
+/*!
+ * \brief Simulate an external vectored interrupt to the ISA bus
+ *
+ * This allows you to simulate an external vectored interrupt.
+ *
+ * \param kvm Pointer to the current kvm_context
+ * \param irq Vector number
+ * \return 0 on success
+ */
+int kvm_inject_isa_irq(kvm_context_t kvm, unsigned irq);
+
+/*!
+ * \brief Simulate an external vectored interrupt to the APIC bus
+ *
+ * This allows you to simulate a vectored interrupt via the LAPIC mechanism.
+ *
+ * \param kvm Pointer to the current kvm_context
+ * \param dest Encoded destination
+ * \param trig_mode 0=edge-trigger, 1=level-trigger
+ * \param dest_mode Destination mode encoding
+ * \param delivery_mode Delivery_mode encoding
+ * \param vector The vector number
+ * \return 0 on success
+ */
+int kvm_apic_bus_deliver(kvm_context_t kvm, int dest, int trig_mode,
+			 int dest_mode, int delivery_mode, int vector);
 int kvm_guest_debug(kvm_context_t, int vcpu, struct kvm_debug_guest *dbg);
 
 /*!
diff --git a/user/main.c b/user/main.c
index c5da89c..88ae9f4 100644
--- a/user/main.c
+++ b/user/main.c
@@ -180,7 +180,7 @@ int main(int ac, char **av)
 	    fprintf(stderr, "kvm_init failed\n");
 	    return 1;
 	}
-	if (kvm_create(kvm, 128 * 1024 * 1024, &vm_mem) < 0) {
+	if (kvm_create(kvm, 128 * 1024 * 1024, 1, &vm_mem) < 0) {
 	    kvm_finalize(kvm);
 	    fprintf(stderr, "kvm_create failed\n");
 	    return 1;


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/