Re: kernel device reset support

Dong, Eddie
2007-10-10T06:17:29Z

[-- Attachment #1: Type: text/plain, Size: 5015 bytes --]

Avi Kivity wrote:
> Dong, Eddie wrote:
>> Avi Kivity wrote:
>>
>>> But, for an ungraceful reset, nothing prevents an AP from
>>> issuing a reset?
>>>
>>
>> Mmm, Yes, but I think current architecture can't handle this.
>> The thread where AP issues "RESET" will continue run, which
>> means it becomes BSP now and wake up other APs later on.
>> Or We can block that AP first and then inform BSP to do
>> RESET job. Here we need to block the AP in kernel
>> so that we can wake up.
>>
>
> It should call vcpu_halt() immediately after reset.

?? Can user level be able to enter kernel HALT state.

>
>> It can be a future task which is not that high priority IMO.
>> I will focus on SMP boot 1st. Your opnion?
>>
>
> Agree. But let's make it close to the complete solution.
>

Yes, halt all APs and let BSP do reset ops in user level.
Will post patch to Qemu to support SMP reboot some time later.

>
> The test for vcpu->requests already exists (and is needed for tlb
> flushes) so there is no additional performance hit.

OK, that makes sense. changed. please verify.
User level is split into libkvm and qemu.

thx,eddie

commit a0aed7040befe198491254d3459aeb5897f5f2c1
Author: root <root@vt32-pae.(none)>
Date: Wed Oct 10 13:45:58 2007 +0800

Add VM reset support in kernel side to
reset the kernel devices and force APs
to enter wait for INIT/SIPI state.

Signed-off-by: Yaozu (Eddie) Dong <eddie.dong-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 4ab487c..81c9af1 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -68,6 +68,7 @@
* vcpu->requests bit members
*/
#define KVM_TLB_FLUSH 0
+#define KVM_FROZEN 1

/*
* Address types:
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 0b2894a..33f16bd 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -2189,9 +2189,17 @@ again:

vcpu->guest_mode = 1;

- if (vcpu->requests)
+ if (vcpu->requests) {
if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
kvm_x86_ops->tlb_flush(vcpu);
+ if (test_and_clear_bit(KVM_FROZEN, &vcpu->requests)) {
+ local_irq_enable();
+ preempt_enable();
+ r = -EINTR;
+ kvm_run->exit_reason = KVM_EXIT_FROZEN;
+ goto out;
+ }
+ }

kvm_x86_ops->run(vcpu, kvm_run);

@@ -2245,6 +2253,8 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu
*vcpu, struct kvm_run *kvm_run)
vcpu_load(vcpu);

if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
+ if (irqchip_in_kernel(vcpu->kvm) && vcpu->apic)
+ kvm_lapic_reset(vcpu);
kvm_vcpu_block(vcpu);
vcpu_put(vcpu);
return -EAGAIN;
@@ -3143,6 +3153,54 @@ out:
return r;
}

+/*
+ * Reset kernel devices.
+ */
+void kvm_reset_devices(struct kvm *kvm)
+{
+ kvm_pic_reset(&pic_irqchip(kvm)->pics[1]);
+ kvm_pic_reset(&pic_irqchip(kvm)->pics[0]);
+ pic_irqchip(kvm)->output = 0;
+ kvm_ioapic_reset(kvm->vioapic);
+}
+
+/*
+ * Kernel side VM Reset.
+ * NOTE here: User level code must guarantee only the BSP
+ * thread can do this call.
+ *
+ */
+int kvm_vm_reset(struct kvm *kvm)
+{
+ struct kvm_vcpu *vcpu;
+ int i;
+
+ for (i = 0; i < KVM_MAX_VCPUS; i++) {
+ vcpu = kvm->vcpus[i];
+ if (!vcpu)
+ continue;
+ /* active VCPU */
+ if (vcpu->vcpu_id) {
+ vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
+ set_bit(KVM_FROZEN, &vcpu->requests);
+ kvm_vcpu_kick(vcpu);
+ /*
+ * Wait till the AP entered waiting for
+ * INIT/SIPI state
+ */
+ while (test_bit(KVM_FROZEN, &vcpu->requests))
+ schedule();
+ }
+ else {
+ vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
+ kvm_lapic_reset(vcpu);
+ }
+ }
+ /* Now only BSP is running... */
+ kvm_reset_devices(kvm);
+ return 0;
+}
+
static long kvm_vm_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
@@ -3228,6 +3286,12 @@ static long kvm_vm_ioctl(struct file *filp,
} else
goto out;
break;
+ case KVM_RESET:
+ r = -EINVAL;
+ if (!irqchip_in_kernel(kvm))
+ goto out;
+ r = kvm_vm_reset(kvm);
+ break;
case KVM_IRQ_LINE: {
struct kvm_irq_level irq_event;

diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index cd4ac12..ea9c004 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -127,7 +127,8 @@ enum kvm_exit_reason {
KVM_EXIT_SHUTDOWN = 8,
KVM_EXIT_FAIL_ENTRY = 9,
KVM_EXIT_INTR = 10,
- KVM_EXIT_SET_TPR = 11
+ KVM_EXIT_SET_TPR = 11,
+ KVM_EXIT_FROZEN = 12
};

/* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
@@ -383,6 +384,7 @@ struct kvm_signal_mask {
#define KVM_IRQ_LINE _IOW(KVMIO, 0x61, struct
kvm_irq_level)
#define KVM_GET_IRQCHIP _IOWR(KVMIO, 0x62, struct
kvm_irqchip)
#define KVM_SET_IRQCHIP _IOR(KVMIO, 0x63, struct
kvm_irqchip)
+#define KVM_RESET _IO(KVMIO, 0x64)

/*
* ioctls for vcpu fds

[-- Attachment #2: rbt-u2.patch --]
[-- Type: application/octet-stream, Size: 748 bytes --]

commit 50ddc4b612976847715a81da61e3dce5b486d49e
Author: root <root@vt32-pae.(none)>
Date: Wed Oct 10 13:54:58 2007 +0800

Add callback for kernel reset operation.

Signed-off-by: Yaozu (Eddie) Dong <eddie.dong@intel.com>

diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index 4d0bb93..d81e7e1 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -721,6 +721,10 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size, int boot_device,
vmport_init(env);
}

+#ifdef USE_KVM
+ if (kvm_allowed && kvm_irqchip_in_kernel(kvm_context))
+ qemu_register_reset(kvm_reset_kernel, kvm_context);
+#endif
/* allocate RAM */
ram_addr = qemu_ram_alloc(ram_size);
cpu_register_physical_memory(0, ram_size, ram_addr);

[-- Attachment #3: rbt-u1.patch --]
[-- Type: application/octet-stream, Size: 1334 bytes --]

commit 40dd3641611d585a914450bdf4410c626eeaf563
Author: root <root@vt32-pae.(none)>
Date: Wed Oct 10 13:56:59 2007 +0800

Add kernel reset support API and a handler for
KVM_EXIT_FROZEN.

Signed-off-by: Yaozu (Eddie) Dong <eddie.dong@intel.com>

diff --git a/user/kvmctl.c b/user/kvmctl.c
index 5f63c64..e5370da 100644
--- a/user/kvmctl.c
+++ b/user/kvmctl.c
@@ -1335,6 +1335,8 @@ again:
break;
case KVM_EXIT_SET_TPR:
break;
+ case KVM_EXIT_FROZEN:
+ break;
default:
fprintf(stderr, "unhandled vm exit: 0x%x\n", run->exit_reason);
kvm_show_regs(kvm, vcpu);
@@ -1407,3 +1409,10 @@ int kvm_irqchip_in_kernel(kvm_context_t kvm)
{
return kvm->irqchip_in_kernel;
}
+
+void kvm_reset_kernel(kvm_context_t kvm)
+{
+ if (kvm_irqchip_in_kernel(kvm))
+ return ioctl(kvm->vm_fd, KVM_RESET, 0);
+}
+
diff --git a/user/kvmctl.h b/user/kvmctl.h
index 09cf6d7..9f56027 100644
--- a/user/kvmctl.h
+++ b/user/kvmctl.h
@@ -453,6 +453,13 @@ int kvm_dirty_pages_log_enable_all(kvm_context_t kvm);
int kvm_dirty_pages_log_reset(kvm_context_t kvm);

/*!
+ * \brief Reset kernel devices.
+ *
+ * \param kvm Pointer to the current kvm_context
+ */
+void kvm_reset_kernel(kvm_context_t kvm);
+
+/*!
* \brief Query whether in kernel irqchip is used
*
* \param kvm Pointer to the current kvm_context

[-- Attachment #4: rbt-k3.patch --]
[-- Type: application/octet-stream, Size: 3662 bytes --]

commit a0aed7040befe198491254d3459aeb5897f5f2c1
Author: root <root@vt32-pae.(none)>
Date: Wed Oct 10 13:45:58 2007 +0800

Add VM reset support in kernel side to
reset the kernel devices and force APs
to enter wait for INIT/SIPI state.

Signed-off-by: Yaozu (Eddie) Dong <eddie.dong@intel.com>

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 4ab487c..81c9af1 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -68,6 +68,7 @@
* vcpu->requests bit members
*/
#define KVM_TLB_FLUSH 0
+#define KVM_FROZEN 1

/*
* Address types:
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 0b2894a..33f16bd 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -2189,9 +2189,17 @@ again:

vcpu->guest_mode = 1;

- if (vcpu->requests)
+ if (vcpu->requests) {
if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
kvm_x86_ops->tlb_flush(vcpu);
+ if (test_and_clear_bit(KVM_FROZEN, &vcpu->requests)) {
+ local_irq_enable();
+ preempt_enable();
+ r = -EINTR;
+ kvm_run->exit_reason = KVM_EXIT_FROZEN;
+ goto out;
+ }
+ }

kvm_x86_ops->run(vcpu, kvm_run);

@@ -2245,6 +2253,8 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
vcpu_load(vcpu);

if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
+ if (irqchip_in_kernel(vcpu->kvm) && vcpu->apic)
+ kvm_lapic_reset(vcpu);
kvm_vcpu_block(vcpu);
vcpu_put(vcpu);
return -EAGAIN;
@@ -3143,6 +3153,54 @@ out:
return r;
}

+/*
+ * Reset kernel devices.
+ */
+void kvm_reset_devices(struct kvm *kvm)
+{
+ kvm_pic_reset(&pic_irqchip(kvm)->pics[1]);
+ kvm_pic_reset(&pic_irqchip(kvm)->pics[0]);
+ pic_irqchip(kvm)->output = 0;
+ kvm_ioapic_reset(kvm->vioapic);
+}
+
+/*
+ * Kernel side VM Reset.
+ * NOTE here: User level code must guarantee only the BSP
+ * thread can do this call.
+ *
+ */
+int kvm_vm_reset(struct kvm *kvm)
+{
+ struct kvm_vcpu *vcpu;
+ int i;
+
+ for (i = 0; i < KVM_MAX_VCPUS; i++) {
+ vcpu = kvm->vcpus[i];
+ if (!vcpu)
+ continue;
+ /* active VCPU */
+ if (vcpu->vcpu_id) {
+ vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
+ set_bit(KVM_FROZEN, &vcpu->requests);
+ kvm_vcpu_kick(vcpu);
+ /*
+ * Wait till the AP entered waiting for
+ * INIT/SIPI state
+ */
+ while (test_bit(KVM_FROZEN, &vcpu->requests))
+ schedule();
+ }
+ else {
+ vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
+ kvm_lapic_reset(vcpu);
+ }
+ }
+ /* Now only BSP is running... */
+ kvm_reset_devices(kvm);
+ return 0;
+}
+
static long kvm_vm_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
@@ -3228,6 +3286,12 @@ static long kvm_vm_ioctl(struct file *filp,
} else
goto out;
break;
+ case KVM_RESET:
+ r = -EINVAL;
+ if (!irqchip_in_kernel(kvm))
+ goto out;
+ r = kvm_vm_reset(kvm);
+ break;
case KVM_IRQ_LINE: {
struct kvm_irq_level irq_event;

diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index cd4ac12..ea9c004 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -127,7 +127,8 @@ enum kvm_exit_reason {
KVM_EXIT_SHUTDOWN = 8,
KVM_EXIT_FAIL_ENTRY = 9,
KVM_EXIT_INTR = 10,
- KVM_EXIT_SET_TPR = 11
+ KVM_EXIT_SET_TPR = 11,
+ KVM_EXIT_FROZEN = 12
};

/* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
@@ -383,6 +384,7 @@ struct kvm_signal_mask {
#define KVM_IRQ_LINE _IOW(KVMIO, 0x61, struct kvm_irq_level)
#define KVM_GET_IRQCHIP _IOWR(KVMIO, 0x62, struct kvm_irqchip)
#define KVM_SET_IRQCHIP _IOR(KVMIO, 0x63, struct kvm_irqchip)
+#define KVM_RESET _IO(KVMIO, 0x64)

/*
* ioctls for vcpu fds

[-- Attachment #5: Type: text/plain, Size: 314 bytes --]

-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/

[-- Attachment #6: Type: text/plain, Size: 186 bytes --]

_______________________________________________
kvm-devel mailing list
kvm-devel-5NWGOfrQmneRv+LV9MX5uipxlwaOVQ5f@public.gmane.org