[PATCH RFC/WIPv2 0/6] toolstack-based approach to pvhvm guest kexec

[Vitaly Kuznetsov <vkuznets@redhat.com>]

When a PVHVM linux guest performs kexec there are lots of things which
require taking care of:
- shared info, vcpu_info
- grants
- event channels
- ...
Instead of taking care of all these things we can rebuild the domain
performing kexec from scratch doing so-called soft-reboot.

The idea was suggested by David Vrabel, Jan Beulich, and Konrad Rzeszutek Wilk.

Main RFC part:
I'm not sure my suggested XENMEM_transfer op is the right way to go:
- If we steal pages from a particular domain it should be dead as it makes no
  sense to it after such the call.
- we need to copy all L1-L4 pages, rebuild the shared info ... for PV. This
  will result in additional complexity in libxc (rebuilding the p2m).
- we also need to keep track of all copied L1-L4 pages for PV as we'll be
  re-creating p2m.

I can see three possible ways to go:
1) Forbid the call for PV and remove this part from libxc. The easiest way to
   go (and PV kexec/kdump will require additional work on kernel side anyway).
2) Go ahead and rebuild p2m in libxc (similar to what we have for save/restore
    path).
3) Instead of XENMEM_transfer introduce special oneshot domain kill op which
   will follow the same domain_kill path but instead of relinquishing resources
   it will reassign them. I suppose it will be posible to reassign L1-L4 pages
   and pages from xenheap here as well.

What would you say?

WIP part:
- PV support is broken.
- Not sure huge pages will work well.
- Not sure about ARM/PVH.
- Not tested with qemu-upstream.

P.S. The patch series can be tested with PVHVM Linux guest with the following
modifications:

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index c0cb11f..33c5cdd 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1810,6 +1814,22 @@ static struct notifier_block xen_hvm_cpu_notifier = {
   .notifier_call   = xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress) {
+	   xen_reboot(SHUTDOWN_soft_reset);
+	   }
+}
+
+static void xen_pvhvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
	init_hvm_pv_info();
@@ -1826,6 +1846,10 @@ static void __init xen_hvm_guest_init(void)
   x86_init.irqs.intr_init = xen_init_IRQ;
   xen_hvm_init_time_ops();
   xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC
+	machine_ops.shutdown = xen_pvhvm_kexec_shutdown;
+	machine_ops.crash_shutdown = xen_pvhvm_crash_shutdown;
+#endif
 }
 
 static bool xen_nopv = false;
diff --git a/include/xen/interface/sched.h b/include/xen/interface/sched.h
index 9ce0839..b5942a8 100644
--- a/include/xen/interface/sched.h
+++ b/include/xen/interface/sched.h
@@ -107,5 +107,6 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time expired.     */
+#define SHUTDOWN_soft_reset 5  /* Soft-reset for kexec.                      */
 
 #endif /* __XEN_PUBLIC_SCHED_H__ */

Vitaly Kuznetsov (6):
  Introduce XENMEM_transfer operation
  libxc: support XENMEM_transfer operation
  libxc: introduce soft reset
  xen: Introduce SHUTDOWN_soft_reset shutdown reason
  libxl: support SHUTDOWN_soft_reset shutdown reason
  libxl: soft reset support

 tools/libxc/Makefile               |   1 +
 tools/libxc/xc_domain.c            |  19 +++
 tools/libxc/xc_domain_soft_reset.c | 300 +++++++++++++++++++++++++++++++++++++
 tools/libxc/xenctrl.h              |   6 +
 tools/libxc/xenguest.h             |  19 +++
 tools/libxl/libxl.h                |   6 +
 tools/libxl/libxl_create.c         | 100 +++++++++++--
 tools/libxl/libxl_internal.h       |   5 +
 tools/libxl/libxl_types.idl        |   1 +
 tools/libxl/xl_cmdimpl.c           |  31 +++-
 tools/python/xen/lowlevel/xl/xl.c  |   1 +
 xen/common/memory.c                | 178 ++++++++++++++++++++++
 xen/common/shutdown.c              |   7 +
 xen/include/public/memory.h        |  32 +++-
 xen/include/public/sched.h         |   3 +-
 15 files changed, 695 insertions(+), 14 deletions(-)
 create mode 100644 tools/libxc/xc_domain_soft_reset.c

-- 
1.9.3