[PATCH v2 0/4] Implement CPU hotplug on Arm

[PATCH v2 0/4] Implement CPU hotplug on Arm

[Mykyta Poturai]

This series implements support for CPU hotplug/unplug on Arm. To achieve this,
several things need to be done:

1. XEN_SYSCTL_CPU_HOTPLUG_* calls implemented.
2. timer and GIC maintenance interrupts switched to static irqactions to remove
the need for freeing them during release_irq.
3. Enabled the build of xen-hptool on Arm.

Tested on QEMU.

v1->v2:
* see individual patches

Mykyta Poturai (4):
  arm/time: Use static irqaction
  arm/gic: Use static irqaction
  arm/sysctl: Implement cpu hotplug ops
  tools: Allow building xen-hptool without CONFIG_MIGRATE

 config/Tools.mk.in               |  1 +
 tools/configure                  | 30 +++++++++++++++++++++
 tools/configure.ac               |  1 +
 tools/libs/guest/Makefile.common |  4 +++
 tools/misc/Makefile              |  2 +-
 xen/arch/arm/gic.c               | 11 ++++++--
 xen/arch/arm/sysctl.c            | 45 ++++++++++++++++++++++++++++++++
 xen/arch/arm/time.c              | 21 ++++++++++++---
 8 files changed, 108 insertions(+), 7 deletions(-)
 mode change 100755 => 100644 tools/configure

-- 
2.34.1

[PATCH v2 1/4] arm/time: Use static irqaction

[Mykyta Poturai]

When stopping a core deinit_timer_interrupt is called in non-alloc
context, which causes xfree in release_irq to fail an assert.

To fix this, switch to a statically allocated irqaction that does not
need to be freed in release_irq.

Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>

v1->v2:
* Use percpu actions
---
 xen/arch/arm/time.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/xen/arch/arm/time.c b/xen/arch/arm/time.c
index e74d30d258..59349467de 100644
--- a/xen/arch/arm/time.c
+++ b/xen/arch/arm/time.c
@@ -303,9 +303,15 @@ static void check_timer_irq_cfg(unsigned int irq, const char *which)
            "WARNING: %s-timer IRQ%u is not level triggered.\n", which, irq);
 }
 
+DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_hyp);
+DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_virt);
+
 /* Set up the timer interrupt on this CPU */
 void init_timer_interrupt(void)
 {
+    struct irqaction *hyp_action = &this_cpu(irq_hyp);
+    struct irqaction *virt_action = &this_cpu(irq_virt);
+
     /* Sensible defaults */
     WRITE_SYSREG64(0, CNTVOFF_EL2);     /* No VM-specific offset */
     /* Do not let the VMs program the physical timer, only read the physical counter */
@@ -314,10 +320,17 @@ void init_timer_interrupt(void)
     WRITE_SYSREG(0, CNTHP_CTL_EL2);   /* Hypervisor's timer disabled */
     isb();
 
-    request_irq(timer_irq[TIMER_HYP_PPI], 0, htimer_interrupt,
-                "hyptimer", NULL);
-    request_irq(timer_irq[TIMER_VIRT_PPI], 0, vtimer_interrupt,
-                   "virtimer", NULL);
+    hyp_action->name = "hyptimer";
+    hyp_action->handler = htimer_interrupt;
+    hyp_action->dev_id = NULL;
+    hyp_action->free_on_release = 0;
+    setup_irq(timer_irq[TIMER_HYP_PPI], 0, hyp_action);
+
+    virt_action->name = "virtimer";
+    virt_action->handler = vtimer_interrupt;
+    virt_action->dev_id = NULL;
+    virt_action->free_on_release = 0;
+    setup_irq(timer_irq[TIMER_VIRT_PPI], 0, virt_action);
 
     check_timer_irq_cfg(timer_irq[TIMER_HYP_PPI], "hypervisor");
     check_timer_irq_cfg(timer_irq[TIMER_VIRT_PPI], "virtual");
-- 
2.34.1

[PATCH v2 2/4] arm/gic: Use static irqaction

[Mykyta Poturai]

When stopping a core cpu_gic_callback is called in non-alloc
context, which causes xfree in release_irq to fail an assert.

To fix this, switch to a statically allocated irqaction that does not
need to be freed in release_irq.

Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>

v1->v2:
* use percpu actions
---
 xen/arch/arm/gic.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index 260ee64cca..ed6853bb32 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -386,10 +386,17 @@ void gic_dump_info(struct vcpu *v)
     gic_hw_ops->dump_state(v);
 }
 
+DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_maintenance);
+
 void init_maintenance_interrupt(void)
 {
-    request_irq(gic_hw_ops->info->maintenance_irq, 0, maintenance_interrupt,
-                "irq-maintenance", NULL);
+    struct irqaction *maintenance = &this_cpu(irq_maintenance);
+
+    maintenance->name = "irq-maintenance";
+    maintenance->handler = maintenance_interrupt;
+    maintenance->dev_id = NULL;
+    maintenance->free_on_release = 0;
+    setup_irq(gic_hw_ops->info->maintenance_irq, 0, maintenance);
 }
 
 int gic_make_hwdom_dt_node(const struct domain *d,
-- 
2.34.1

[PATCH v2 3/4] arm/sysctl: Implement cpu hotplug ops

[Mykyta Poturai]

Implement XEN_SYSCTL_CPU_HOTPLUG_{ONLINE,OFFLINE} calls to allow for
enabling/disabling CPU cores in runtime.

Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>

v1->v2:
* remove SMT ops
* remove cpu == 0 checks
* add XSM hooks
* only implement for 64bit Arm
---
 xen/arch/arm/sysctl.c | 45 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/xen/arch/arm/sysctl.c b/xen/arch/arm/sysctl.c
index 32cab4feff..fecd649db1 100644
--- a/xen/arch/arm/sysctl.c
+++ b/xen/arch/arm/sysctl.c
@@ -12,6 +12,8 @@
 #include <xen/dt-overlay.h>
 #include <xen/errno.h>
 #include <xen/hypercall.h>
+#include <xen/cpu.h>
+#include <xsm/xsm.h>
 #include <asm/arm64/sve.h>
 #include <public/sysctl.h>
 
@@ -23,6 +25,42 @@ void arch_do_physinfo(struct xen_sysctl_physinfo *pi)
                                        XEN_SYSCTL_PHYSCAP_ARM_SVE_MASK);
 }
 
+#ifdef CONFIG_ARM_64
+static long cpu_up_helper(void *data)
+{
+    unsigned long cpu = (unsigned long) data;
+    return cpu_up(cpu);
+}
+
+static long cpu_down_helper(void *data)
+{
+    unsigned long cpu = (unsigned long) data;
+    return cpu_down(cpu);
+}
+
+static long cpu_hotplug_sysctl(struct xen_sysctl_cpu_hotplug *hotplug)
+{
+    int ret;
+
+    switch (hotplug->op) {
+        case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
+            ret = xsm_resource_plug_core(XSM_HOOK);
+            if ( ret )
+                return ret;
+            return continue_hypercall_on_cpu(0, cpu_up_helper, _p(hotplug->cpu));
+
+        case XEN_SYSCTL_CPU_HOTPLUG_OFFLINE:
+            ret = xsm_resource_unplug_core(XSM_HOOK);
+            if ( ret )
+                return ret;
+            return continue_hypercall_on_cpu(0, cpu_down_helper, _p(hotplug->cpu));
+
+        default:
+            return -EOPNOTSUPP;
+    }
+}
+#endif
+
 long arch_do_sysctl(struct xen_sysctl *sysctl,
                     XEN_GUEST_HANDLE_PARAM(xen_sysctl_t) u_sysctl)
 {
@@ -34,6 +72,13 @@ long arch_do_sysctl(struct xen_sysctl *sysctl,
         ret = dt_overlay_sysctl(&sysctl->u.dt_overlay);
         break;
 
+/* CPU Hotplug only implemented for 64-bit Arm */
+#ifdef CONFIG_ARM_64
+    case XEN_SYSCTL_cpu_hotplug:
+        ret = cpu_hotplug_sysctl(&sysctl->u.cpu_hotplug);
+        break;
+#endif
+
     default:
         ret = -ENOSYS;
         break;
-- 
2.34.1

[PATCH v2 4/4] tools: Allow building xen-hptool without CONFIG_MIGRATE

[Mykyta Poturai]

With CPU hotplug sysctls implemented on Arm it becomes useful to have a
tool for calling them. Introduce a new congifure option "hptool" to
allow building hptool separately from other migration tools, and enable
it by default.

Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>

v1->v2:
* switch to configure from legacy config
---
 config/Tools.mk.in               |  1 +
 tools/configure                  | 30 ++++++++++++++++++++++++++++++
 tools/configure.ac               |  1 +
 tools/libs/guest/Makefile.common |  4 ++++
 tools/misc/Makefile              |  2 +-
 5 files changed, 37 insertions(+), 1 deletion(-)
 mode change 100755 => 100644 tools/configure

diff --git a/config/Tools.mk.in b/config/Tools.mk.in
index e47ac23d11..eb4855d93d 100644
--- a/config/Tools.mk.in
+++ b/config/Tools.mk.in
@@ -49,6 +49,7 @@ CONFIG_LIBNL        := @libnl@
 CONFIG_GOLANG       := @golang@
 CONFIG_PYGRUB       := @pygrub@
 CONFIG_LIBFSIMAGE   := @libfsimage@
+CONFIG_HPTOOL       := @hptool@
 
 CONFIG_SYSTEMD      := @systemd@
 XEN_SYSTEMD_DIR     := @SYSTEMD_DIR@
diff --git a/tools/configure b/tools/configure
old mode 100755
new mode 100644
index 5abd44e21e..5cf5381c0a
--- a/tools/configure
+++ b/tools/configure
@@ -728,6 +728,7 @@ LD86
 AS86
 ipxe
 LINUX_BACKEND_MODULES
+hptool
 pygrub
 golang
 seabios
@@ -834,6 +835,7 @@ enable_ovmf
 enable_seabios
 enable_golang
 enable_pygrub
+enable_hptool
 with_linux_backend_modules
 enable_ipxe
 with_system_ipxe
@@ -1519,6 +1521,7 @@ Optional Features:
   --disable-seabios       Disable SeaBIOS (default is ENABLED)
   --disable-golang        Disable Go tools (default is ENABLED)
   --disable-pygrub        Disable pygrub (default is ENABLED)
+  --disable-hptool        Disable hptool (default is ENABLED)
   --enable-ipxe           Enable in-tree IPXE, (DEFAULT is off, see also
                           --with-system-ipxe)
   --enable-rombios        Enable ROMBIOS, (DEFAULT is on if ipxe is enabled,
@@ -4807,6 +4810,33 @@ pygrub=$ax_cv_pygrub
 
 
 
+# Check whether --enable-hptool was given.
+if test ${enable_hptool+y}
+then :
+  enableval=$enable_hptool;
+fi
+
+
+if test "x$enable_hptool" = "xno"
+then :
+
+    ax_cv_hptool="n"
+
+elif test "x$enable_hptool" = "xyes"
+then :
+
+    ax_cv_hptool="y"
+
+elif test -z $ax_cv_hptool
+then :
+
+    ax_cv_hptool="y"
+
+fi
+hptool=$ax_cv_hptool
+
+
+
 
 # Check whether --with-linux-backend-modules was given.
 if test ${with_linux_backend_modules+y}
diff --git a/tools/configure.ac b/tools/configure.ac
index dada1c3b15..3a0644ef89 100644
--- a/tools/configure.ac
+++ b/tools/configure.ac
@@ -90,6 +90,7 @@ AX_ARG_DEFAULT_DISABLE([ovmf], [Enable OVMF])
 AX_ARG_DEFAULT_ENABLE([seabios], [Disable SeaBIOS])
 AX_ARG_DEFAULT_ENABLE([golang], [Disable Go tools])
 AX_ARG_DEFAULT_ENABLE([pygrub], [Disable pygrub])
+AX_ARG_DEFAULT_ENABLE([hptool], [Disable hptool])
 
 AC_ARG_WITH([linux-backend-modules],
     AS_HELP_STRING([--with-linux-backend-modules="mod1 mod2"],
diff --git a/tools/libs/guest/Makefile.common b/tools/libs/guest/Makefile.common
index a026a2f662..774b1d5392 100644
--- a/tools/libs/guest/Makefile.common
+++ b/tools/libs/guest/Makefile.common
@@ -25,6 +25,10 @@ OBJS-y       += xg_core.o
 OBJS-$(CONFIG_X86) += xg_core_x86.o
 OBJS-$(CONFIG_ARM) += xg_core_arm.o
 
+ifneq (,$(filter y,$(CONFIG_MIGRATE)$(CONFIG_HPTOOL)))
+OBJS-y += xg_offline_page.o
+endif
+
 vpath %.c ../../../xen/common/libelf
 
 LIBELF_OBJS += libelf-tools.o libelf-loader.o
diff --git a/tools/misc/Makefile b/tools/misc/Makefile
index c26e544e83..f783f16ae6 100644
--- a/tools/misc/Makefile
+++ b/tools/misc/Makefile
@@ -16,7 +16,7 @@ INSTALL_BIN                    += xencov_split
 INSTALL_BIN += $(INSTALL_BIN-y)
 
 # Everything to be installed in regular sbin/
-INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
+INSTALL_SBIN-$(CONFIG_HPTOOL)  += xen-hptool
 INSTALL_SBIN-$(CONFIG_X86)     += xen-hvmcrash
 INSTALL_SBIN-$(CONFIG_X86)     += xen-hvmctx
 INSTALL_SBIN-$(CONFIG_X86)     += xen-lowmemd
-- 
2.34.1