[PATCH v2] x86/smt: Support for enabling/disabling SMT at runtime

* [PATCH v2] x86/smt: Support for enabling/disabling SMT at runtime
@ 2019-04-08 17:02 Andrew Cooper
  2019-04-09 12:14 ` Jan Beulich
  2019-04-09 12:36 ` Wei Liu
  0 siblings, 2 replies; 4+ messages in thread
From: Andrew Cooper @ 2019-04-08 17:02 UTC (permalink / raw)
  To: Xen-devel; +Cc: Andrew Cooper, Wei Liu, Jan Beulich, Roger Pau Monné

Currently, a user can in combine the output of `xl info -n`, the APCI tables,
and some manual CPUID data to figure out which CPU numbers to feed into
`xen-hptool cpu-offline` to effectively disable SMT at runtime.

A more convenient option is to teach Xen how to perform this action.

Extend XEN_SYSCTL_cpu_hotplug with two new operations.  Introduce a new
smt_up_down_helper() which wrap the cpu_{up,down}_helper() helpers with logic
which understands siblings based on their APIC_ID.

Add libxc stubs, and extend xen-hptool with smt-{enable,disable} options.
These are intended to be shorthands for a loop over cpu-{online,offline}.  It
is intended for use in production scenarios where debugging options such as
`maxcpus=` or other manual plug/unplug configuration has not been used.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
CC: Jan Beulich <JBeulich@suse.com>
CC: Wei Liu <wei.liu2@citrix.com>
CC: Roger Pau Monné <roger.pau@citrix.com>

v2:
 * Fold smt_{up,down}_helper() into smt_up_down_helper()
 * Fix calculation of sibling_mask
 * More clearly document the behaviour and expected restrictions.

While testing this, I've found a further bug.  This is preexisting and can be
provoked on staging, although it is easier to provoke with this patch.

(XEN) Assertion 'entry->prev->next == entry' failed at /local/xen.git/xen/include/xen/list.h:184
(XEN) ----[ Xen-4.13-unstable  x86_64  debug=y   Not tainted ]----
(XEN) CPU:    0
(XEN) RIP:    e008:[<ffff82d080241fad>] migrate_timer+0x1df/0x2da
(XEN) RFLAGS: 0000000000010087   CONTEXT: hypervisor
(XEN) rax: ffff830837088038   rbx: ffff830575928890   rcx: ffff83084d69e320
(XEN) rdx: ffff830575928898   rsi: ffff82d08094002f   rdi: ffff830575928890
(XEN) rbp: ffff8300abc27d28   rsp: ffff8300abc27cc8   r8:  0000000000000000
(XEN) r9:  ffff830575928780   r10: ffff830575928010   r11: 0000000000000000
(XEN) r12: 0000000000000000   r13: 0000000000000000   r14: ffff82d080962080
(XEN) r15: ffff82d080962080   cr0: 000000008005003b   cr4: 00000000001526e0
(XEN) cr3: 00000000abc17000   cr2: ffff880072a9f8c0
(XEN) fsb: 0000000000000000   gsb: ffff88007ce00000   gss: 0000000000000000
(XEN) ds: 0000   es: 0000   fs: 0000   gs: 0000   ss: e010   cs: e008
(XEN) Xen code around <ffff82d080241fad> (migrate_timer+0x1df/0x2da):
(XEN)  8b 4b 10 48 3b 11 74 02 <0f> 0b 48 89 48 08 48 89 01 48 b8 00 01 10 00 01
(XEN) Xen stack trace from rsp=ffff8300abc27cc8:
<snip>
(XEN) Xen call trace:
(XEN)    [<ffff82d080241fad>] migrate_timer+0x1df/0x2da
(XEN)    [<ffff82d0802998b7>] do_IRQ+0x4fc/0x64e
(XEN)    [<ffff82d0803988aa>] common_interrupt+0x10a/0x120
(XEN)    [<ffff82d0802f861c>] mwait-idle.c#mwait_idle+0x31d/0x370
(XEN)    [<ffff82d080289499>] domain.c#idle_loop+0xa8/0xc3
(XEN)
(XEN)
(XEN) ****************************************
(XEN) Panic on CPU 0:
(XEN) Assertion 'entry->prev->next == entry' failed at /local/xen.git/xen/include/xen/list.h:184
(XEN) ****************************************
---
 tools/libxc/include/xenctrl.h |  2 ++
 tools/libxc/xc_cpu_hotplug.c  | 26 ++++++++++++++++++++
 tools/misc/xen-hptool.c       | 56 ++++++++++++++++++++++++++++++++++++++++++
 xen/arch/x86/sysctl.c         | 57 ++++++++++++++++++++++++++++++++++++++++++-
 xen/include/public/sysctl.h   | 19 +++++++++++++++
 5 files changed, 159 insertions(+), 1 deletion(-)

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index a3628e5..49a6b2a 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -1854,6 +1854,8 @@ int xc_pm_reset_cxstat(xc_interface *xch, int cpuid);
 
 int xc_cpu_online(xc_interface *xch, int cpu);
 int xc_cpu_offline(xc_interface *xch, int cpu);
+int xc_smt_enable(xc_interface *xch);
+int xc_smt_disable(xc_interface *xch);
 
 /* 
  * cpufreq para name of this structure named 
diff --git a/tools/libxc/xc_cpu_hotplug.c b/tools/libxc/xc_cpu_hotplug.c
index 58c2a0f..2ea9825 100644
--- a/tools/libxc/xc_cpu_hotplug.c
+++ b/tools/libxc/xc_cpu_hotplug.c
@@ -46,3 +46,29 @@ int xc_cpu_offline(xc_interface *xch, int cpu)
     return ret;
 }
 
+int xc_smt_enable(xc_interface *xch)
+{
+    DECLARE_SYSCTL;
+    int ret;
+
+    sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+    sysctl.u.cpu_hotplug.cpu = 0;
+    sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+    ret = xc_sysctl(xch, &sysctl);
+
+    return ret;
+}
+
+int xc_smt_disable(xc_interface *xch)
+{
+    DECLARE_SYSCTL;
+    int ret;
+
+    sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+    sysctl.u.cpu_hotplug.cpu = 0;
+    sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE;
+    ret = xc_sysctl(xch, &sysctl);
+
+    return ret;
+}
+
diff --git a/tools/misc/xen-hptool.c b/tools/misc/xen-hptool.c
index 40cd966..6e27d9c 100644
--- a/tools/misc/xen-hptool.c
+++ b/tools/misc/xen-hptool.c
@@ -19,6 +19,8 @@ void show_help(void)
             "  mem-online    <mfn>      online MEMORY <mfn>\n"
             "  mem-offline   <mfn>      offline MEMORY <mfn>\n"
             "  mem-status    <mfn>      query Memory status<mfn>\n"
+            "  smt-enable               onlines all SMT threads\n"
+            "  smt-disable              offlines all SMT threads\n"
            );
 }
 
@@ -304,6 +306,58 @@ static int hp_cpu_offline_func(int argc, char *argv[])
     return ret;
 }
 
+static int main_smt_enable(int argc, char *argv[])
+{
+    int ret;
+
+    if ( argc )
+    {
+        show_help();
+        return -1;
+    }
+
+    for ( ;; )
+    {
+        ret = xc_smt_enable(xch);
+        if ( (ret >= 0) || (errno != EBUSY) )
+            break;
+    }
+
+    if ( ret < 0 )
+        fprintf(stderr, "Unable to enable SMT: errno %d, %s\n",
+                errno, strerror(errno));
+    else
+        printf("Enabled SMT\n");
+
+    return ret;
+}
+
+static int main_smt_disable(int argc, char *argv[])
+{
+    int ret;
+
+    if ( argc )
+    {
+        show_help();
+        return -1;
+    }
+
+    for ( ;; )
+    {
+        ret = xc_smt_disable(xch);
+        if ( (ret >= 0) || (errno != EBUSY) )
+            break;
+    }
+
+    if ( ret < 0 )
+        fprintf(stderr, "Unable to disable SMT: errno %d, %s\n",
+                errno, strerror(errno));
+    else
+        printf("Disabled SMT\n");
+
+    return ret;
+}
+
 struct {
     const char *name;
     int (*function)(int argc, char *argv[]);
@@ -314,6 +368,8 @@ struct {
     { "mem-status", hp_mem_query_func},
     { "mem-online", hp_mem_online_func},
     { "mem-offline", hp_mem_offline_func},
+    { "smt-enable", main_smt_enable },
+    { "smt-disable", main_smt_disable },
 };
 
 
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index cff4415..8acb5f8 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -114,6 +114,48 @@ long cpu_down_helper(void *data)
     return ret;
 }
 
+static long smt_up_down_helper(void *data)
+{
+    bool up = (bool)data;
+    unsigned int cpu, sibling_mask = boot_cpu_data.x86_num_siblings - 1;
+    int ret = 0;
+
+    opt_smt = up;
+
+    for_each_present_cpu ( cpu )
+    {
+        /* Skip primary siblings (those whose thread id is 0). */
+        if ( !(x86_cpu_to_apicid[cpu] & sibling_mask) )
+            continue;
+
+        ret = up ? cpu_up_helper(_p(cpu))
+                 : cpu_down_helper(_p(cpu));
+
+        if ( ret && ret != -EEXIST )
+            break;
+
+        /*
+         * Ensure forward progress by only considering preemption when we have
+         * changed the state of one or more cpus.
+         */
+        if ( ret != -EEXIST && general_preempt_check() )
+        {
+            /* In tasklet context - can't create a contination. */
+            ret = -EBUSY;
+            break;
+        }
+
+        ret = 0; /* Avoid exiting with -EEXIST in the success case. */
+    }
+
+    if ( !ret )
+        printk(XENLOG_INFO "SMT %s - online CPUs {%*pbl}\n",
+               up ? "enabled" : "disabled",
+               nr_cpu_ids, cpumask_bits(&cpu_online_map));
+
+    return ret;
+}
+
 void arch_do_physinfo(struct xen_sysctl_physinfo *pi)
 {
     memcpy(pi->hw_cap, boot_cpu_data.x86_capability,
@@ -137,11 +179,12 @@ long arch_do_sysctl(
     case XEN_SYSCTL_cpu_hotplug:
     {
         unsigned int cpu = sysctl->u.cpu_hotplug.cpu;
+        unsigned int op  = sysctl->u.cpu_hotplug.op;
         bool plug;
         long (*fn)(void *);
         void *hcpu;
 
-        switch ( sysctl->u.cpu_hotplug.op )
+        switch ( op )
         {
         case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
             plug = true;
@@ -155,6 +198,18 @@ long arch_do_sysctl(
             hcpu = _p(cpu);
             break;
 
+        case XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE:
+        case XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE:
+            if ( !cpu_has_htt || boot_cpu_data.x86_num_siblings < 2 )
+            {
+                ret = -EOPNOTSUPP;
+                break;
+            }
+            plug = op == XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+            fn = smt_up_down_helper;
+            hcpu = _p(plug);
+            break;
+
         default:
             ret = -EOPNOTSUPP;
             break;
diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
index c49b4dc..9e8903d 100644
--- a/xen/include/public/sysctl.h
+++ b/xen/include/public/sysctl.h
@@ -246,8 +246,27 @@ struct xen_sysctl_get_pmstat {
 struct xen_sysctl_cpu_hotplug {
     /* IN variables */
     uint32_t cpu;   /* Physical cpu. */
+
+    /* Single CPU enable/disable. */
 #define XEN_SYSCTL_CPU_HOTPLUG_ONLINE  0
 #define XEN_SYSCTL_CPU_HOTPLUG_OFFLINE 1
+
+    /*
+     * SMT enable/disable.
+     *
+     * These two ops loop over all present CPUs, and either online or offline
+     * every non-primary sibling thread (those with a thread id which is not
+     * 0).
+     *
+     * They are intended as a shorthand for identifying and feeding the cpu
+     * numbers individually to HOTPLUG_{ON,OFF}LINE.
+     *
+     * These are not expected to be used in conjunction with debugging options
+     * such as `maxcpus=` or when other manual configuration of offline cpus
+     * is in use.
+     */
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE  2
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE 3
     uint32_t op;    /* hotplug opcode */
 };
 
-- 
2.1.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

^ permalink raw reply related	[flat|nested] 4+ messages in thread