* [PATCH v3 1/9] x86/mce: handle LMCE locally
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
@ 2017-03-30 6:19 ` Haozhong Zhang
2017-03-30 14:35 ` Jan Beulich
2017-03-30 6:19 ` [PATCH v3 2/9] x86/mce_intel: detect and enable LMCE on Intel host Haozhong Zhang
` (7 subsequent siblings)
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:19 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Jan Beulich, Andrew Cooper
LMCE is sent to only one CPU thread, so MCE handler, barriers and
softirq handler should go without waiting for other CPUs, when
handling LMCE. Note LMCE is still broadcast to all vcpus as regular
MCE on Intel CPU right now.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Do not add nowait argument to currently unused mce_barrier().
* Do not access the global variable 'severity_cpu' when handling LMCE.
* Rename per-cpu flag 'mce_in_process' to 'nonlocal_mce_in_progress'
to clarify it's for non-local MCE.
* Add code comments for tricky points.
---
xen/arch/x86/cpu/mcheck/barrier.c | 12 ++--
xen/arch/x86/cpu/mcheck/barrier.h | 12 +++-
xen/arch/x86/cpu/mcheck/mcaction.c | 4 +-
xen/arch/x86/cpu/mcheck/mce.c | 122 ++++++++++++++++++++++++++++---------
xen/arch/x86/cpu/mcheck/mce.h | 2 +
xen/arch/x86/cpu/mcheck/x86_mca.h | 4 +-
6 files changed, 118 insertions(+), 38 deletions(-)
diff --git a/xen/arch/x86/cpu/mcheck/barrier.c b/xen/arch/x86/cpu/mcheck/barrier.c
index 5dce1fb..0b3b091 100644
--- a/xen/arch/x86/cpu/mcheck/barrier.c
+++ b/xen/arch/x86/cpu/mcheck/barrier.c
@@ -16,11 +16,11 @@ void mce_barrier_dec(struct mce_softirq_barrier *bar)
atomic_dec(&bar->val);
}
-void mce_barrier_enter(struct mce_softirq_barrier *bar)
+void mce_barrier_enter(struct mce_softirq_barrier *bar, bool nowait)
{
int gen;
- if (!mce_broadcast)
+ if ( !mce_broadcast || nowait )
return;
atomic_inc(&bar->ingen);
gen = atomic_read(&bar->outgen);
@@ -34,11 +34,11 @@ void mce_barrier_enter(struct mce_softirq_barrier *bar)
}
}
-void mce_barrier_exit(struct mce_softirq_barrier *bar)
+void mce_barrier_exit(struct mce_softirq_barrier *bar, bool nowait)
{
int gen;
- if ( !mce_broadcast )
+ if ( !mce_broadcast || nowait )
return;
atomic_inc(&bar->outgen);
gen = atomic_read(&bar->ingen);
@@ -54,6 +54,6 @@ void mce_barrier_exit(struct mce_softirq_barrier *bar)
void mce_barrier(struct mce_softirq_barrier *bar)
{
- mce_barrier_enter(bar);
- mce_barrier_exit(bar);
+ mce_barrier_enter(bar, false);
+ mce_barrier_exit(bar, false);
}
diff --git a/xen/arch/x86/cpu/mcheck/barrier.h b/xen/arch/x86/cpu/mcheck/barrier.h
index 87f7550..f2613e6 100644
--- a/xen/arch/x86/cpu/mcheck/barrier.h
+++ b/xen/arch/x86/cpu/mcheck/barrier.h
@@ -25,6 +25,14 @@ void mce_barrier_init(struct mce_softirq_barrier *);
void mce_barrier_dec(struct mce_softirq_barrier *);
/*
+ * If nowait is true, mce_barrier_enter/exit() will return immediately
+ * without touching the barrier. It's used when handling a LMCE which
+ * is received on only one CPU and thus does not invoke
+ * mce_barrier_enter/exit() calls on all CPUs.
+ *
+ * If nowait is false, mce_barrier_enter/exit() will handle the given
+ * barrier as below.
+ *
* Increment the generation number and the value. The generation number
* is incremented when entering a barrier. This way, it can be checked
* on exit if a CPU is trying to re-enter the barrier. This can happen
@@ -36,8 +44,8 @@ void mce_barrier_dec(struct mce_softirq_barrier *);
* These barrier functions should always be paired, so that the
* counter value will reach 0 again after all CPUs have exited.
*/
-void mce_barrier_enter(struct mce_softirq_barrier *);
-void mce_barrier_exit(struct mce_softirq_barrier *);
+void mce_barrier_enter(struct mce_softirq_barrier *, bool nowait);
+void mce_barrier_exit(struct mce_softirq_barrier *, bool nowait);
void mce_barrier(struct mce_softirq_barrier *);
diff --git a/xen/arch/x86/cpu/mcheck/mcaction.c b/xen/arch/x86/cpu/mcheck/mcaction.c
index dab9eac..ca17d22 100644
--- a/xen/arch/x86/cpu/mcheck/mcaction.c
+++ b/xen/arch/x86/cpu/mcheck/mcaction.c
@@ -96,7 +96,9 @@ mc_memerr_dhandler(struct mca_binfo *binfo,
bank->mc_addr = gfn << PAGE_SHIFT |
(bank->mc_addr & (PAGE_SIZE -1 ));
- if (fill_vmsr_data(bank, d, global->mc_gstatus,
+ /* TODO: support injecting LMCE */
+ if (fill_vmsr_data(bank, d,
+ global->mc_gstatus & ~MCG_STATUS_LMCE,
vmce_vcpuid == VMCE_INJECT_BROADCAST))
{
mce_printk(MCE_QUIET, "Fill vMCE# data for DOM%d "
diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c
index 11d0e23..c1a59bf 100644
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -42,6 +42,13 @@ DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, poll_bankmask);
DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, no_cmci_banks);
DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, mce_clear_banks);
+/*
+ * Flag to indicate that at least one non-local MCE on this CPU has
+ * not been completed handled. It's set by mcheck_cmn_handler() and
+ * cleared by mce_softirq().
+ */
+static DEFINE_PER_CPU(bool, nonlocal_mce_in_progress);
+
static void intpose_init(void);
static void mcinfo_clear(struct mc_info *);
struct mca_banks *mca_allbanks;
@@ -186,7 +193,29 @@ static struct mce_softirq_barrier mce_trap_bar;
*/
static DEFINE_SPINLOCK(mce_logout_lock);
+/*
+ * 'severity_cpu' is used in both mcheck_cmn_handler() and mce_softirq().
+ *
+ * When handling broadcasting MCE, MCE barriers take effect to prevent
+ * 'severity_cpu' being modified in one function on one CPU and accessed
+ * in another on a different CPU.
+ *
+ * When handling LMCE, mce_barrier_enter() and mce_barrier_exit() are
+ * effectively NOP, so it's possible that mcheck_cmn_handler() is handling
+ * a LMCE on CPUx while mce_softirq() is handling another LMCE on CPUy.
+ * If both are modifying 'severity_cpu', they may interfere with each
+ * other. Therefore, it's better for mcheck_cmn_handler() and mce_softirq()
+ * to avoid accessing 'severity_cpu' when handling LMCE, unless other
+ * approaches are taken to avoid the above interference.
+ */
static atomic_t severity_cpu = ATOMIC_INIT(-1);
+/*
+ * The following two global variables are used only in mcheck_cmn_handler().
+ * Because there can be at most one MCE (including LMCE) outstanding
+ * in the hardware platform, we don't need to worry about the above
+ * interference, where two code paths handing different MCE's access
+ * the same global variables.
+ */
static atomic_t found_error = ATOMIC_INIT(0);
static cpumask_t mce_fatal_cpus;
@@ -395,6 +424,7 @@ mcheck_mca_logout(enum mca_source who, struct mca_banks *bankmask,
sp->errcnt = errcnt;
sp->ripv = (gstatus & MCG_STATUS_RIPV) != 0;
sp->eipv = (gstatus & MCG_STATUS_EIPV) != 0;
+ sp->lmce = (gstatus & MCG_STATUS_LMCE) != 0;
sp->uc = uc;
sp->pcc = pcc;
sp->recoverable = recover;
@@ -458,6 +488,7 @@ void mcheck_cmn_handler(const struct cpu_user_regs *regs)
uint64_t gstatus;
mctelem_cookie_t mctc = NULL;
struct mca_summary bs;
+ bool lmce;
mce_spin_lock(&mce_logout_lock);
@@ -467,6 +498,10 @@ void mcheck_cmn_handler(const struct cpu_user_regs *regs)
}
mctc = mcheck_mca_logout(MCA_MCE_SCAN, bankmask, &bs, clear_bank);
+ lmce = bs.lmce;
+ if (!lmce)
+ this_cpu(nonlocal_mce_in_progress) = true;
+
if (bs.errcnt) {
/*
* Uncorrected errors must be dealt with in softirq context.
@@ -488,8 +523,9 @@ void mcheck_cmn_handler(const struct cpu_user_regs *regs)
}
atomic_set(&found_error, 1);
- /* The last CPU will be take check/clean-up etc */
- atomic_set(&severity_cpu, smp_processor_id());
+ /* The last CPU will be take check/clean-up etc. */
+ if (!lmce)
+ atomic_set(&severity_cpu, smp_processor_id());
mce_printk(MCE_CRITICAL, "MCE: clear_bank map %lx on CPU%d\n",
*((unsigned long*)clear_bank), smp_processor_id());
@@ -501,16 +537,16 @@ void mcheck_cmn_handler(const struct cpu_user_regs *regs)
}
mce_spin_unlock(&mce_logout_lock);
- mce_barrier_enter(&mce_trap_bar);
+ mce_barrier_enter(&mce_trap_bar, lmce);
if ( mctc != NULL && mce_urgent_action(regs, mctc))
cpumask_set_cpu(smp_processor_id(), &mce_fatal_cpus);
- mce_barrier_exit(&mce_trap_bar);
+ mce_barrier_exit(&mce_trap_bar, lmce);
/*
* Wait until everybody has processed the trap.
*/
- mce_barrier_enter(&mce_trap_bar);
- if (atomic_read(&severity_cpu) == smp_processor_id())
+ mce_barrier_enter(&mce_trap_bar, lmce);
+ if (lmce || atomic_read(&severity_cpu) == smp_processor_id())
{
/* According to SDM, if no error bank found on any cpus,
* something unexpected happening, we can't do any
@@ -527,16 +563,16 @@ void mcheck_cmn_handler(const struct cpu_user_regs *regs)
}
atomic_set(&found_error, 0);
}
- mce_barrier_exit(&mce_trap_bar);
+ mce_barrier_exit(&mce_trap_bar, lmce);
/* Clear flags after above fatal check */
- mce_barrier_enter(&mce_trap_bar);
+ mce_barrier_enter(&mce_trap_bar, lmce);
gstatus = mca_rdmsr(MSR_IA32_MCG_STATUS);
if ((gstatus & MCG_STATUS_MCIP) != 0) {
mce_printk(MCE_CRITICAL, "MCE: Clear MCIP@ last step");
mca_wrmsr(MSR_IA32_MCG_STATUS, 0);
}
- mce_barrier_exit(&mce_trap_bar);
+ mce_barrier_exit(&mce_trap_bar, lmce);
raise_softirq(MACHINE_CHECK_SOFTIRQ);
}
@@ -1700,38 +1736,61 @@ static void mce_softirq(void)
{
int cpu = smp_processor_id();
unsigned int workcpu;
+ /*
+ * On platforms supporting broadcasting MCE, if there is a
+ * non-local MCE waiting for process on this CPU, it implies other
+ * CPUs received the same MCE as well. Therefore, mce_softirq()
+ * will be launched on other CPUs as well and compete with this
+ * mce_softirq() to handle all MCE's on all CPUs. In that case, we
+ * should use MCE barriers to sync with other CPUs. Otherwise, we
+ * do not need to wait for other CPUs.
+ */
+ bool nowait = !this_cpu(nonlocal_mce_in_progress);
mce_printk(MCE_VERBOSE, "CPU%d enter softirq\n", cpu);
- mce_barrier_enter(&mce_inside_bar);
+ mce_barrier_enter(&mce_inside_bar, nowait);
/*
- * Everybody is here. Now let's see who gets to do the
- * recovery work. Right now we just see if there's a CPU
- * that did not have any problems, and pick that one.
- *
- * First, just set a default value: the last CPU who reaches this
- * will overwrite the value and become the default.
+ * When LMCE is being handled and no non-local MCE is waiting for
+ * process, mce_softirq() does not need to set 'severity_cpu'. And
+ * it should not, because the modification in this function may
+ * interfere with the simultaneous modification in mce_cmn_handler()
+ * on another CPU.
*/
-
- atomic_set(&severity_cpu, cpu);
-
- mce_barrier_enter(&mce_severity_bar);
- if (!mctelem_has_deferred(cpu))
+ if (!nowait) {
+ /*
+ * Everybody is here. Now let's see who gets to do the
+ * recovery work. Right now we just see if there's a CPU
+ * that did not have any problems, and pick that one.
+ *
+ * First, just set a default value: the last CPU who reaches this
+ * will overwrite the value and become the default.
+ */
atomic_set(&severity_cpu, cpu);
- mce_barrier_exit(&mce_severity_bar);
- /* We choose severity_cpu for further processing */
- if (atomic_read(&severity_cpu) == cpu) {
+ mce_barrier_enter(&mce_severity_bar, nowait);
+ if (!mctelem_has_deferred(cpu))
+ atomic_set(&severity_cpu, cpu);
+ mce_barrier_exit(&mce_severity_bar, nowait);
+ }
+
+ /*
+ * Either no non-local MCE is being handled, or this CPU is chose as
+ * severity_cpu for further processing ...
+ */
+ if (nowait || atomic_read(&severity_cpu) == cpu) {
mce_printk(MCE_VERBOSE, "CPU%d handling errors\n", cpu);
/* Step1: Fill DOM0 LOG buffer, vMCE injection buffer and
* vMCE MSRs virtualization buffer
*/
- for_each_online_cpu(workcpu) {
- mctelem_process_deferred(workcpu, mce_delayed_action);
- }
+ if (nowait)
+ mctelem_process_deferred(cpu, mce_delayed_action);
+ else
+ for_each_online_cpu(workcpu)
+ mctelem_process_deferred(workcpu, mce_delayed_action);
/* Step2: Send Log to DOM0 through vIRQ */
if (dom0_vmce_enabled()) {
@@ -1740,7 +1799,14 @@ static void mce_softirq(void)
}
}
- mce_barrier_exit(&mce_inside_bar);
+ mce_barrier_exit(&mce_inside_bar, nowait);
+
+ /*
+ * At this point, either the only LMCE has been handled, or all MCE's
+ * on this CPU waiting for process have been handled on this CPU (if
+ * it was chose as severity_cpu) or by mce_softirq() on another CPU.
+ */
+ this_cpu(nonlocal_mce_in_progress) = false;
}
/* Machine Check owner judge algorithm:
diff --git a/xen/arch/x86/cpu/mcheck/mce.h b/xen/arch/x86/cpu/mcheck/mce.h
index 10e5ceb..3a9134e 100644
--- a/xen/arch/x86/cpu/mcheck/mce.h
+++ b/xen/arch/x86/cpu/mcheck/mce.h
@@ -109,12 +109,14 @@ struct mca_summary {
int eipv; /* meaningful on #MC */
bool uc; /* UC flag */
bool pcc; /* PCC flag */
+ bool lmce; /* LMCE flag (Intel only) */
bool recoverable; /* software error recoverable flag */
};
DECLARE_PER_CPU(struct mca_banks *, poll_bankmask);
DECLARE_PER_CPU(struct mca_banks *, no_cmci_banks);
DECLARE_PER_CPU(struct mca_banks *, mce_clear_banks);
+DECLARE_PER_CPU(bool, mce_in_process);
extern bool cmci_support;
extern bool is_mc_panic;
diff --git a/xen/arch/x86/cpu/mcheck/x86_mca.h b/xen/arch/x86/cpu/mcheck/x86_mca.h
index 34d1921..de03f82 100644
--- a/xen/arch/x86/cpu/mcheck/x86_mca.h
+++ b/xen/arch/x86/cpu/mcheck/x86_mca.h
@@ -42,7 +42,9 @@
#define MCG_STATUS_RIPV 0x0000000000000001ULL
#define MCG_STATUS_EIPV 0x0000000000000002ULL
#define MCG_STATUS_MCIP 0x0000000000000004ULL
-/* Bits 3-63 are reserved */
+#define MCG_STATUS_LMCE 0x0000000000000008ULL /* Intel specific */
+/* Bits 3-63 are reserved on CPU not supporting LMCE */
+/* Bits 4-63 are reserved on CPU supporting LMCE */
/* Bitfield of MSR_K8_MCi_STATUS registers */
/* MCA error code */
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v3 1/9] x86/mce: handle LMCE locally
2017-03-30 6:19 ` [PATCH v3 1/9] x86/mce: handle LMCE locally Haozhong Zhang
@ 2017-03-30 14:35 ` Jan Beulich
2017-03-31 2:34 ` Haozhong Zhang
0 siblings, 1 reply; 22+ messages in thread
From: Jan Beulich @ 2017-03-30 14:35 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, xen-devel
>>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
> --- a/xen/arch/x86/cpu/mcheck/mce.c
> +++ b/xen/arch/x86/cpu/mcheck/mce.c
> @@ -42,6 +42,13 @@ DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, poll_bankmask);
> DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, no_cmci_banks);
> DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, mce_clear_banks);
>
> +/*
> + * Flag to indicate that at least one non-local MCE on this CPU has
> + * not been completed handled. It's set by mcheck_cmn_handler() and
completely
> + * cleared by mce_softirq().
> + */
> +static DEFINE_PER_CPU(bool, nonlocal_mce_in_progress);
Does a boolean really suffice here? I.e. can't there be a 2nd one
while the first one is still being dealt with?
> @@ -186,7 +193,29 @@ static struct mce_softirq_barrier mce_trap_bar;
> */
> static DEFINE_SPINLOCK(mce_logout_lock);
>
> +/*
> + * 'severity_cpu' is used in both mcheck_cmn_handler() and mce_softirq().
> + *
> + * When handling broadcasting MCE, MCE barriers take effect to prevent
> + * 'severity_cpu' being modified in one function on one CPU and accessed
> + * in another on a different CPU.
> + *
> + * When handling LMCE, mce_barrier_enter() and mce_barrier_exit() are
> + * effectively NOP, so it's possible that mcheck_cmn_handler() is handling
> + * a LMCE on CPUx while mce_softirq() is handling another LMCE on CPUy.
> + * If both are modifying 'severity_cpu', they may interfere with each
> + * other. Therefore, it's better for mcheck_cmn_handler() and mce_softirq()
> + * to avoid accessing 'severity_cpu' when handling LMCE, unless other
> + * approaches are taken to avoid the above interference.
> + */
> static atomic_t severity_cpu = ATOMIC_INIT(-1);
> +/*
> + * The following two global variables are used only in mcheck_cmn_handler().
In which case the better approach would be to move them into
that function. In no event are these "global" variables - they're
static after all.
> @@ -1700,38 +1736,61 @@ static void mce_softirq(void)
> {
> int cpu = smp_processor_id();
> unsigned int workcpu;
> + /*
> + * On platforms supporting broadcasting MCE, if there is a
> + * non-local MCE waiting for process on this CPU, it implies other
> + * CPUs received the same MCE as well. Therefore, mce_softirq()
> + * will be launched on other CPUs as well and compete with this
> + * mce_softirq() to handle all MCE's on all CPUs. In that case, we
> + * should use MCE barriers to sync with other CPUs. Otherwise, we
> + * do not need to wait for other CPUs.
> + */
> + bool nowait = !this_cpu(nonlocal_mce_in_progress);
>
> mce_printk(MCE_VERBOSE, "CPU%d enter softirq\n", cpu);
>
> - mce_barrier_enter(&mce_inside_bar);
> + mce_barrier_enter(&mce_inside_bar, nowait);
>
> /*
> - * Everybody is here. Now let's see who gets to do the
> - * recovery work. Right now we just see if there's a CPU
> - * that did not have any problems, and pick that one.
> - *
> - * First, just set a default value: the last CPU who reaches this
> - * will overwrite the value and become the default.
> + * When LMCE is being handled and no non-local MCE is waiting for
> + * process, mce_softirq() does not need to set 'severity_cpu'. And
> + * it should not, because the modification in this function may
> + * interfere with the simultaneous modification in mce_cmn_handler()
> + * on another CPU.
> */
> -
> - atomic_set(&severity_cpu, cpu);
> -
> - mce_barrier_enter(&mce_severity_bar);
> - if (!mctelem_has_deferred(cpu))
> + if (!nowait) {
> + /*
> + * Everybody is here. Now let's see who gets to do the
> + * recovery work. Right now we just see if there's a CPU
> + * that did not have any problems, and pick that one.
> + *
> + * First, just set a default value: the last CPU who reaches this
> + * will overwrite the value and become the default.
> + */
> atomic_set(&severity_cpu, cpu);
> - mce_barrier_exit(&mce_severity_bar);
>
> - /* We choose severity_cpu for further processing */
> - if (atomic_read(&severity_cpu) == cpu) {
> + mce_barrier_enter(&mce_severity_bar, nowait);
> + if (!mctelem_has_deferred(cpu))
> + atomic_set(&severity_cpu, cpu);
> + mce_barrier_exit(&mce_severity_bar, nowait);
You're in a !nowait conditional - please pass false explicitly to
enter/exit.
> @@ -1740,7 +1799,14 @@ static void mce_softirq(void)
> }
> }
>
> - mce_barrier_exit(&mce_inside_bar);
> + mce_barrier_exit(&mce_inside_bar, nowait);
> +
> + /*
> + * At this point, either the only LMCE has been handled, or all MCE's
> + * on this CPU waiting for process have been handled on this CPU (if
"for processing" or "to be processed" I think.
> --- a/xen/arch/x86/cpu/mcheck/mce.h
> +++ b/xen/arch/x86/cpu/mcheck/mce.h
> @@ -109,12 +109,14 @@ struct mca_summary {
> int eipv; /* meaningful on #MC */
> bool uc; /* UC flag */
> bool pcc; /* PCC flag */
> + bool lmce; /* LMCE flag (Intel only) */
> bool recoverable; /* software error recoverable flag */
> };
>
> DECLARE_PER_CPU(struct mca_banks *, poll_bankmask);
> DECLARE_PER_CPU(struct mca_banks *, no_cmci_banks);
> DECLARE_PER_CPU(struct mca_banks *, mce_clear_banks);
> +DECLARE_PER_CPU(bool, mce_in_process);
This should have been deleted afaict.
Jan
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread* Re: [PATCH v3 1/9] x86/mce: handle LMCE locally
2017-03-30 14:35 ` Jan Beulich
@ 2017-03-31 2:34 ` Haozhong Zhang
2017-03-31 7:24 ` Jan Beulich
0 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-31 2:34 UTC (permalink / raw)
To: Jan Beulich; +Cc: Andrew Cooper, xen-devel
On 03/30/17 08:35 -0600, Jan Beulich wrote:
> >>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
> > --- a/xen/arch/x86/cpu/mcheck/mce.c
> > +++ b/xen/arch/x86/cpu/mcheck/mce.c
> > @@ -42,6 +42,13 @@ DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, poll_bankmask);
> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, no_cmci_banks);
> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, mce_clear_banks);
> >
> > +/*
> > + * Flag to indicate that at least one non-local MCE on this CPU has
> > + * not been completed handled. It's set by mcheck_cmn_handler() and
>
> completely
>
> > + * cleared by mce_softirq().
> > + */
> > +static DEFINE_PER_CPU(bool, nonlocal_mce_in_progress);
>
> Does a boolean really suffice here? I.e. can't there be a 2nd one
> while the first one is still being dealt with?
It's to indicate the existence of a non-local MCE, and mce_softirq()
will call mctelem_process_deferred() to handle all MCE's if it's true,
so a boolean flag is suffice.
Haozhong
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: [PATCH v3 1/9] x86/mce: handle LMCE locally
2017-03-31 2:34 ` Haozhong Zhang
@ 2017-03-31 7:24 ` Jan Beulich
2017-04-06 4:50 ` Haozhong Zhang
0 siblings, 1 reply; 22+ messages in thread
From: Jan Beulich @ 2017-03-31 7:24 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, xen-devel
>>> On 31.03.17 at 04:34, <haozhong.zhang@intel.com> wrote:
> On 03/30/17 08:35 -0600, Jan Beulich wrote:
>> >>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
>> > --- a/xen/arch/x86/cpu/mcheck/mce.c
>> > +++ b/xen/arch/x86/cpu/mcheck/mce.c
>> > @@ -42,6 +42,13 @@ DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, poll_bankmask);
>> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, no_cmci_banks);
>> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, mce_clear_banks);
>> >
>> > +/*
>> > + * Flag to indicate that at least one non-local MCE on this CPU has
>> > + * not been completed handled. It's set by mcheck_cmn_handler() and
>> > + * cleared by mce_softirq().
>> > + */
>> > +static DEFINE_PER_CPU(bool, nonlocal_mce_in_progress);
>>
>> Does a boolean really suffice here? I.e. can't there be a 2nd one
>> while the first one is still being dealt with?
>
> It's to indicate the existence of a non-local MCE, and mce_softirq()
> will call mctelem_process_deferred() to handle all MCE's if it's true,
> so a boolean flag is suffice.
I don't buy this, I'm sorry. What if a 2nd #MC occurs at the
instruction boundary of setting the variable to false (in
mce_softirq())? A subsequent mce_softirq() would then see
the variable being false, wouldn't it?
Jan
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: [PATCH v3 1/9] x86/mce: handle LMCE locally
2017-03-31 7:24 ` Jan Beulich
@ 2017-04-06 4:50 ` Haozhong Zhang
0 siblings, 0 replies; 22+ messages in thread
From: Haozhong Zhang @ 2017-04-06 4:50 UTC (permalink / raw)
To: Jan Beulich; +Cc: Andrew Cooper, xen-devel
On 03/31/17 01:24 -0600, Jan Beulich wrote:
> >>> On 31.03.17 at 04:34, <haozhong.zhang@intel.com> wrote:
> > On 03/30/17 08:35 -0600, Jan Beulich wrote:
> >> >>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
> >> > --- a/xen/arch/x86/cpu/mcheck/mce.c
> >> > +++ b/xen/arch/x86/cpu/mcheck/mce.c
> >> > @@ -42,6 +42,13 @@ DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, poll_bankmask);
> >> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, no_cmci_banks);
> >> > DEFINE_PER_CPU_READ_MOSTLY(struct mca_banks *, mce_clear_banks);
> >> >
> >> > +/*
> >> > + * Flag to indicate that at least one non-local MCE on this CPU has
> >> > + * not been completed handled. It's set by mcheck_cmn_handler() and
> >> > + * cleared by mce_softirq().
> >> > + */
> >> > +static DEFINE_PER_CPU(bool, nonlocal_mce_in_progress);
> >>
> >> Does a boolean really suffice here? I.e. can't there be a 2nd one
> >> while the first one is still being dealt with?
> >
> > It's to indicate the existence of a non-local MCE, and mce_softirq()
> > will call mctelem_process_deferred() to handle all MCE's if it's true,
> > so a boolean flag is suffice.
>
> I don't buy this, I'm sorry. What if a 2nd #MC occurs at the
> instruction boundary of setting the variable to false (in
> mce_softirq())? A subsequent mce_softirq() would then see
> the variable being false, wouldn't it?
>
You are right. I missed the point that MC# can happen during
mce_softirq() and this patch 1 is problematic in that case. Let me
rethink about this.
Thanks,
Haozhong
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH v3 2/9] x86/mce_intel: detect and enable LMCE on Intel host
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
2017-03-30 6:19 ` [PATCH v3 1/9] x86/mce: handle LMCE locally Haozhong Zhang
@ 2017-03-30 6:19 ` Haozhong Zhang
2017-03-30 6:19 ` [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL Haozhong Zhang
` (6 subsequent siblings)
8 siblings, 0 replies; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:19 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Jan Beulich, Andrew Cooper
Enable LMCE if it's supported by the host CPU. If Xen boot parameter
"mce_fb = 1" is present, LMCE will be disabled forcibly.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Make lmce_support static.
---
xen/arch/x86/cpu/mcheck/mce_intel.c | 46 ++++++++++++++++++++++++++++++++-----
xen/arch/x86/cpu/mcheck/x86_mca.h | 5 ++++
xen/include/asm-x86/msr-index.h | 2 ++
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
index fdf57ce..3c57052 100644
--- a/xen/arch/x86/cpu/mcheck/mce_intel.c
+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
@@ -29,6 +29,9 @@ boolean_param("mce_fb", mce_force_broadcast);
static int __read_mostly nr_intel_ext_msrs;
+/* If mce_force_broadcast == 1, lmce_support will be disabled forcibly. */
+static bool __read_mostly lmce_support;
+
/* Intel SDM define bit15~bit0 of IA32_MCi_STATUS as the MC error code */
#define INTEL_MCCOD_MASK 0xFFFF
@@ -698,10 +701,34 @@ static bool mce_is_broadcast(struct cpuinfo_x86 *c)
return false;
}
+static bool intel_enable_lmce(void)
+{
+ uint64_t msr_content;
+
+ /*
+ * Section "Enabling Local Machine Check" in Intel SDM Vol 3
+ * requires software must ensure the LOCK bit and LMCE_ON bit
+ * of MSR_IA32_FEATURE_CONTROL are set before setting
+ * MSR_IA32_MCG_EXT_CTL.LMCE_EN.
+ */
+
+ if ( rdmsr_safe(MSR_IA32_FEATURE_CONTROL, msr_content) )
+ return false;
+
+ if ( (msr_content & IA32_FEATURE_CONTROL_LOCK) &&
+ (msr_content & IA32_FEATURE_CONTROL_LMCE_ON) )
+ {
+ wrmsrl(MSR_IA32_MCG_EXT_CTL, MCG_EXT_CTL_LMCE_EN);
+ return true;
+ }
+
+ return false;
+}
+
/* Check and init MCA */
static void intel_init_mca(struct cpuinfo_x86 *c)
{
- bool broadcast, cmci = false, ser = false;
+ bool broadcast, cmci = false, ser = false, lmce = false;
int ext_num = 0, first;
uint64_t msr_content;
@@ -721,33 +748,40 @@ static void intel_init_mca(struct cpuinfo_x86 *c)
first = mce_firstbank(c);
+ if (!mce_force_broadcast && (msr_content & MCG_LMCE_P))
+ lmce = intel_enable_lmce();
+
#define CAP(enabled, name) ((enabled) ? ", " name : "")
if (smp_processor_id() == 0)
{
dprintk(XENLOG_INFO,
- "MCA capability: firstbank %d, %d ext MSRs%s%s%s\n",
+ "MCA Capability: firstbank %d, extended MCE MSR %d%s%s%s%s\n",
first, ext_num,
CAP(broadcast, "BCAST"),
CAP(ser, "SER"),
- CAP(cmci, "CMCI"));
+ CAP(cmci, "CMCI"),
+ CAP(lmce, "LMCE"));
mce_broadcast = broadcast;
cmci_support = cmci;
ser_support = ser;
+ lmce_support = lmce;
nr_intel_ext_msrs = ext_num;
firstbank = first;
}
else if (cmci != cmci_support || ser != ser_support ||
broadcast != mce_broadcast ||
- first != firstbank || ext_num != nr_intel_ext_msrs)
+ first != firstbank || ext_num != nr_intel_ext_msrs ||
+ lmce != lmce_support)
dprintk(XENLOG_WARNING,
"CPU%u has different MCA capability "
- "(firstbank %d, %d ext MSRs%s%s%s)"
+ "(firstbank %d, extended MCE MSR %d%s%s%s%s)"
" than BSP, may cause undetermined result!!!\n",
smp_processor_id(), first, ext_num,
CAP(broadcast, "BCAST"),
CAP(ser, "SER"),
- CAP(cmci, "CMCI"));
+ CAP(cmci, "CMCI"),
+ CAP(lmce, "LMCE"));
#undef CAP
}
diff --git a/xen/arch/x86/cpu/mcheck/x86_mca.h b/xen/arch/x86/cpu/mcheck/x86_mca.h
index de03f82..0f87bcf 100644
--- a/xen/arch/x86/cpu/mcheck/x86_mca.h
+++ b/xen/arch/x86/cpu/mcheck/x86_mca.h
@@ -36,6 +36,7 @@
#define MCG_TES_P (1ULL<<11) /* Intel specific */
#define MCG_EXT_CNT 16 /* Intel specific */
#define MCG_SER_P (1ULL<<24) /* Intel specific */
+#define MCG_LMCE_P (1ULL<<27) /* Intel specific */
/* Other bits are reserved */
/* Bitfield of the MSR_IA32_MCG_STATUS register */
@@ -46,6 +47,10 @@
/* Bits 3-63 are reserved on CPU not supporting LMCE */
/* Bits 4-63 are reserved on CPU supporting LMCE */
+/* Bitfield of MSR_IA32_MCG_EXT_CTL register (Intel Specific) */
+#define MCG_EXT_CTL_LMCE_EN (1ULL<<0)
+/* Other bits are reserved */
+
/* Bitfield of MSR_K8_MCi_STATUS registers */
/* MCA error code */
#define MCi_STATUS_MCA 0x000000000000ffffULL
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 771e750..756b23d 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -51,6 +51,7 @@
#define MSR_IA32_MCG_CAP 0x00000179
#define MSR_IA32_MCG_STATUS 0x0000017a
#define MSR_IA32_MCG_CTL 0x0000017b
+#define MSR_IA32_MCG_EXT_CTL 0x000004d0
#define MSR_IA32_PEBS_ENABLE 0x000003f1
#define MSR_IA32_DS_AREA 0x00000600
@@ -296,6 +297,7 @@
#define IA32_FEATURE_CONTROL_SENTER_PARAM_CTL 0x7f00
#define IA32_FEATURE_CONTROL_ENABLE_SENTER 0x8000
#define IA32_FEATURE_CONTROL_SGX_ENABLE 0x40000
+#define IA32_FEATURE_CONTROL_LMCE_ON 0x100000
#define MSR_IA32_TSC_ADJUST 0x0000003b
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
2017-03-30 6:19 ` [PATCH v3 1/9] x86/mce: handle LMCE locally Haozhong Zhang
2017-03-30 6:19 ` [PATCH v3 2/9] x86/mce_intel: detect and enable LMCE on Intel host Haozhong Zhang
@ 2017-03-30 6:19 ` Haozhong Zhang
2017-03-30 14:36 ` Jan Beulich
2017-03-31 5:11 ` Tian, Kevin
2017-03-30 6:19 ` [PATCH v3 4/9] x86/vmce: emulate MSR_IA32_MCG_EXT_CTL Haozhong Zhang
` (5 subsequent siblings)
8 siblings, 2 replies; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:19 UTC (permalink / raw)
To: xen-devel
Cc: Haozhong Zhang, Kevin Tian, Jun Nakajima, Jan Beulich,
Andrew Cooper
If MCG_LMCE_P is present in guest MSR_IA32_MCG_CAP, then set LMCE and
LOCK bits in guest MSR_IA32_FEATURE_CONTROL. Intel SDM requires those
bits are set before SW can enable LMCE.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: Jun Nakajima <jun.nakajima@intel.com>
Cc: Kevin Tian <kevin.tian@intel.com>
Changes in v3:
* Rename vmce_support_lmce() to vmce_has_lmce().
* Add const to "curr" in vmx_msr_read_intercept().
---
xen/arch/x86/cpu/mcheck/mce_intel.c | 4 ++++
xen/arch/x86/hvm/vmx/vmx.c | 9 +++++++++
xen/arch/x86/hvm/vmx/vvmx.c | 4 ----
xen/include/asm-x86/mce.h | 1 +
4 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
index 3c57052..8528452 100644
--- a/xen/arch/x86/cpu/mcheck/mce_intel.c
+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
@@ -946,3 +946,7 @@ int vmce_intel_rdmsr(const struct vcpu *v, uint32_t msr, uint64_t *val)
return 1;
}
+bool vmce_has_lmce(const struct vcpu *v)
+{
+ return v->arch.vmce.mcg_cap & MCG_LMCE_P;
+}
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index d201956..9ee1856 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -55,6 +55,7 @@
#include <asm/hvm/nestedhvm.h>
#include <asm/altp2m.h>
#include <asm/event.h>
+#include <asm/mce.h>
#include <asm/monitor.h>
#include <public/arch-x86/cpuid.h>
@@ -2745,6 +2746,8 @@ static int is_last_branch_msr(u32 ecx)
static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
{
+ const struct vcpu *curr = current;
+
HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x", msr);
switch ( msr )
@@ -2762,6 +2765,12 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
__vmread(GUEST_IA32_DEBUGCTL, msr_content);
break;
case MSR_IA32_FEATURE_CONTROL:
+ *msr_content = IA32_FEATURE_CONTROL_LOCK;
+ if ( vmce_has_lmce(curr) )
+ *msr_content |= IA32_FEATURE_CONTROL_LMCE_ON;
+ if ( nestedhvm_enabled(curr->domain) )
+ *msr_content |= IA32_FEATURE_CONTROL_ENABLE_VMXON_OUTSIDE_SMX;
+ break;
case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_VMFUNC:
if ( !nvmx_msr_read_intercept(msr, msr_content) )
goto gp_fault;
diff --git a/xen/arch/x86/hvm/vmx/vvmx.c b/xen/arch/x86/hvm/vmx/vvmx.c
index 09e4250..f41186d 100644
--- a/xen/arch/x86/hvm/vmx/vvmx.c
+++ b/xen/arch/x86/hvm/vmx/vvmx.c
@@ -2084,10 +2084,6 @@ int nvmx_msr_read_intercept(unsigned int msr, u64 *msr_content)
data = gen_vmx_msr(data, VMX_ENTRY_CTLS_DEFAULT1, host_data);
break;
- case MSR_IA32_FEATURE_CONTROL:
- data = IA32_FEATURE_CONTROL_LOCK |
- IA32_FEATURE_CONTROL_ENABLE_VMXON_OUTSIDE_SMX;
- break;
case MSR_IA32_VMX_VMCS_ENUM:
/* The max index of VVMCS encoding is 0x1f. */
data = 0x1f << 1;
diff --git a/xen/include/asm-x86/mce.h b/xen/include/asm-x86/mce.h
index 549bef3..56ad1f9 100644
--- a/xen/include/asm-x86/mce.h
+++ b/xen/include/asm-x86/mce.h
@@ -36,6 +36,7 @@ extern void vmce_init_vcpu(struct vcpu *);
extern int vmce_restore_vcpu(struct vcpu *, const struct hvm_vmce_vcpu *);
extern int vmce_wrmsr(uint32_t msr, uint64_t val);
extern int vmce_rdmsr(uint32_t msr, uint64_t *val);
+extern bool vmce_has_lmce(const struct vcpu *v);
extern unsigned int nr_mce_banks;
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL
2017-03-30 6:19 ` [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL Haozhong Zhang
@ 2017-03-30 14:36 ` Jan Beulich
2017-03-31 5:11 ` Tian, Kevin
1 sibling, 0 replies; 22+ messages in thread
From: Jan Beulich @ 2017-03-30 14:36 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, Kevin Tian, Jun Nakajima, xen-devel
>>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
> If MCG_LMCE_P is present in guest MSR_IA32_MCG_CAP, then set LMCE and
> LOCK bits in guest MSR_IA32_FEATURE_CONTROL. Intel SDM requires those
> bits are set before SW can enable LMCE.
>
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL
2017-03-30 6:19 ` [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL Haozhong Zhang
2017-03-30 14:36 ` Jan Beulich
@ 2017-03-31 5:11 ` Tian, Kevin
1 sibling, 0 replies; 22+ messages in thread
From: Tian, Kevin @ 2017-03-31 5:11 UTC (permalink / raw)
To: Zhang, Haozhong, xen-devel@lists.xen.org
Cc: Andrew Cooper, Nakajima, Jun, Jan Beulich
> From: Zhang, Haozhong
> Sent: Thursday, March 30, 2017 2:20 PM
>
> If MCG_LMCE_P is present in guest MSR_IA32_MCG_CAP, then set LMCE and
> LOCK bits in guest MSR_IA32_FEATURE_CONTROL. Intel SDM requires those
> bits are set before SW can enable LMCE.
>
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH v3 4/9] x86/vmce: emulate MSR_IA32_MCG_EXT_CTL
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (2 preceding siblings ...)
2017-03-30 6:19 ` [PATCH v3 3/9] x86/vmx: expose LMCE feature via guest MSR_IA32_FEATURE_CONTROL Haozhong Zhang
@ 2017-03-30 6:19 ` Haozhong Zhang
2017-03-30 14:41 ` Jan Beulich
2017-03-30 6:19 ` [PATCH v3 5/9] x86/vmce: enable injecting LMCE to guest on Intel host Haozhong Zhang
` (4 subsequent siblings)
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:19 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Jan Beulich, Andrew Cooper
If MCG_LMCE_P is present in guest MSR_IA32_MCG_CAP, then allow guest
to read/write MSR_IA32_MCG_EXT_CTL.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Replace lmce_enabled in struct hvm_vmce_vcpu and struct vmce by
mcg_ext_ctl.
* Drop the change to XEN_DOMCTL_INTERFACE_VERSION, as it was changed
by c/s 33e5c325 recently.
---
xen/arch/x86/cpu/mcheck/vmce.c | 34 +++++++++++++++++++++++++++++++++-
xen/include/asm-x86/mce.h | 1 +
xen/include/public/arch-x86/hvm/save.h | 1 +
3 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/xen/arch/x86/cpu/mcheck/vmce.c b/xen/arch/x86/cpu/mcheck/vmce.c
index d591d31..2106706 100644
--- a/xen/arch/x86/cpu/mcheck/vmce.c
+++ b/xen/arch/x86/cpu/mcheck/vmce.c
@@ -90,6 +90,7 @@ int vmce_restore_vcpu(struct vcpu *v, const struct hvm_vmce_vcpu *ctxt)
v->arch.vmce.mcg_cap = ctxt->caps;
v->arch.vmce.bank[0].mci_ctl2 = ctxt->mci_ctl2_bank0;
v->arch.vmce.bank[1].mci_ctl2 = ctxt->mci_ctl2_bank1;
+ v->arch.vmce.mcg_ext_ctl = ctxt->mcg_ext_ctl;
return 0;
}
@@ -199,6 +200,26 @@ int vmce_rdmsr(uint32_t msr, uint64_t *val)
mce_printk(MCE_VERBOSE, "MCE: %pv: rd MCG_CTL %#"PRIx64"\n", cur, *val);
break;
+ case MSR_IA32_MCG_EXT_CTL:
+ /*
+ * If MCG_LMCE_P is present in guest MSR_IA32_MCG_CAP, the LMCE and LOCK
+ * bits are always set in guest MSR_IA32_FEATURE_CONTROL by Xen, so it
+ * does not need to check them here.
+ */
+ if ( cur->arch.vmce.mcg_cap & MCG_LMCE_P )
+ {
+ *val = cur->arch.vmce.mcg_ext_ctl;
+ mce_printk(MCE_VERBOSE, "MCE: %pv: rd MCG_EXT_CTL %#"PRIx64"\n",
+ cur, *val);
+ }
+ else
+ {
+ ret = -1;
+ mce_printk(MCE_VERBOSE, "MCE: %pv: rd MCG_EXT_CTL, not supported\n",
+ cur);
+ }
+ break;
+
default:
ret = mce_bank_msr(cur, msr) ? bank_mce_rdmsr(cur, msr, val) : 0;
break;
@@ -308,6 +329,16 @@ int vmce_wrmsr(uint32_t msr, uint64_t val)
mce_printk(MCE_VERBOSE, "MCE: %pv: MCG_CAP is r/o\n", cur);
break;
+ case MSR_IA32_MCG_EXT_CTL:
+ if ( (cur->arch.vmce.mcg_cap & MCG_LMCE_P) &&
+ !(val & ~MCG_EXT_CTL_LMCE_EN) )
+ cur->arch.vmce.mcg_ext_ctl = val;
+ else
+ ret = -1;
+ mce_printk(MCE_VERBOSE, "MCE: %pv: wr MCG_EXT_CTL %"PRIx64"%s\n",
+ cur, val, (ret == -1) ? ", not supported" : "");
+ break;
+
default:
ret = mce_bank_msr(cur, msr) ? bank_mce_wrmsr(cur, msr, val) : 0;
break;
@@ -326,7 +357,8 @@ static int vmce_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h)
struct hvm_vmce_vcpu ctxt = {
.caps = v->arch.vmce.mcg_cap,
.mci_ctl2_bank0 = v->arch.vmce.bank[0].mci_ctl2,
- .mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2
+ .mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2,
+ .mcg_ext_ctl = v->arch.vmce.mcg_ext_ctl,
};
err = hvm_save_entry(VMCE_VCPU, v->vcpu_id, h, &ctxt);
diff --git a/xen/include/asm-x86/mce.h b/xen/include/asm-x86/mce.h
index 56ad1f9..35f9962 100644
--- a/xen/include/asm-x86/mce.h
+++ b/xen/include/asm-x86/mce.h
@@ -27,6 +27,7 @@ struct vmce_bank {
struct vmce {
uint64_t mcg_cap;
uint64_t mcg_status;
+ uint64_t mcg_ext_ctl;
spinlock_t lock;
struct vmce_bank bank[GUEST_MC_BANK_NUM];
};
diff --git a/xen/include/public/arch-x86/hvm/save.h b/xen/include/public/arch-x86/hvm/save.h
index 66ae1a2..ec581cc 100644
--- a/xen/include/public/arch-x86/hvm/save.h
+++ b/xen/include/public/arch-x86/hvm/save.h
@@ -599,6 +599,7 @@ struct hvm_vmce_vcpu {
uint64_t caps;
uint64_t mci_ctl2_bank0;
uint64_t mci_ctl2_bank1;
+ uint64_t mcg_ext_ctl;
};
DECLARE_HVM_SAVE_TYPE(VMCE_VCPU, 18, struct hvm_vmce_vcpu);
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 5/9] x86/vmce: enable injecting LMCE to guest on Intel host
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (3 preceding siblings ...)
2017-03-30 6:19 ` [PATCH v3 4/9] x86/vmce: emulate MSR_IA32_MCG_EXT_CTL Haozhong Zhang
@ 2017-03-30 6:19 ` Haozhong Zhang
2017-03-30 14:51 ` Jan Beulich
2017-03-30 6:20 ` [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP Haozhong Zhang
` (3 subsequent siblings)
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:19 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Jan Beulich, Andrew Cooper
Inject LMCE to guest if the host MCE is LMCE and the affected vcpu is
known. Otherwise, broadcast MCE to all vcpus on Intel host.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Adjust a check in mc_memerr_dhandler() and add code comment for it.
---
xen/arch/x86/cpu/mcheck/mcaction.c | 23 ++++++++++++++++-------
xen/arch/x86/cpu/mcheck/vmce.c | 11 ++++++++++-
xen/arch/x86/cpu/mcheck/vmce.h | 2 +-
3 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/xen/arch/x86/cpu/mcheck/mcaction.c b/xen/arch/x86/cpu/mcheck/mcaction.c
index ca17d22..2556bb0 100644
--- a/xen/arch/x86/cpu/mcheck/mcaction.c
+++ b/xen/arch/x86/cpu/mcheck/mcaction.c
@@ -44,6 +44,7 @@ mc_memerr_dhandler(struct mca_binfo *binfo,
unsigned long mfn, gfn;
uint32_t status;
int vmce_vcpuid;
+ uint16_t mc_vcpuid;
if (!mc_check_addr(bank->mc_status, bank->mc_misc, MC_ADDR_PHYSICAL)) {
dprintk(XENLOG_WARNING,
@@ -88,18 +89,26 @@ mc_memerr_dhandler(struct mca_binfo *binfo,
goto vmce_failed;
}
- if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
- global->mc_vcpuid == XEN_MC_VCPUID_INVALID)
+ mc_vcpuid = global->mc_vcpuid;
+ if (mc_vcpuid == XEN_MC_VCPUID_INVALID ||
+ /*
+ * Because MC# may happen asynchronously with the actual
+ * operation that triggers the error, the domain ID as
+ * well as the vCPU ID collected in 'global' at MC# are
+ * not always precise. In that case, fallback to broadcast.
+ */
+ global->mc_domid != bank->mc_domid ||
+ (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
+ (!(global->mc_gstatus & MCG_STATUS_LMCE) ||
+ !(d->vcpu[mc_vcpuid]->arch.vmce.mcg_ext_ctl &
+ MCG_EXT_CTL_LMCE_EN))))
vmce_vcpuid = VMCE_INJECT_BROADCAST;
else
- vmce_vcpuid = global->mc_vcpuid;
+ vmce_vcpuid = mc_vcpuid;
bank->mc_addr = gfn << PAGE_SHIFT |
(bank->mc_addr & (PAGE_SIZE -1 ));
- /* TODO: support injecting LMCE */
- if (fill_vmsr_data(bank, d,
- global->mc_gstatus & ~MCG_STATUS_LMCE,
- vmce_vcpuid == VMCE_INJECT_BROADCAST))
+ if (fill_vmsr_data(bank, d, global->mc_gstatus, vmce_vcpuid))
{
mce_printk(MCE_QUIET, "Fill vMCE# data for DOM%d "
"failed\n", bank->mc_domid);
diff --git a/xen/arch/x86/cpu/mcheck/vmce.c b/xen/arch/x86/cpu/mcheck/vmce.c
index 2106706..e19be69 100644
--- a/xen/arch/x86/cpu/mcheck/vmce.c
+++ b/xen/arch/x86/cpu/mcheck/vmce.c
@@ -464,14 +464,23 @@ static int vcpu_fill_mc_msrs(struct vcpu *v, uint64_t mcg_status,
}
int fill_vmsr_data(struct mcinfo_bank *mc_bank, struct domain *d,
- uint64_t gstatus, bool broadcast)
+ uint64_t gstatus, int vmce_vcpuid)
{
struct vcpu *v = d->vcpu[0];
+ bool broadcast = (vmce_vcpuid == VMCE_INJECT_BROADCAST);
int ret, err;
if ( mc_bank->mc_domid == DOMID_INVALID )
return -EINVAL;
+ if ( broadcast )
+ gstatus &= ~MCG_STATUS_LMCE;
+ else if ( gstatus & MCG_STATUS_LMCE )
+ {
+ ASSERT(vmce_vcpuid >=0 && vmce_vcpuid < d->max_vcpus);
+ v = d->vcpu[vmce_vcpuid];
+ }
+
/*
* vMCE with the actual error information is injected to vCPU0,
* and, if broadcast is required, we choose to inject less severe
diff --git a/xen/arch/x86/cpu/mcheck/vmce.h b/xen/arch/x86/cpu/mcheck/vmce.h
index 74f6381..2797e00 100644
--- a/xen/arch/x86/cpu/mcheck/vmce.h
+++ b/xen/arch/x86/cpu/mcheck/vmce.h
@@ -17,7 +17,7 @@ int vmce_amd_rdmsr(const struct vcpu *, uint32_t msr, uint64_t *val);
int vmce_amd_wrmsr(struct vcpu *, uint32_t msr, uint64_t val);
int fill_vmsr_data(struct mcinfo_bank *mc_bank, struct domain *d,
- uint64_t gstatus, bool broadcast);
+ uint64_t gstatus, int vmce_vcpuid);
#define VMCE_INJECT_BROADCAST (-1)
int inject_vmce(struct domain *d, int vcpu);
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v3 5/9] x86/vmce: enable injecting LMCE to guest on Intel host
2017-03-30 6:19 ` [PATCH v3 5/9] x86/vmce: enable injecting LMCE to guest on Intel host Haozhong Zhang
@ 2017-03-30 14:51 ` Jan Beulich
0 siblings, 0 replies; 22+ messages in thread
From: Jan Beulich @ 2017-03-30 14:51 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, xen-devel
>>> On 30.03.17 at 08:19, <haozhong.zhang@intel.com> wrote:
> --- a/xen/arch/x86/cpu/mcheck/mcaction.c
> +++ b/xen/arch/x86/cpu/mcheck/mcaction.c
> @@ -44,6 +44,7 @@ mc_memerr_dhandler(struct mca_binfo *binfo,
> unsigned long mfn, gfn;
> uint32_t status;
> int vmce_vcpuid;
> + uint16_t mc_vcpuid;
Any reason this can't be plain unsigned int? Nowadays we try to
stay away from fixed width types where they aren't really needed.
> --- a/xen/arch/x86/cpu/mcheck/vmce.c
> +++ b/xen/arch/x86/cpu/mcheck/vmce.c
> @@ -464,14 +464,23 @@ static int vcpu_fill_mc_msrs(struct vcpu *v, uint64_t mcg_status,
> }
>
> int fill_vmsr_data(struct mcinfo_bank *mc_bank, struct domain *d,
> - uint64_t gstatus, bool broadcast)
> + uint64_t gstatus, int vmce_vcpuid)
> {
> struct vcpu *v = d->vcpu[0];
> + bool broadcast = (vmce_vcpuid == VMCE_INJECT_BROADCAST);
> int ret, err;
>
> if ( mc_bank->mc_domid == DOMID_INVALID )
> return -EINVAL;
>
> + if ( broadcast )
> + gstatus &= ~MCG_STATUS_LMCE;
> + else if ( gstatus & MCG_STATUS_LMCE )
> + {
> + ASSERT(vmce_vcpuid >=0 && vmce_vcpuid < d->max_vcpus);
Missing blank.
With these taken care of
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Jan
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (4 preceding siblings ...)
2017-03-30 6:19 ` [PATCH v3 5/9] x86/vmce: enable injecting LMCE to guest on Intel host Haozhong Zhang
@ 2017-03-30 6:20 ` Haozhong Zhang
2017-03-30 14:57 ` Jan Beulich
2017-03-30 6:20 ` [PATCH v3 7/9] xen/mce: add support of vLMCE injection to XEN_MC_inject_v2 Haozhong Zhang
` (2 subsequent siblings)
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:20 UTC (permalink / raw)
To: xen-devel
Cc: Haozhong Zhang, Ian Jackson, Wei Liu, Jan Beulich, Andrew Cooper
If LMCE is supported by host and ' mca_caps = [ "lmce" ] ' is present
in xl config, the LMCE capability will be exposed in guest MSR_IA32_MCG_CAP.
By default, LMCE is not exposed to guest so as to keep the backwards migration
compatibility.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Rename HVMMCA_CAP_* to XEN_HVM_MCA_CAP_*.
* Use mk_xen_ullong() to define XEN_HVM_MCA_CAP_LMCE and start from
bit 0.
* Make lmce_support non-static in this patch rather than in patch 2.
* Parse 'mca_caps' in xl.
* Add LIBXL_HAVE_MCA_CAPS.
---
docs/man/xl.cfg.pod.5.in | 24 ++++++++++++++++++++++++
tools/libxl/libxl.h | 7 +++++++
tools/libxl/libxl_dom.c | 15 +++++++++++++++
tools/libxl/libxl_types.idl | 1 +
tools/xl/xl_parse.c | 31 ++++++++++++++++++++++++++++++--
xen/arch/x86/cpu/mcheck/mce.h | 1 +
xen/arch/x86/cpu/mcheck/mce_intel.c | 2 +-
xen/arch/x86/cpu/mcheck/vmce.c | 19 ++++++++++++++++++-
xen/arch/x86/hvm/hvm.c | 5 +++++
xen/include/asm-x86/mce.h | 1 +
xen/include/public/hvm/params.h | 7 ++++++-
11 files changed, 108 insertions(+), 5 deletions(-)
diff --git a/docs/man/xl.cfg.pod.5.in b/docs/man/xl.cfg.pod.5.in
index 991960b..3469970 100644
--- a/docs/man/xl.cfg.pod.5.in
+++ b/docs/man/xl.cfg.pod.5.in
@@ -2023,6 +2023,30 @@ natively or via hardware backwards compatibility support.
=back
+=head3 x86
+
+=over 4
+
+=item B<mca_caps=[ "CAP", "CAP", ... ]>
+
+(HVM only) Enable MCA capabilities besides default ones enabled
+by Xen hypervisor for the HVM domain. "CAP" can be one in the
+following list:
+
+=over 4
+
+=item B<"lmce">
+
+Intel local MCE
+
+=item B<default>
+
+No MCA capabilities in above list are enabled.
+
+=back
+
+=back
+
=head1 SEE ALSO
=over 4
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 833f866..c8a0b6e 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -910,6 +910,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, const libxl_mac *src);
* If this is defined, the Code and Data Prioritization feature is supported.
*/
#define LIBXL_HAVE_PSR_CDP 1
+
+/*
+ * LIBXL_HAVE_MCA_CAPS
+ *
+ * If this is defined, setting MCA capabilities for HVM domain is supported.
+ */
+#define LIBXL_HAVE_MCA_CAPS 1
#endif
/*
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index cf03ded..9a015eb 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -279,6 +279,17 @@ err:
libxl_bitmap_dispose(&enlightenments);
return ERROR_FAIL;
}
+
+static int hvm_set_mca_capabilities(libxl__gc *gc, uint32_t domid,
+ libxl_domain_build_info *const info)
+{
+ unsigned long caps = info->u.hvm.mca_caps;
+
+ if (!caps)
+ return 0;
+
+ return xc_hvm_param_set(CTX->xch, domid, HVM_PARAM_MCA_CAP, caps);
+}
#endif
static void hvm_set_conf_params(xc_interface *handle, uint32_t domid,
@@ -442,6 +453,10 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
rc = hvm_set_viridian_features(gc, domid, info);
if (rc)
return rc;
+
+ rc = hvm_set_mca_capabilities(gc, domid, info);
+ if (rc)
+ return rc;
#endif
}
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 69e789a..8e9d4ed 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -551,6 +551,7 @@ libxl_domain_build_info = Struct("domain_build_info",[
("serial_list", libxl_string_list),
("rdm", libxl_rdm_reserve),
("rdm_mem_boundary_memkb", MemKB),
+ ("mca_caps", uint64),
])),
("pv", Struct(None, [("kernel", string),
("slack_memkb", MemKB),
diff --git a/tools/xl/xl_parse.c b/tools/xl/xl_parse.c
index b72f990..586cb6a 100644
--- a/tools/xl/xl_parse.c
+++ b/tools/xl/xl_parse.c
@@ -18,6 +18,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <xen/hvm/e820.h>
+#include <xen/hvm/params.h>
#include <libxl.h>
#include <libxl_utils.h>
@@ -813,8 +814,9 @@ void parse_config_data(const char *config_source,
XLU_Config *config;
XLU_ConfigList *cpus, *vbds, *nics, *pcis, *cvfbs, *cpuids, *vtpms,
*usbctrls, *usbdevs;
- XLU_ConfigList *channels, *ioports, *irqs, *iomem, *viridian, *dtdevs;
- int num_ioports, num_irqs, num_iomem, num_cpus, num_viridian;
+ XLU_ConfigList *channels, *ioports, *irqs, *iomem, *viridian, *dtdevs,
+ *mca_caps;
+ int num_ioports, num_irqs, num_iomem, num_cpus, num_viridian, num_mca_caps;
int pci_power_mgmt = 0;
int pci_msitranslate = 0;
int pci_permissive = 0;
@@ -1179,6 +1181,31 @@ void parse_config_data(const char *config_source,
if (!xlu_cfg_get_long (config, "rdm_mem_boundary", &l, 0))
b_info->u.hvm.rdm_mem_boundary_memkb = l * 1024;
+
+ switch (xlu_cfg_get_list(config, "mca_caps",
+ &mca_caps, &num_mca_caps, 1))
+ {
+ case 0: /* Success */
+ for (i = 0; i < num_mca_caps; i++) {
+ buf = xlu_cfg_get_listitem(mca_caps, i);
+ if (!strcmp(buf, "lmce"))
+ b_info->u.hvm.mca_caps |= XEN_HVM_MCA_CAP_LMCE;
+ else {
+ fprintf(stderr, "ERROR: unrecognized MCA capability '%s'.\n",
+ buf);
+ exit(-ERROR_FAIL);
+ }
+ }
+ break;
+
+ case ESRCH: /* Option not present */
+ break;
+
+ default:
+ fprintf(stderr, "ERROR: unable to parse mca_caps.\n");
+ exit(-ERROR_FAIL);
+ }
+
break;
case LIBXL_DOMAIN_TYPE_PV:
{
diff --git a/xen/arch/x86/cpu/mcheck/mce.h b/xen/arch/x86/cpu/mcheck/mce.h
index 3a9134e..247d9aa 100644
--- a/xen/arch/x86/cpu/mcheck/mce.h
+++ b/xen/arch/x86/cpu/mcheck/mce.h
@@ -38,6 +38,7 @@ enum mcheck_type {
};
extern uint8_t cmci_apic_vector;
+extern bool lmce_support;
/* Init functions */
enum mcheck_type amd_mcheck_init(struct cpuinfo_x86 *c);
diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
index 8528452..a7886c5 100644
--- a/xen/arch/x86/cpu/mcheck/mce_intel.c
+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
@@ -30,7 +30,7 @@ boolean_param("mce_fb", mce_force_broadcast);
static int __read_mostly nr_intel_ext_msrs;
/* If mce_force_broadcast == 1, lmce_support will be disabled forcibly. */
-static bool __read_mostly lmce_support;
+bool __read_mostly lmce_support;
/* Intel SDM define bit15~bit0 of IA32_MCi_STATUS as the MC error code */
#define INTEL_MCCOD_MASK 0xFFFF
diff --git a/xen/arch/x86/cpu/mcheck/vmce.c b/xen/arch/x86/cpu/mcheck/vmce.c
index e19be69..704afe8 100644
--- a/xen/arch/x86/cpu/mcheck/vmce.c
+++ b/xen/arch/x86/cpu/mcheck/vmce.c
@@ -74,7 +74,7 @@ int vmce_restore_vcpu(struct vcpu *v, const struct hvm_vmce_vcpu *ctxt)
unsigned long guest_mcg_cap;
if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL )
- guest_mcg_cap = INTEL_GUEST_MCG_CAP;
+ guest_mcg_cap = INTEL_GUEST_MCG_CAP | MCG_LMCE_P;
else
guest_mcg_cap = AMD_GUEST_MCG_CAP;
@@ -546,3 +546,20 @@ int unmmap_broken_page(struct domain *d, mfn_t mfn, unsigned long gfn)
return rc;
}
+int vmce_enable_mca_cap(struct domain *d, uint64_t cap)
+{
+ struct vcpu *v;
+
+ if ( cap & ~XEN_HVM_MCA_CAP_MASK )
+ return -EINVAL;
+
+ if ( cap & XEN_HVM_MCA_CAP_LMCE )
+ {
+ if ( !lmce_support )
+ return -EINVAL;
+ for_each_vcpu(d, v)
+ v->arch.vmce.mcg_cap |= MCG_LMCE_P;
+ }
+
+ return 0;
+}
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 0282986..43def04 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -3972,6 +3972,7 @@ static int hvm_allow_set_param(struct domain *d,
case HVM_PARAM_IOREQ_SERVER_PFN:
case HVM_PARAM_NR_IOREQ_SERVER_PAGES:
case HVM_PARAM_ALTP2M:
+ case HVM_PARAM_MCA_CAP:
if ( value != 0 && a->value != value )
rc = -EEXIST;
break;
@@ -4183,6 +4184,10 @@ static int hvmop_set_param(
(0x10000 / 8) + 1) << 32);
a.value |= VM86_TSS_UPDATED;
break;
+
+ case HVM_PARAM_MCA_CAP:
+ rc = vmce_enable_mca_cap(d, a.value);
+ break;
}
if ( rc != 0 )
diff --git a/xen/include/asm-x86/mce.h b/xen/include/asm-x86/mce.h
index 35f9962..d2933c9 100644
--- a/xen/include/asm-x86/mce.h
+++ b/xen/include/asm-x86/mce.h
@@ -38,6 +38,7 @@ extern int vmce_restore_vcpu(struct vcpu *, const struct hvm_vmce_vcpu *);
extern int vmce_wrmsr(uint32_t msr, uint64_t val);
extern int vmce_rdmsr(uint32_t msr, uint64_t *val);
extern bool vmce_has_lmce(const struct vcpu *v);
+extern int vmce_enable_mca_cap(struct domain *d, uint64_t cap);
extern unsigned int nr_mce_banks;
diff --git a/xen/include/public/hvm/params.h b/xen/include/public/hvm/params.h
index 19c9eb8..e6ee2f6 100644
--- a/xen/include/public/hvm/params.h
+++ b/xen/include/public/hvm/params.h
@@ -264,6 +264,11 @@
*/
#define HVM_PARAM_VM86_TSS_SIZED 37
-#define HVM_NR_PARAMS 38
+/* Enable MCA capabilities. */
+#define HVM_PARAM_MCA_CAP 38
+#define XEN_HVM_MCA_CAP_LMCE (xen_mk_ullong(1) << 0)
+#define XEN_HVM_MCA_CAP_MASK XEN_HVM_MCA_CAP_LMCE
+
+#define HVM_NR_PARAMS 39
#endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP
2017-03-30 6:20 ` [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP Haozhong Zhang
@ 2017-03-30 14:57 ` Jan Beulich
2017-03-30 16:34 ` Wei Liu
0 siblings, 1 reply; 22+ messages in thread
From: Jan Beulich @ 2017-03-30 14:57 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, Wei Liu, Ian Jackson, xen-devel
>>> On 30.03.17 at 08:20, <haozhong.zhang@intel.com> wrote:
> If LMCE is supported by host and ' mca_caps = [ "lmce" ] ' is present
> in xl config, the LMCE capability will be exposed in guest MSR_IA32_MCG_CAP.
> By default, LMCE is not exposed to guest so as to keep the backwards migration
> compatibility.
>
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
Hypervisor side
Reviewed-by: Jan Beulich <jbeulich@suse.com>
For the tools side, doesn't the new HVM param need saving /
restoring? If so an addition to the array in
tools/libxc/xc_sr_save_x86_hvm.c:write_hvm_params() is going
to be needed.
Jan
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP
2017-03-30 14:57 ` Jan Beulich
@ 2017-03-30 16:34 ` Wei Liu
0 siblings, 0 replies; 22+ messages in thread
From: Wei Liu @ 2017-03-30 16:34 UTC (permalink / raw)
To: Jan Beulich
Cc: Haozhong Zhang, xen-devel, Wei Liu, Ian Jackson, Andrew Cooper
On Thu, Mar 30, 2017 at 08:57:05AM -0600, Jan Beulich wrote:
> >>> On 30.03.17 at 08:20, <haozhong.zhang@intel.com> wrote:
> > If LMCE is supported by host and ' mca_caps = [ "lmce" ] ' is present
> > in xl config, the LMCE capability will be exposed in guest MSR_IA32_MCG_CAP.
> > By default, LMCE is not exposed to guest so as to keep the backwards migration
> > compatibility.
> >
> > Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
>
> Hypervisor side
> Reviewed-by: Jan Beulich <jbeulich@suse.com>
>
> For the tools side, doesn't the new HVM param need saving /
> restoring? If so an addition to the array in
> tools/libxc/xc_sr_save_x86_hvm.c:write_hvm_params() is going
> to be needed.
>
Actually quite a few of HVM params are missing. It works because libxl
actually sets them, I think.
Haozhong, the best way to check is to `xl migrate guest localhost`.
Wei.
> Jan
>
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH v3 7/9] xen/mce: add support of vLMCE injection to XEN_MC_inject_v2
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (5 preceding siblings ...)
2017-03-30 6:20 ` [PATCH v3 6/9] x86/vmce, tools/libxl: expose LMCE capability in guest MSR_IA32_MCG_CAP Haozhong Zhang
@ 2017-03-30 6:20 ` Haozhong Zhang
2017-03-30 15:01 ` Jan Beulich
2017-03-30 6:20 ` [PATCH v3 8/9] tools/libxc: add support of injecting MC# to specified CPUs Haozhong Zhang
2017-03-30 6:20 ` [PATCH v3 9/9] tools/xen-mceinj: add support of injecting LMCE Haozhong Zhang
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:20 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Jan Beulich, Andrew Cooper
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Changes in v3:
* Add a check to avoid XEN_MC_INJECT_CPU_BROADCAST being used
with XEN_MC_INJECT_TYPE_LMCE.
---
xen/arch/x86/cpu/mcheck/mce.c | 25 ++++++++++++++++++++++++-
xen/include/public/arch-x86/xen-mca.h | 1 +
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c
index c1a59bf..0de19ae 100644
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -1521,11 +1521,12 @@ long do_mca(XEN_GUEST_HANDLE_PARAM(xen_mc_t) u_xen_mc)
{
const cpumask_t *cpumap;
cpumask_var_t cmv;
+ bool broadcast = op->u.mc_inject_v2.flags & XEN_MC_INJECT_CPU_BROADCAST;
if (nr_mce_banks == 0)
return x86_mcerr("do_mca #MC", -ENODEV);
- if ( op->u.mc_inject_v2.flags & XEN_MC_INJECT_CPU_BROADCAST )
+ if ( broadcast )
cpumap = &cpu_online_map;
else
{
@@ -1565,6 +1566,28 @@ long do_mca(XEN_GUEST_HANDLE_PARAM(xen_mc_t) u_xen_mc)
}
break;
+ case XEN_MC_INJECT_TYPE_LMCE:
+ if ( !lmce_support )
+ {
+ ret = x86_mcerr("No LMCE support in platform", -EINVAL);
+ break;
+ }
+ if ( broadcast )
+ {
+ ret = x86_mcerr("XEN_MC_INJECT_CPU_BROADCAST cannot be used "
+ "with XEN_MC_INJECT_TYPE_LMCE",
+ -EINVAL);
+ break;
+ }
+ /* Ensure at most one CPU is specified. */
+ if ( nr_cpu_ids > cpumask_next(cpumask_first(cpumap), cpumap) )
+ {
+ ret = x86_mcerr("More than one CPU specified", -EINVAL);
+ break;
+ }
+ on_selected_cpus(cpumap, x86_mc_mceinject, NULL, 1);
+ break;
+
default:
ret = x86_mcerr("Wrong mca type\n", -EINVAL);
break;
diff --git a/xen/include/public/arch-x86/xen-mca.h b/xen/include/public/arch-x86/xen-mca.h
index 7db9907..dc35267 100644
--- a/xen/include/public/arch-x86/xen-mca.h
+++ b/xen/include/public/arch-x86/xen-mca.h
@@ -414,6 +414,7 @@ struct xen_mc_mceinject {
#define XEN_MC_INJECT_TYPE_MASK 0x7
#define XEN_MC_INJECT_TYPE_MCE 0x0
#define XEN_MC_INJECT_TYPE_CMCI 0x1
+#define XEN_MC_INJECT_TYPE_LMCE 0x2
#define XEN_MC_INJECT_CPU_BROADCAST 0x8
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH v3 7/9] xen/mce: add support of vLMCE injection to XEN_MC_inject_v2
2017-03-30 6:20 ` [PATCH v3 7/9] xen/mce: add support of vLMCE injection to XEN_MC_inject_v2 Haozhong Zhang
@ 2017-03-30 15:01 ` Jan Beulich
0 siblings, 0 replies; 22+ messages in thread
From: Jan Beulich @ 2017-03-30 15:01 UTC (permalink / raw)
To: Haozhong Zhang; +Cc: Andrew Cooper, xen-devel
>>> On 30.03.17 at 08:20, <haozhong.zhang@intel.com> wrote:
> @@ -1565,6 +1566,28 @@ long do_mca(XEN_GUEST_HANDLE_PARAM(xen_mc_t) u_xen_mc)
> }
> break;
>
> + case XEN_MC_INJECT_TYPE_LMCE:
> + if ( !lmce_support )
> + {
> + ret = x86_mcerr("No LMCE support in platform", -EINVAL);
Please drop "in platform".
> + break;
> + }
> + if ( broadcast )
> + {
> + ret = x86_mcerr("XEN_MC_INJECT_CPU_BROADCAST cannot be used "
> + "with XEN_MC_INJECT_TYPE_LMCE",
Please shorten this by dropping the symbolic prefixes. Perhaps simply
"Broadcast cannot be used with LMCE"?
> + -EINVAL);
> + break;
> + }
> + /* Ensure at most one CPU is specified. */
> + if ( nr_cpu_ids > cpumask_next(cpumask_first(cpumap), cpumap) )
> + {
> + ret = x86_mcerr("More than one CPU specified", -EINVAL);
"... for LMCE"
With that
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Jan
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH v3 8/9] tools/libxc: add support of injecting MC# to specified CPUs
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (6 preceding siblings ...)
2017-03-30 6:20 ` [PATCH v3 7/9] xen/mce: add support of vLMCE injection to XEN_MC_inject_v2 Haozhong Zhang
@ 2017-03-30 6:20 ` Haozhong Zhang
2017-03-31 11:14 ` Wei Liu
2017-03-30 6:20 ` [PATCH v3 9/9] tools/xen-mceinj: add support of injecting LMCE Haozhong Zhang
8 siblings, 1 reply; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:20 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Ian Jackson, Wei Liu
Though XEN_MC_inject_v2 allows injecting MC# to specified CPUs, the
current xc_mca_op() does not use this feature and not provide an
interface to callers. This commit add a new xc_mca_op_inject_v2() that
receives a cpumap providing the set of target CPUs.
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Changes in v3:
* Use goto error handling style.
---
tools/libxc/include/xenctrl.h | 2 ++
tools/libxc/xc_misc.c | 52 ++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index 2d97d36..2399956 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -1796,6 +1796,8 @@ int xc_cpuid_apply_policy(xc_interface *xch,
void xc_cpuid_to_str(const unsigned int *regs,
char **strs); /* some strs[] may be NULL if ENOMEM */
int xc_mca_op(xc_interface *xch, struct xen_mc *mc);
+int xc_mca_op_inject_v2(xc_interface *xch, unsigned int flags,
+ xc_cpumap_t cpumap, unsigned int nr_cpus);
#endif
struct xc_px_val {
diff --git a/tools/libxc/xc_misc.c b/tools/libxc/xc_misc.c
index 88084fd..2303293 100644
--- a/tools/libxc/xc_misc.c
+++ b/tools/libxc/xc_misc.c
@@ -341,7 +341,57 @@ int xc_mca_op(xc_interface *xch, struct xen_mc *mc)
xc_hypercall_bounce_post(xch, mc);
return ret;
}
-#endif
+
+int xc_mca_op_inject_v2(xc_interface *xch, unsigned int flags,
+ xc_cpumap_t cpumap, unsigned int nr_bits)
+{
+ int ret = -1;
+ struct xen_mc mc_buf, *mc = &mc_buf;
+ struct xen_mc_inject_v2 *inject = &mc->u.mc_inject_v2;
+
+ DECLARE_HYPERCALL_BOUNCE(cpumap, 0, XC_HYPERCALL_BUFFER_BOUNCE_IN);
+ DECLARE_HYPERCALL_BOUNCE(mc, sizeof(*mc), XC_HYPERCALL_BUFFER_BOUNCE_BOTH);
+
+ memset(mc, 0, sizeof(*mc));
+
+ if ( cpumap )
+ {
+ if ( !nr_bits )
+ {
+ errno = EINVAL;
+ goto out;
+ }
+
+ HYPERCALL_BOUNCE_SET_SIZE(cpumap, (nr_bits + 7) / 8);
+ if ( xc_hypercall_bounce_pre(xch, cpumap) )
+ {
+ PERROR("Could not bounce cpumap memory buffer");
+ goto out;
+ }
+ set_xen_guest_handle(inject->cpumap.bitmap, cpumap);
+ inject->cpumap.nr_bits = nr_bits;
+ }
+
+ inject->flags = flags;
+ mc->cmd = XEN_MC_inject_v2;
+ mc->interface_version = XEN_MCA_INTERFACE_VERSION;
+
+ if ( xc_hypercall_bounce_pre(xch, mc) )
+ {
+ PERROR("Could not bounce xen_mc memory buffer");
+ goto out_free_cpumap;
+ }
+
+ ret = xencall1(xch->xcall, __HYPERVISOR_mca, HYPERCALL_BUFFER_AS_ARG(mc));
+
+ xc_hypercall_bounce_post(xch, mc);
+out_free_cpumap:
+ if ( cpumap )
+ xc_hypercall_bounce_post(xch, cpumap);
+out:
+ return ret;
+}
+#endif /* __i386__ || __x86_64__ */
int xc_perfc_reset(xc_interface *xch)
{
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH v3 9/9] tools/xen-mceinj: add support of injecting LMCE
2017-03-30 6:19 [PATCH v3 0/9] Add LMCE support Haozhong Zhang
` (7 preceding siblings ...)
2017-03-30 6:20 ` [PATCH v3 8/9] tools/libxc: add support of injecting MC# to specified CPUs Haozhong Zhang
@ 2017-03-30 6:20 ` Haozhong Zhang
8 siblings, 0 replies; 22+ messages in thread
From: Haozhong Zhang @ 2017-03-30 6:20 UTC (permalink / raw)
To: xen-devel; +Cc: Haozhong Zhang, Ian Jackson, Wei Liu
If option '-l' or '--lmce' is specified and the host supports LMCE,
xen-mceinj will inject LMCE to CPU specified by '-c' (or CPU0 if '-c'
is not present).
Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
---
Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
---
tools/tests/mce-test/tools/xen-mceinj.c | 50 +++++++++++++++++++++++++++++++--
1 file changed, 48 insertions(+), 2 deletions(-)
diff --git a/tools/tests/mce-test/tools/xen-mceinj.c b/tools/tests/mce-test/tools/xen-mceinj.c
index bae5a46..380e421 100644
--- a/tools/tests/mce-test/tools/xen-mceinj.c
+++ b/tools/tests/mce-test/tools/xen-mceinj.c
@@ -56,6 +56,8 @@
#define MSR_IA32_MC0_MISC 0x00000403
#define MSR_IA32_MC0_CTL2 0x00000280
+#define MCG_STATUS_LMCE 0x8
+
struct mce_info {
const char *description;
uint8_t mcg_stat;
@@ -113,6 +115,7 @@ static struct mce_info mce_table[] = {
#define LOGFILE stdout
int dump;
+int lmce;
struct xen_mc_msrinject msr_inj;
static void Lprintf(const char *fmt, ...)
@@ -212,6 +215,35 @@ static int inject_mce(xc_interface *xc_handle, int cpu_nr)
return xc_mca_op(xc_handle, &mc);
}
+static int inject_lmce(xc_interface *xc_handle, unsigned int cpu)
+{
+ uint8_t *cpumap = NULL;
+ size_t cpumap_size, line, shift;
+ unsigned int nr_cpus;
+ int ret;
+
+ nr_cpus = mca_cpuinfo(xc_handle);
+ if ( !nr_cpus )
+ err(xc_handle, "Failed to get mca_cpuinfo");
+ if ( cpu >= nr_cpus )
+ err(xc_handle, "-c %u is larger than %u", cpu, nr_cpus - 1);
+
+ cpumap_size = (nr_cpus + 7) / 8;
+ cpumap = malloc(cpumap_size);
+ if ( !cpumap )
+ err(xc_handle, "Failed to allocate cpumap\n");
+ memset(cpumap, 0, cpumap_size);
+ line = cpu / 8;
+ shift = cpu % 8;
+ memset(cpumap + line, 1 << shift, 1);
+
+ ret = xc_mca_op_inject_v2(xc_handle, XEN_MC_INJECT_TYPE_LMCE,
+ cpumap, cpumap_size * 8);
+
+ free(cpumap);
+ return ret;
+}
+
static uint64_t bank_addr(int bank, int type)
{
uint64_t addr;
@@ -330,8 +362,15 @@ static int inject(xc_interface *xc_handle, struct mce_info *mce,
uint32_t cpu_nr, uint32_t domain, uint64_t gaddr)
{
int ret = 0;
+ uint8_t mcg_status = mce->mcg_stat;
- ret = inject_mcg_status(xc_handle, cpu_nr, mce->mcg_stat, domain);
+ if ( lmce )
+ {
+ if ( mce->cmci )
+ err(xc_handle, "No support to inject CMCI as LMCE");
+ mcg_status |= MCG_STATUS_LMCE;
+ }
+ ret = inject_mcg_status(xc_handle, cpu_nr, mcg_status, domain);
if ( ret )
err(xc_handle, "Failed to inject MCG_STATUS MSR");
@@ -354,6 +393,8 @@ static int inject(xc_interface *xc_handle, struct mce_info *mce,
err(xc_handle, "Failed to inject MSR");
if ( mce->cmci )
ret = inject_cmci(xc_handle, cpu_nr);
+ else if ( lmce )
+ ret = inject_lmce(xc_handle, cpu_nr);
else
ret = inject_mce(xc_handle, cpu_nr);
if ( ret )
@@ -393,6 +434,7 @@ static struct option opts[] = {
{"dump", 0, 0, 'D'},
{"help", 0, 0, 'h'},
{"page", 0, 0, 'p'},
+ {"lmce", 0, 0, 'l'},
{"", 0, 0, '\0'}
};
@@ -409,6 +451,7 @@ static void help(void)
" -d, --domain=DOMID target domain, the default is Xen itself\n"
" -h, --help print this page\n"
" -p, --page=ADDR physical address to report\n"
+ " -l, --lmce inject as LMCE (Intel only)\n"
" -t, --type=ERROR error type\n");
for ( i = 0; i < MCE_TABLE_SIZE; i++ )
@@ -438,7 +481,7 @@ int main(int argc, char *argv[])
}
while ( 1 ) {
- c = getopt_long(argc, argv, "c:Dd:t:hp:", opts, &opt_index);
+ c = getopt_long(argc, argv, "c:Dd:t:hp:l", opts, &opt_index);
if ( c == -1 )
break;
switch ( c ) {
@@ -463,6 +506,9 @@ int main(int argc, char *argv[])
case 't':
type = strtol(optarg, NULL, 0);
break;
+ case 'l':
+ lmce = 1;
+ break;
case 'h':
default:
help();
--
2.10.1
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related [flat|nested] 22+ messages in thread