* [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 11:11 ` Andrew Cooper
2013-11-20 3:27 ` [PATCH 2/8] x86: handle CQM resource when creating/destroying guests dongxiao.xu
` (6 subsequent siblings)
7 siblings, 1 reply; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Detect platform QoS feature status and enumerate the resource types,
one of which is to monitor the L3 cache occupancy.
Also introduce a Xen grub command line parameter to control the
QoS feature status globally.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
xen/arch/x86/Makefile | 1 +
xen/arch/x86/cpu/intel.c | 6 +++
xen/arch/x86/pqos.c | 89 ++++++++++++++++++++++++++++++++++++++
xen/arch/x86/setup.c | 3 ++
xen/include/asm-x86/cpufeature.h | 1 +
xen/include/asm-x86/pqos.h | 37 ++++++++++++++++
6 files changed, 137 insertions(+)
create mode 100644 xen/arch/x86/pqos.c
create mode 100644 xen/include/asm-x86/pqos.h
diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile
index d502bdf..54962e0 100644
--- a/xen/arch/x86/Makefile
+++ b/xen/arch/x86/Makefile
@@ -58,6 +58,7 @@ obj-y += crash.o
obj-y += tboot.o
obj-y += hpet.o
obj-y += xstate.o
+obj-y += pqos.o
obj-$(crash_debug) += gdbstub.o
diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c
index 27fe762..f0d83ea 100644
--- a/xen/arch/x86/cpu/intel.c
+++ b/xen/arch/x86/cpu/intel.c
@@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c)
( c->cpuid_level >= 0x00000006 ) &&
( cpuid_eax(0x00000006) & (1u<<2) ) )
set_bit(X86_FEATURE_ARAT, c->x86_capability);
+
+ /* Check platform QoS monitoring capability */
+ if ((c->cpuid_level >= 0x00000007) &&
+ (cpuid_ebx(0x00000007) & (1u<<12)))
+ set_bit(X86_FEATURE_QOSM, c->x86_capability);
+
}
static struct cpu_dev intel_cpu_dev __cpuinitdata = {
diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
new file mode 100644
index 0000000..617cce1
--- /dev/null
+++ b/xen/arch/x86/pqos.c
@@ -0,0 +1,89 @@
+/*
+ * pqos.c: Platform QoS related service for guest.
+ *
+ * Copyright (c) 2013, Intel Corporation
+ * Author: Jiongxi Li <jiongxi.li@intel.com>
+ * Author: Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+#include <asm/processor.h>
+#include <xen/init.h>
+#include <asm/pqos.h>
+
+static bool_t pqos_enabled = 1;
+boolean_param("pqos", pqos_enabled);
+
+unsigned int cqm_res_count = 0;
+unsigned int cqm_upscaling_factor = 0;
+bool_t cqm_enabled = 0;
+struct cqm_res_struct *cqm_res_array = NULL;
+
+static void init_cqm(void)
+{
+ unsigned int eax, edx;
+ unsigned int max_cqm_rmid;
+
+ cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid, &edx);
+ if ( !(edx & QOS_MONITOR_EVTID_L3) )
+ return;
+
+ cqm_res_count = max_cqm_rmid + 1;
+
+ cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count);
+ if ( !cqm_res_array )
+ {
+ cqm_res_count = 0;
+ return;
+ }
+
+ cqm_enabled = 1;
+
+ /* Reserve RMID 0 for all domains not being monitored */
+ cqm_res_array[0].inuse = 1;
+
+ printk("Cache QoS Monitoring Enabled.\n");
+}
+
+static void init_qos_monitor(void)
+{
+ unsigned int qm_features;
+ unsigned int eax, ebx, ecx;
+
+ if ( !(boot_cpu_has(X86_FEATURE_QOSM)) )
+ return;
+
+ cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
+
+ if ( qm_features & QOS_MONITOR_TYPE_L3 )
+ init_cqm();
+}
+
+void init_platform_qos(void)
+{
+ if ( !pqos_enabled )
+ return;
+
+ init_qos_monitor();
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 5bf4ee0..95418e4 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -48,6 +48,7 @@
#include <asm/setup.h>
#include <xen/cpu.h>
#include <asm/nmi.h>
+#include <asm/pqos.h>
/* opt_nosmp: If true, secondary processors are ignored. */
static bool_t __initdata opt_nosmp;
@@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p)
domain_unpause_by_systemcontroller(dom0);
+ init_platform_qos();
+
reset_stack_and_jump(init_done);
}
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 1cfaf94..ca59668 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -147,6 +147,7 @@
#define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */
#define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context ID */
#define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional Memory */
+#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring capability */
#define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */
#define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access Prevention */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
new file mode 100644
index 0000000..2184ea9
--- /dev/null
+++ b/xen/include/asm-x86/pqos.h
@@ -0,0 +1,37 @@
+/*
+ * pqos.h: Platform QoS related service for guest.
+ *
+ * Copyright (c) 2013, Intel Corporation
+ * Author: Jiongxi Li <jiongxi.li@intel.com>
+ * Author: Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+#ifndef ASM_PQOS_H
+#define ASM_PQOS_H
+
+/* QoS Resource Type Enumeration */
+#define QOS_MONITOR_TYPE_L3 0x2
+
+/* QoS Monitoring Event ID */
+#define QOS_MONITOR_EVTID_L3 0x1
+
+struct cqm_res_struct {
+ bool_t inuse;
+ uint16_t domain_id;
+};
+
+void init_platform_qos(void);
+
+#endif
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
2013-11-20 3:27 ` [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature dongxiao.xu
@ 2013-11-20 11:11 ` Andrew Cooper
2013-11-20 12:40 ` Xu, Dongxiao
0 siblings, 1 reply; 17+ messages in thread
From: Andrew Cooper @ 2013-11-20 11:11 UTC (permalink / raw)
To: dongxiao.xu; +Cc: xen-devel
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> From: Dongxiao Xu <dongxiao.xu@intel.com>
>
> Detect platform QoS feature status and enumerate the resource types,
> one of which is to monitor the L3 cache occupancy.
>
> Also introduce a Xen grub command line parameter to control the
> QoS feature status globally.
>
> Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
> xen/arch/x86/Makefile | 1 +
> xen/arch/x86/cpu/intel.c | 6 +++
> xen/arch/x86/pqos.c | 89 ++++++++++++++++++++++++++++++++++++++
> xen/arch/x86/setup.c | 3 ++
> xen/include/asm-x86/cpufeature.h | 1 +
> xen/include/asm-x86/pqos.h | 37 ++++++++++++++++
> 6 files changed, 137 insertions(+)
> create mode 100644 xen/arch/x86/pqos.c
> create mode 100644 xen/include/asm-x86/pqos.h
>
> diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile
> index d502bdf..54962e0 100644
> --- a/xen/arch/x86/Makefile
> +++ b/xen/arch/x86/Makefile
> @@ -58,6 +58,7 @@ obj-y += crash.o
> obj-y += tboot.o
> obj-y += hpet.o
> obj-y += xstate.o
> +obj-y += pqos.o
>
> obj-$(crash_debug) += gdbstub.o
>
> diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c
> index 27fe762..f0d83ea 100644
> --- a/xen/arch/x86/cpu/intel.c
> +++ b/xen/arch/x86/cpu/intel.c
> @@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c)
> ( c->cpuid_level >= 0x00000006 ) &&
> ( cpuid_eax(0x00000006) & (1u<<2) ) )
> set_bit(X86_FEATURE_ARAT, c->x86_capability);
> +
> + /* Check platform QoS monitoring capability */
> + if ((c->cpuid_level >= 0x00000007) &&
> + (cpuid_ebx(0x00000007) & (1u<<12)))
> + set_bit(X86_FEATURE_QOSM, c->x86_capability);
> +
> }
>
> static struct cpu_dev intel_cpu_dev __cpuinitdata = {
> diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> new file mode 100644
> index 0000000..617cce1
> --- /dev/null
> +++ b/xen/arch/x86/pqos.c
> @@ -0,0 +1,89 @@
> +/*
> + * pqos.c: Platform QoS related service for guest.
> + *
> + * Copyright (c) 2013, Intel Corporation
> + * Author: Jiongxi Li <jiongxi.li@intel.com>
> + * Author: Dongxiao Xu <dongxiao.xu@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> + * Place - Suite 330, Boston, MA 02111-1307 USA.
> + */
> +#include <asm/processor.h>
> +#include <xen/init.h>
> +#include <asm/pqos.h>
> +
> +static bool_t pqos_enabled = 1;
> +boolean_param("pqos", pqos_enabled);
> +
> +unsigned int cqm_res_count = 0;
> +unsigned int cqm_upscaling_factor = 0;
> +bool_t cqm_enabled = 0;
> +struct cqm_res_struct *cqm_res_array = NULL;
> +
> +static void init_cqm(void)
__init
> +{
> + unsigned int eax, edx;
> + unsigned int max_cqm_rmid;
> +
> + cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid, &edx);
> + if ( !(edx & QOS_MONITOR_EVTID_L3) )
> + return;
> +
> + cqm_res_count = max_cqm_rmid + 1;
> +
> + cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count);
> + if ( !cqm_res_array )
> + {
> + cqm_res_count = 0;
> + return;
> + }
> +
> + cqm_enabled = 1;
> +
> + /* Reserve RMID 0 for all domains not being monitored */
> + cqm_res_array[0].inuse = 1;
> +
> + printk("Cache QoS Monitoring Enabled.\n");
This should probably be a XENLOG_INFO, and we tend to omit full stops.
> +}
> +
> +static void init_qos_monitor(void)
__init
> +{
> + unsigned int qm_features;
> + unsigned int eax, ebx, ecx;
> +
> + if ( !(boot_cpu_has(X86_FEATURE_QOSM)) )
> + return;
> +
> + cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
> +
> + if ( qm_features & QOS_MONITOR_TYPE_L3 )
> + init_cqm();
> +}
> +
> +void init_platform_qos(void)
__init
Are there expected to be future platform qos monitoring abilities?
> +{
> + if ( !pqos_enabled )
> + return;
> +
> + init_qos_monitor();
> +}
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-file-style: "BSD"
> + * c-basic-offset: 4
> + * tab-width: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
> index 5bf4ee0..95418e4 100644
> --- a/xen/arch/x86/setup.c
> +++ b/xen/arch/x86/setup.c
> @@ -48,6 +48,7 @@
> #include <asm/setup.h>
> #include <xen/cpu.h>
> #include <asm/nmi.h>
> +#include <asm/pqos.h>
>
> /* opt_nosmp: If true, secondary processors are ignored. */
> static bool_t __initdata opt_nosmp;
> @@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p)
>
> domain_unpause_by_systemcontroller(dom0);
>
> + init_platform_qos();
> +
> reset_stack_and_jump(init_done);
> }
>
> diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
> index 1cfaf94..ca59668 100644
> --- a/xen/include/asm-x86/cpufeature.h
> +++ b/xen/include/asm-x86/cpufeature.h
> @@ -147,6 +147,7 @@
> #define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */
> #define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context ID */
> #define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional Memory */
> +#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring capability */
> #define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */
> #define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access Prevention */
>
> diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> new file mode 100644
> index 0000000..2184ea9
> --- /dev/null
> +++ b/xen/include/asm-x86/pqos.h
> @@ -0,0 +1,37 @@
> +/*
> + * pqos.h: Platform QoS related service for guest.
> + *
> + * Copyright (c) 2013, Intel Corporation
> + * Author: Jiongxi Li <jiongxi.li@intel.com>
> + * Author: Dongxiao Xu <dongxiao.xu@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> + * Place - Suite 330, Boston, MA 02111-1307 USA.
> + */
> +#ifndef ASM_PQOS_H
> +#define ASM_PQOS_H
> +
> +/* QoS Resource Type Enumeration */
> +#define QOS_MONITOR_TYPE_L3 0x2
> +
> +/* QoS Monitoring Event ID */
> +#define QOS_MONITOR_EVTID_L3 0x1
> +
> +struct cqm_res_struct {
> + bool_t inuse;
> + uint16_t domain_id;
domid_t please. Also, reversing the order of the parameters would be a
good idea, packing-wise.
~Andrew
> +};
> +
> +void init_platform_qos(void);
> +
> +#endif
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
2013-11-20 11:11 ` Andrew Cooper
@ 2013-11-20 12:40 ` Xu, Dongxiao
0 siblings, 0 replies; 17+ messages in thread
From: Xu, Dongxiao @ 2013-11-20 12:40 UTC (permalink / raw)
To: Andrew Cooper; +Cc: xen-devel@lists.xen.org
> -----Original Message-----
> From: Andrew Cooper [mailto:andrew.cooper3@citrix.com]
> Sent: Wednesday, November 20, 2013 7:12 PM
> To: Xu, Dongxiao
> Cc: xen-devel@lists.xen.org
> Subject: Re: [Xen-devel] [PATCH 1/8] x86: detect and initialize Cache QoS
> Monitoring feature
>
> On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> > From: Dongxiao Xu <dongxiao.xu@intel.com>
> >
> > Detect platform QoS feature status and enumerate the resource types,
> > one of which is to monitor the L3 cache occupancy.
> >
> > Also introduce a Xen grub command line parameter to control the
> > QoS feature status globally.
> >
> > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> > ---
> > xen/arch/x86/Makefile | 1 +
> > xen/arch/x86/cpu/intel.c | 6 +++
> > xen/arch/x86/pqos.c | 89
> ++++++++++++++++++++++++++++++++++++++
> > xen/arch/x86/setup.c | 3 ++
> > xen/include/asm-x86/cpufeature.h | 1 +
> > xen/include/asm-x86/pqos.h | 37 ++++++++++++++++
> > 6 files changed, 137 insertions(+)
> > create mode 100644 xen/arch/x86/pqos.c
> > create mode 100644 xen/include/asm-x86/pqos.h
> >
> > diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile
> > index d502bdf..54962e0 100644
> > --- a/xen/arch/x86/Makefile
> > +++ b/xen/arch/x86/Makefile
> > @@ -58,6 +58,7 @@ obj-y += crash.o
> > obj-y += tboot.o
> > obj-y += hpet.o
> > obj-y += xstate.o
> > +obj-y += pqos.o
> >
> > obj-$(crash_debug) += gdbstub.o
> >
> > diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c
> > index 27fe762..f0d83ea 100644
> > --- a/xen/arch/x86/cpu/intel.c
> > +++ b/xen/arch/x86/cpu/intel.c
> > @@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c)
> > ( c->cpuid_level >= 0x00000006 ) &&
> > ( cpuid_eax(0x00000006) & (1u<<2) ) )
> > set_bit(X86_FEATURE_ARAT, c->x86_capability);
> > +
> > + /* Check platform QoS monitoring capability */
> > + if ((c->cpuid_level >= 0x00000007) &&
> > + (cpuid_ebx(0x00000007) & (1u<<12)))
> > + set_bit(X86_FEATURE_QOSM, c->x86_capability);
> > +
> > }
> >
> > static struct cpu_dev intel_cpu_dev __cpuinitdata = {
> > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> > new file mode 100644
> > index 0000000..617cce1
> > --- /dev/null
> > +++ b/xen/arch/x86/pqos.c
> > @@ -0,0 +1,89 @@
> > +/*
> > + * pqos.c: Platform QoS related service for guest.
> > + *
> > + * Copyright (c) 2013, Intel Corporation
> > + * Author: Jiongxi Li <jiongxi.li@intel.com>
> > + * Author: Dongxiao Xu <dongxiao.xu@intel.com>
> > + *
> > + * This program is free software; you can redistribute it and/or modify it
> > + * under the terms and conditions of the GNU General Public License,
> > + * version 2, as published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope it will be useful, but WITHOUT
> > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
> or
> > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
> License for
> > + * more details.
> > + *
> > + * You should have received a copy of the GNU General Public License along
> with
> > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> > + * Place - Suite 330, Boston, MA 02111-1307 USA.
> > + */
> > +#include <asm/processor.h>
> > +#include <xen/init.h>
> > +#include <asm/pqos.h>
> > +
> > +static bool_t pqos_enabled = 1;
> > +boolean_param("pqos", pqos_enabled);
> > +
> > +unsigned int cqm_res_count = 0;
> > +unsigned int cqm_upscaling_factor = 0;
> > +bool_t cqm_enabled = 0;
> > +struct cqm_res_struct *cqm_res_array = NULL;
> > +
> > +static void init_cqm(void)
>
> __init
OK.
>
> > +{
> > + unsigned int eax, edx;
> > + unsigned int max_cqm_rmid;
> > +
> > + cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid,
> &edx);
> > + if ( !(edx & QOS_MONITOR_EVTID_L3) )
> > + return;
> > +
> > + cqm_res_count = max_cqm_rmid + 1;
> > +
> > + cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count);
> > + if ( !cqm_res_array )
> > + {
> > + cqm_res_count = 0;
> > + return;
> > + }
> > +
> > + cqm_enabled = 1;
> > +
> > + /* Reserve RMID 0 for all domains not being monitored */
> > + cqm_res_array[0].inuse = 1;
> > +
> > + printk("Cache QoS Monitoring Enabled.\n");
>
> This should probably be a XENLOG_INFO, and we tend to omit full stops.
OK.
>
> > +}
> > +
> > +static void init_qos_monitor(void)
>
> __init
OK.
>
> > +{
> > + unsigned int qm_features;
> > + unsigned int eax, ebx, ecx;
> > +
> > + if ( !(boot_cpu_has(X86_FEATURE_QOSM)) )
> > + return;
> > +
> > + cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
> > +
> > + if ( qm_features & QOS_MONITOR_TYPE_L3 )
> > + init_cqm();
> > +}
> > +
> > +void init_platform_qos(void)
>
> __init
OK.
>
> Are there expected to be future platform qos monitoring abilities?
Yes.
>
> > +{
> > + if ( !pqos_enabled )
> > + return;
> > +
> > + init_qos_monitor();
> > +}
> > +
> > +/*
> > + * Local variables:
> > + * mode: C
> > + * c-file-style: "BSD"
> > + * c-basic-offset: 4
> > + * tab-width: 4
> > + * indent-tabs-mode: nil
> > + * End:
> > + */
> > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
> > index 5bf4ee0..95418e4 100644
> > --- a/xen/arch/x86/setup.c
> > +++ b/xen/arch/x86/setup.c
> > @@ -48,6 +48,7 @@
> > #include <asm/setup.h>
> > #include <xen/cpu.h>
> > #include <asm/nmi.h>
> > +#include <asm/pqos.h>
> >
> > /* opt_nosmp: If true, secondary processors are ignored. */
> > static bool_t __initdata opt_nosmp;
> > @@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p)
> >
> > domain_unpause_by_systemcontroller(dom0);
> >
> > + init_platform_qos();
> > +
> > reset_stack_and_jump(init_done);
> > }
> >
> > diff --git a/xen/include/asm-x86/cpufeature.h
> b/xen/include/asm-x86/cpufeature.h
> > index 1cfaf94..ca59668 100644
> > --- a/xen/include/asm-x86/cpufeature.h
> > +++ b/xen/include/asm-x86/cpufeature.h
> > @@ -147,6 +147,7 @@
> > #define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */
> > #define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context
> ID */
> > #define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional
> Memory */
> > +#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring
> capability */
> > #define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as
> zero */
> > #define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access
> Prevention */
> >
> > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> > new file mode 100644
> > index 0000000..2184ea9
> > --- /dev/null
> > +++ b/xen/include/asm-x86/pqos.h
> > @@ -0,0 +1,37 @@
> > +/*
> > + * pqos.h: Platform QoS related service for guest.
> > + *
> > + * Copyright (c) 2013, Intel Corporation
> > + * Author: Jiongxi Li <jiongxi.li@intel.com>
> > + * Author: Dongxiao Xu <dongxiao.xu@intel.com>
> > + *
> > + * This program is free software; you can redistribute it and/or modify it
> > + * under the terms and conditions of the GNU General Public License,
> > + * version 2, as published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope it will be useful, but WITHOUT
> > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
> or
> > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
> License for
> > + * more details.
> > + *
> > + * You should have received a copy of the GNU General Public License along
> with
> > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
> > + * Place - Suite 330, Boston, MA 02111-1307 USA.
> > + */
> > +#ifndef ASM_PQOS_H
> > +#define ASM_PQOS_H
> > +
> > +/* QoS Resource Type Enumeration */
> > +#define QOS_MONITOR_TYPE_L3 0x2
> > +
> > +/* QoS Monitoring Event ID */
> > +#define QOS_MONITOR_EVTID_L3 0x1
> > +
> > +struct cqm_res_struct {
> > + bool_t inuse;
> > + uint16_t domain_id;
>
> domid_t please. Also, reversing the order of the parameters would be a
> good idea, packing-wise.
OK.
Will reflect above suggestions in next version patches.
Thanks,
Dongxiao
>
> ~Andrew
>
> > +};
> > +
> > +void init_platform_qos(void);
> > +
> > +#endif
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 2/8] x86: handle CQM resource when creating/destroying guests
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
2013-11-20 3:27 ` [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 11:32 ` Andrew Cooper
2013-11-20 3:27 ` [PATCH 3/8] tools: " dongxiao.xu
` (5 subsequent siblings)
7 siblings, 1 reply; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Allocate an RMID for a guest when it is created. This per-guest
RMID will be used to monitor Cache QoS related data. The RMID will
be relinquished when guest is destroyed.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
xen/arch/x86/domain.c | 12 +++++++++
xen/arch/x86/pqos.c | 58 ++++++++++++++++++++++++++++++++++++++++++
xen/common/domctl.c | 5 +++-
xen/include/asm-x86/domain.h | 2 ++
xen/include/asm-x86/pqos.h | 4 +++
xen/include/public/domctl.h | 3 +++
xen/include/xen/sched.h | 3 +++
7 files changed, 86 insertions(+), 1 deletion(-)
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index a3868f9..5f920cd 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -60,6 +60,7 @@
#include <xen/numa.h>
#include <xen/iommu.h>
#include <compat/vcpu.h>
+#include <asm/pqos.h>
DEFINE_PER_CPU(struct vcpu *, curr_vcpu);
DEFINE_PER_CPU(unsigned long, cr4);
@@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags)
tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
spin_lock_init(&d->arch.vtsc_lock);
+ /* Allocate CQM RMID for guest */
+ if ( !!(domcr_flags & DOMCRF_pqos_cqm) )
+ d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id);
+ else if ( system_support_cqm() )
+ d->arch.pqos_cqm_rmid = 0;
+ else
+ d->arch.pqos_cqm_rmid = -1;
+
return 0;
fail:
@@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d)
free_xenheap_page(d->shared_info);
cleanup_domain_irq_mapping(d);
+
+ free_cqm_resource(d->domain_id);
+ d->arch.pqos_cqm_rmid = 0;
}
unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long guest_cr4)
diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
index 617cce1..895d892 100644
--- a/xen/arch/x86/pqos.c
+++ b/xen/arch/x86/pqos.c
@@ -20,6 +20,7 @@
*/
#include <asm/processor.h>
#include <xen/init.h>
+#include <xen/spinlock.h>
#include <asm/pqos.h>
static bool_t pqos_enabled = 1;
@@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0;
unsigned int cqm_upscaling_factor = 0;
bool_t cqm_enabled = 0;
struct cqm_res_struct *cqm_res_array = NULL;
+static DEFINE_SPINLOCK(cqm_lock);
static void init_cqm(void)
{
@@ -78,6 +80,62 @@ void init_platform_qos(void)
init_qos_monitor();
}
+bool_t system_support_cqm(void)
+{
+ return cqm_enabled;
+}
+
+int alloc_cqm_resource(domid_t domain_id)
+{
+ int i, rmid = -1;
+ unsigned long flags;
+
+ if ( !system_support_cqm() )
+ goto out;
+
+ spin_lock_irqsave(&cqm_lock, flags);
+ for ( i = 0; i < cqm_res_count; i++ )
+ {
+ if ( cqm_res_array[i].inuse)
+ continue;
+
+ rmid = i;
+ cqm_res_array[rmid].inuse = 1;
+ cqm_res_array[rmid].domain_id = domain_id;
+ break;
+ }
+ spin_unlock_irqrestore(&cqm_lock, flags);
+
+ /* No CQM RMID available, assign RMID=0 by default */
+ if ( rmid == -1 )
+ rmid = 0;
+
+out:
+ return rmid;
+}
+
+void free_cqm_resource(domid_t domain_id)
+{
+ int rmid;
+ unsigned long flags;
+
+ if ( !system_support_cqm() )
+ return;
+
+ spin_lock_irqsave(&cqm_lock, flags);
+ /* CQM RMID 0 is reserved, enumerate from 1 */
+ for ( rmid = 1; rmid < cqm_res_count; rmid++ )
+ {
+ if ( cqm_res_array[rmid].domain_id != domain_id )
+ continue;
+
+ cqm_res_array[rmid].inuse = 0;
+ cqm_res_array[rmid].domain_id = 0;
+ break;
+ }
+ spin_unlock_irqrestore(&cqm_lock, flags);
+}
+
/*
* Local variables:
* mode: C
diff --git a/xen/common/domctl.c b/xen/common/domctl.c
index 904d27b..1c2e320 100644
--- a/xen/common/domctl.c
+++ b/xen/common/domctl.c
@@ -425,7 +425,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
| XEN_DOMCTL_CDF_pvh_guest
| XEN_DOMCTL_CDF_hap
| XEN_DOMCTL_CDF_s3_integrity
- | XEN_DOMCTL_CDF_oos_off)) )
+ | XEN_DOMCTL_CDF_oos_off
+ | XEN_DOMCTL_CDF_pqos_cqm)) )
break;
dom = op->domain;
@@ -467,6 +468,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
domcr_flags |= DOMCRF_s3_integrity;
if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off )
domcr_flags |= DOMCRF_oos_off;
+ if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm )
+ domcr_flags |= DOMCRF_pqos_cqm;
d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref);
if ( IS_ERR(d) )
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index 9d39061..b0479aa 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -313,6 +313,8 @@ struct arch_domain
spinlock_t e820_lock;
struct e820entry *e820;
unsigned int nr_e820;
+
+ int pqos_cqm_rmid; /* CQM RMID assigned to the domain */
} __cacheline_aligned;
#define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list))
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
index 2184ea9..7e32fa5 100644
--- a/xen/include/asm-x86/pqos.h
+++ b/xen/include/asm-x86/pqos.h
@@ -34,4 +34,8 @@ struct cqm_res_struct {
void init_platform_qos(void);
+bool_t system_support_cqm(void);
+int alloc_cqm_resource(domid_t);
+void free_cqm_resource(domid_t);
+
#endif
diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index 01a3652..47a850a 100644
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -62,6 +62,9 @@ struct xen_domctl_createdomain {
/* Is this a PVH guest (as opposed to an HVM or PV guest)? */
#define _XEN_DOMCTL_CDF_pvh_guest 4
#define XEN_DOMCTL_CDF_pvh_guest (1U<<_XEN_DOMCTL_CDF_pvh_guest)
+ /* Enable pqos-cqm? */
+#define _XEN_DOMCTL_CDF_pqos_cqm 5
+#define XEN_DOMCTL_CDF_pqos_cqm (1U<<_XEN_DOMCTL_CDF_pqos_cqm)
uint32_t flags;
};
typedef struct xen_domctl_createdomain xen_domctl_createdomain_t;
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index cbdf377..3a42656 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -507,6 +507,9 @@ struct domain *domain_create(
/* DOMCRF_pvh: Create PV domain in HVM container. */
#define _DOMCRF_pvh 5
#define DOMCRF_pvh (1U<<_DOMCRF_pvh)
+ /* DOMCRF_pqos_cqm: Create a domain with CQM support */
+#define _DOMCRF_pqos_cqm 6
+#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm)
/*
* rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 2/8] x86: handle CQM resource when creating/destroying guests
2013-11-20 3:27 ` [PATCH 2/8] x86: handle CQM resource when creating/destroying guests dongxiao.xu
@ 2013-11-20 11:32 ` Andrew Cooper
2013-11-20 12:49 ` Xu, Dongxiao
0 siblings, 1 reply; 17+ messages in thread
From: Andrew Cooper @ 2013-11-20 11:32 UTC (permalink / raw)
To: dongxiao.xu; +Cc: xen-devel
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> From: Dongxiao Xu <dongxiao.xu@intel.com>
>
> Allocate an RMID for a guest when it is created. This per-guest
> RMID will be used to monitor Cache QoS related data. The RMID will
> be relinquished when guest is destroyed.
>
> Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
> xen/arch/x86/domain.c | 12 +++++++++
> xen/arch/x86/pqos.c | 58 ++++++++++++++++++++++++++++++++++++++++++
> xen/common/domctl.c | 5 +++-
> xen/include/asm-x86/domain.h | 2 ++
> xen/include/asm-x86/pqos.h | 4 +++
> xen/include/public/domctl.h | 3 +++
> xen/include/xen/sched.h | 3 +++
> 7 files changed, 86 insertions(+), 1 deletion(-)
>
> diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
> index a3868f9..5f920cd 100644
> --- a/xen/arch/x86/domain.c
> +++ b/xen/arch/x86/domain.c
> @@ -60,6 +60,7 @@
> #include <xen/numa.h>
> #include <xen/iommu.h>
> #include <compat/vcpu.h>
> +#include <asm/pqos.h>
>
> DEFINE_PER_CPU(struct vcpu *, curr_vcpu);
> DEFINE_PER_CPU(unsigned long, cr4);
> @@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags)
> tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
> spin_lock_init(&d->arch.vtsc_lock);
>
> + /* Allocate CQM RMID for guest */
> + if ( !!(domcr_flags & DOMCRF_pqos_cqm) )
> + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id);
> + else if ( system_support_cqm() )
> + d->arch.pqos_cqm_rmid = 0;
> + else
> + d->arch.pqos_cqm_rmid = -1;
> +
This logic looks overly-complicated, and calling alloc_cqm_resource()
before checking system_support_cqm() seems the wrong way around.
> return 0;
>
> fail:
> @@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d)
>
> free_xenheap_page(d->shared_info);
> cleanup_domain_irq_mapping(d);
> +
> + free_cqm_resource(d->domain_id);
> + d->arch.pqos_cqm_rmid = 0;
> }
>
> unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long guest_cr4)
> diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> index 617cce1..895d892 100644
> --- a/xen/arch/x86/pqos.c
> +++ b/xen/arch/x86/pqos.c
> @@ -20,6 +20,7 @@
> */
> #include <asm/processor.h>
> #include <xen/init.h>
> +#include <xen/spinlock.h>
> #include <asm/pqos.h>
>
> static bool_t pqos_enabled = 1;
> @@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0;
> unsigned int cqm_upscaling_factor = 0;
> bool_t cqm_enabled = 0;
> struct cqm_res_struct *cqm_res_array = NULL;
> +static DEFINE_SPINLOCK(cqm_lock);
>
> static void init_cqm(void)
> {
> @@ -78,6 +80,62 @@ void init_platform_qos(void)
> init_qos_monitor();
> }
>
> +bool_t system_support_cqm(void)
"system_supports_cqm"
> +{
> + return cqm_enabled;
> +}
> +
> +int alloc_cqm_resource(domid_t domain_id)
Perhaps alloc_rmid() given its current semantics?
> +{
> + int i, rmid = -1;
> + unsigned long flags;
> +
> + if ( !system_support_cqm() )
> + goto out;
Probably better as an ASSERT(). The caller should really not try to
allocate cqm resources if the system doesn't have cqm enabled.
> +
> + spin_lock_irqsave(&cqm_lock, flags);
> + for ( i = 0; i < cqm_res_count; i++ )
> + {
> + if ( cqm_res_array[i].inuse)
> + continue;
> +
> + rmid = i;
> + cqm_res_array[rmid].inuse = 1;
> + cqm_res_array[rmid].domain_id = domain_id;
> + break;
> + }
> + spin_unlock_irqrestore(&cqm_lock, flags);
> +
> + /* No CQM RMID available, assign RMID=0 by default */
> + if ( rmid == -1 )
> + rmid = 0;
> +
> +out:
> + return rmid;
> +}
> +
> +void free_cqm_resource(domid_t domain_id)
> +{
> + int rmid;
> + unsigned long flags;
> +
> + if ( !system_support_cqm() )
> + return;
> +
> + spin_lock_irqsave(&cqm_lock, flags);
> + /* CQM RMID 0 is reserved, enumerate from 1 */
> + for ( rmid = 1; rmid < cqm_res_count; rmid++ )
> + {
> + if ( cqm_res_array[rmid].domain_id != domain_id )
> + continue;
> +
> + cqm_res_array[rmid].inuse = 0;
> + cqm_res_array[rmid].domain_id = 0;
> + break;
> + }
> + spin_unlock_irqrestore(&cqm_lock, flags);
Can you not use d->arch.pqos_cqm_rmid here to save the search ?
> +}
> +
> /*
> * Local variables:
> * mode: C
> diff --git a/xen/common/domctl.c b/xen/common/domctl.c
> index 904d27b..1c2e320 100644
> --- a/xen/common/domctl.c
> +++ b/xen/common/domctl.c
> @@ -425,7 +425,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
> | XEN_DOMCTL_CDF_pvh_guest
> | XEN_DOMCTL_CDF_hap
> | XEN_DOMCTL_CDF_s3_integrity
> - | XEN_DOMCTL_CDF_oos_off)) )
> + | XEN_DOMCTL_CDF_oos_off
> + | XEN_DOMCTL_CDF_pqos_cqm)) )
> break;
>
> dom = op->domain;
> @@ -467,6 +468,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
> domcr_flags |= DOMCRF_s3_integrity;
> if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off )
> domcr_flags |= DOMCRF_oos_off;
> + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm )
> + domcr_flags |= DOMCRF_pqos_cqm;
>
> d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref);
> if ( IS_ERR(d) )
> diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
> index 9d39061..b0479aa 100644
> --- a/xen/include/asm-x86/domain.h
> +++ b/xen/include/asm-x86/domain.h
> @@ -313,6 +313,8 @@ struct arch_domain
> spinlock_t e820_lock;
> struct e820entry *e820;
> unsigned int nr_e820;
> +
> + int pqos_cqm_rmid; /* CQM RMID assigned to the domain */
> } __cacheline_aligned;
>
> #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list))
> diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> index 2184ea9..7e32fa5 100644
> --- a/xen/include/asm-x86/pqos.h
> +++ b/xen/include/asm-x86/pqos.h
> @@ -34,4 +34,8 @@ struct cqm_res_struct {
>
> void init_platform_qos(void);
>
> +bool_t system_support_cqm(void);
> +int alloc_cqm_resource(domid_t);
> +void free_cqm_resource(domid_t);
> +
> #endif
> diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> index 01a3652..47a850a 100644
> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -62,6 +62,9 @@ struct xen_domctl_createdomain {
> /* Is this a PVH guest (as opposed to an HVM or PV guest)? */
> #define _XEN_DOMCTL_CDF_pvh_guest 4
> #define XEN_DOMCTL_CDF_pvh_guest (1U<<_XEN_DOMCTL_CDF_pvh_guest)
> + /* Enable pqos-cqm? */
> +#define _XEN_DOMCTL_CDF_pqos_cqm 5
> +#define XEN_DOMCTL_CDF_pqos_cqm (1U<<_XEN_DOMCTL_CDF_pqos_cqm)
> uint32_t flags;
> };
> typedef struct xen_domctl_createdomain xen_domctl_createdomain_t;
> diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
> index cbdf377..3a42656 100644
> --- a/xen/include/xen/sched.h
> +++ b/xen/include/xen/sched.h
> @@ -507,6 +507,9 @@ struct domain *domain_create(
> /* DOMCRF_pvh: Create PV domain in HVM container. */
> #define _DOMCRF_pvh 5
> #define DOMCRF_pvh (1U<<_DOMCRF_pvh)
> + /* DOMCRF_pqos_cqm: Create a domain with CQM support */
> +#define _DOMCRF_pqos_cqm 6
> +#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm)
>
> /*
> * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 2/8] x86: handle CQM resource when creating/destroying guests
2013-11-20 11:32 ` Andrew Cooper
@ 2013-11-20 12:49 ` Xu, Dongxiao
0 siblings, 0 replies; 17+ messages in thread
From: Xu, Dongxiao @ 2013-11-20 12:49 UTC (permalink / raw)
To: Andrew Cooper; +Cc: xen-devel@lists.xen.org
> -----Original Message-----
> From: Andrew Cooper [mailto:andrew.cooper3@citrix.com]
> Sent: Wednesday, November 20, 2013 7:33 PM
> To: Xu, Dongxiao
> Cc: xen-devel@lists.xen.org
> Subject: Re: [Xen-devel] [PATCH 2/8] x86: handle CQM resource when
> creating/destroying guests
>
> On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> > From: Dongxiao Xu <dongxiao.xu@intel.com>
> >
> > Allocate an RMID for a guest when it is created. This per-guest
> > RMID will be used to monitor Cache QoS related data. The RMID will
> > be relinquished when guest is destroyed.
> >
> > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> > ---
> > xen/arch/x86/domain.c | 12 +++++++++
> > xen/arch/x86/pqos.c | 58
> ++++++++++++++++++++++++++++++++++++++++++
> > xen/common/domctl.c | 5 +++-
> > xen/include/asm-x86/domain.h | 2 ++
> > xen/include/asm-x86/pqos.h | 4 +++
> > xen/include/public/domctl.h | 3 +++
> > xen/include/xen/sched.h | 3 +++
> > 7 files changed, 86 insertions(+), 1 deletion(-)
> >
> > diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
> > index a3868f9..5f920cd 100644
> > --- a/xen/arch/x86/domain.c
> > +++ b/xen/arch/x86/domain.c
> > @@ -60,6 +60,7 @@
> > #include <xen/numa.h>
> > #include <xen/iommu.h>
> > #include <compat/vcpu.h>
> > +#include <asm/pqos.h>
> >
> > DEFINE_PER_CPU(struct vcpu *, curr_vcpu);
> > DEFINE_PER_CPU(unsigned long, cr4);
> > @@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int
> domcr_flags)
> > tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
> > spin_lock_init(&d->arch.vtsc_lock);
> >
> > + /* Allocate CQM RMID for guest */
> > + if ( !!(domcr_flags & DOMCRF_pqos_cqm) )
> > + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id);
> > + else if ( system_support_cqm() )
> > + d->arch.pqos_cqm_rmid = 0;
> > + else
> > + d->arch.pqos_cqm_rmid = -1;
> > +
>
> This logic looks overly-complicated, and calling alloc_cqm_resource()
> before checking system_support_cqm() seems the wrong way around.
OK.
>
> > return 0;
> >
> > fail:
> > @@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d)
> >
> > free_xenheap_page(d->shared_info);
> > cleanup_domain_irq_mapping(d);
> > +
> > + free_cqm_resource(d->domain_id);
> > + d->arch.pqos_cqm_rmid = 0;
> > }
> >
> > unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long
> guest_cr4)
> > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> > index 617cce1..895d892 100644
> > --- a/xen/arch/x86/pqos.c
> > +++ b/xen/arch/x86/pqos.c
> > @@ -20,6 +20,7 @@
> > */
> > #include <asm/processor.h>
> > #include <xen/init.h>
> > +#include <xen/spinlock.h>
> > #include <asm/pqos.h>
> >
> > static bool_t pqos_enabled = 1;
> > @@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0;
> > unsigned int cqm_upscaling_factor = 0;
> > bool_t cqm_enabled = 0;
> > struct cqm_res_struct *cqm_res_array = NULL;
> > +static DEFINE_SPINLOCK(cqm_lock);
> >
> > static void init_cqm(void)
> > {
> > @@ -78,6 +80,62 @@ void init_platform_qos(void)
> > init_qos_monitor();
> > }
> >
> > +bool_t system_support_cqm(void)
>
> "system_supports_cqm"
OK
>
> > +{
> > + return cqm_enabled;
> > +}
> > +
> > +int alloc_cqm_resource(domid_t domain_id)
>
> Perhaps alloc_rmid() given its current semantics?
>
> > +{
> > + int i, rmid = -1;
> > + unsigned long flags;
> > +
> > + if ( !system_support_cqm() )
> > + goto out;
>
> Probably better as an ASSERT(). The caller should really not try to
> allocate cqm resources if the system doesn't have cqm enabled.
OK.
>
> > +
> > + spin_lock_irqsave(&cqm_lock, flags);
> > + for ( i = 0; i < cqm_res_count; i++ )
> > + {
> > + if ( cqm_res_array[i].inuse)
> > + continue;
> > +
> > + rmid = i;
> > + cqm_res_array[rmid].inuse = 1;
> > + cqm_res_array[rmid].domain_id = domain_id;
> > + break;
> > + }
> > + spin_unlock_irqrestore(&cqm_lock, flags);
> > +
> > + /* No CQM RMID available, assign RMID=0 by default */
> > + if ( rmid == -1 )
> > + rmid = 0;
> > +
> > +out:
> > + return rmid;
> > +}
> > +
> > +void free_cqm_resource(domid_t domain_id)
> > +{
> > + int rmid;
> > + unsigned long flags;
> > +
> > + if ( !system_support_cqm() )
> > + return;
> > +
> > + spin_lock_irqsave(&cqm_lock, flags);
> > + /* CQM RMID 0 is reserved, enumerate from 1 */
> > + for ( rmid = 1; rmid < cqm_res_count; rmid++ )
> > + {
> > + if ( cqm_res_array[rmid].domain_id != domain_id )
> > + continue;
> > +
> > + cqm_res_array[rmid].inuse = 0;
> > + cqm_res_array[rmid].domain_id = 0;
> > + break;
> > + }
> > + spin_unlock_irqrestore(&cqm_lock, flags);
>
> Can you not use d->arch.pqos_cqm_rmid here to save the search ?
This needs to pass the "d" as a parameter, but seems a good idea.
Will reflect above suggestions in next version patch.
Thanks,
Dongxiao
>
> > +}
> > +
> > /*
> > * Local variables:
> > * mode: C
> > diff --git a/xen/common/domctl.c b/xen/common/domctl.c
> > index 904d27b..1c2e320 100644
> > --- a/xen/common/domctl.c
> > +++ b/xen/common/domctl.c
> > @@ -425,7 +425,8 @@ long
> do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
> > | XEN_DOMCTL_CDF_pvh_guest
> > | XEN_DOMCTL_CDF_hap
> > | XEN_DOMCTL_CDF_s3_integrity
> > - | XEN_DOMCTL_CDF_oos_off)) )
> > + | XEN_DOMCTL_CDF_oos_off
> > + | XEN_DOMCTL_CDF_pqos_cqm)) )
> > break;
> >
> > dom = op->domain;
> > @@ -467,6 +468,8 @@ long
> do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
> > domcr_flags |= DOMCRF_s3_integrity;
> > if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off )
> > domcr_flags |= DOMCRF_oos_off;
> > + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm )
> > + domcr_flags |= DOMCRF_pqos_cqm;
> >
> > d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref);
> > if ( IS_ERR(d) )
> > diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
> > index 9d39061..b0479aa 100644
> > --- a/xen/include/asm-x86/domain.h
> > +++ b/xen/include/asm-x86/domain.h
> > @@ -313,6 +313,8 @@ struct arch_domain
> > spinlock_t e820_lock;
> > struct e820entry *e820;
> > unsigned int nr_e820;
> > +
> > + int pqos_cqm_rmid; /* CQM RMID assigned to the domain */
> > } __cacheline_aligned;
> >
> > #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list))
> > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> > index 2184ea9..7e32fa5 100644
> > --- a/xen/include/asm-x86/pqos.h
> > +++ b/xen/include/asm-x86/pqos.h
> > @@ -34,4 +34,8 @@ struct cqm_res_struct {
> >
> > void init_platform_qos(void);
> >
> > +bool_t system_support_cqm(void);
> > +int alloc_cqm_resource(domid_t);
> > +void free_cqm_resource(domid_t);
> > +
> > #endif
> > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> > index 01a3652..47a850a 100644
> > --- a/xen/include/public/domctl.h
> > +++ b/xen/include/public/domctl.h
> > @@ -62,6 +62,9 @@ struct xen_domctl_createdomain {
> > /* Is this a PVH guest (as opposed to an HVM or PV guest)? */
> > #define _XEN_DOMCTL_CDF_pvh_guest 4
> > #define XEN_DOMCTL_CDF_pvh_guest
> (1U<<_XEN_DOMCTL_CDF_pvh_guest)
> > + /* Enable pqos-cqm? */
> > +#define _XEN_DOMCTL_CDF_pqos_cqm 5
> > +#define XEN_DOMCTL_CDF_pqos_cqm
> (1U<<_XEN_DOMCTL_CDF_pqos_cqm)
> > uint32_t flags;
> > };
> > typedef struct xen_domctl_createdomain xen_domctl_createdomain_t;
> > diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
> > index cbdf377..3a42656 100644
> > --- a/xen/include/xen/sched.h
> > +++ b/xen/include/xen/sched.h
> > @@ -507,6 +507,9 @@ struct domain *domain_create(
> > /* DOMCRF_pvh: Create PV domain in HVM container. */
> > #define _DOMCRF_pvh 5
> > #define DOMCRF_pvh (1U<<_DOMCRF_pvh)
> > + /* DOMCRF_pqos_cqm: Create a domain with CQM support */
> > +#define _DOMCRF_pqos_cqm 6
> > +#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm)
> >
> > /*
> > * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 3/8] tools: handle CQM resource when creating/destroying guests
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
2013-11-20 3:27 ` [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature dongxiao.xu
2013-11-20 3:27 ` [PATCH 2/8] x86: handle CQM resource when creating/destroying guests dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 3:27 ` [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest dongxiao.xu
` (4 subsequent siblings)
7 siblings, 0 replies; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
If a guest is created with parameter "pqos_cqm" set to 1, an RMID
will be allocated to the guest. This RMID will be relinquished when
guest is destroyed.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
tools/libxl/libxl_create.c | 3 +++
tools/libxl/libxl_types.idl | 1 +
tools/libxl/xl_cmdimpl.c | 2 ++
3 files changed, 6 insertions(+)
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 5e9cdcc..454c69d 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -40,6 +40,8 @@ int libxl__domain_create_info_setdefault(libxl__gc *gc,
libxl_defbool_setdefault(&c_info->run_hotplug_scripts, true);
+ libxl_defbool_setdefault(&c_info->pqos_cqm, false);
+
return 0;
}
@@ -454,6 +456,7 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_create_info *info,
}
flags |= XEN_DOMCTL_CDF_hap;
}
+ flags |= libxl_defbool_val(info->pqos_cqm) ? XEN_DOMCTL_CDF_pqos_cqm : 0;
*domid = -1;
/* Ultimately, handle is an array of 16 uint8_t, same as uuid */
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index de5bac3..22688d8 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -275,6 +275,7 @@ libxl_domain_create_info = Struct("domain_create_info",[
("poolid", uint32),
("run_hotplug_scripts",libxl_defbool),
("pvh", libxl_defbool),
+ ("pqos_cqm", libxl_defbool),
], dir=DIR_IN)
libxl_domain_restore_params = Struct("domain_restore_params", [
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 8690ec7..84a604f 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -670,6 +670,8 @@ static void parse_config_data(const char *config_source,
exit(1);
}
+ xlu_cfg_get_defbool(config, "pqos_cqm", &c_info->pqos_cqm, 0);
+
libxl_domain_build_info_init_type(b_info, c_info->type);
if (blkdev_start)
b_info->blkdev_start = strdup(blkdev_start);
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
` (2 preceding siblings ...)
2013-11-20 3:27 ` [PATCH 3/8] tools: " dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 11:44 ` Andrew Cooper
2013-11-20 3:27 ` [PATCH 5/8] tools: " dongxiao.xu
` (3 subsequent siblings)
7 siblings, 1 reply; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Add hypervisor side support for dynamically attach and detach CQM
services for a certain guest.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
xen/arch/x86/domctl.c | 66 +++++++++++++++++++++++++++++++++++++++++++
xen/include/public/domctl.h | 15 ++++++++++
2 files changed, 81 insertions(+)
diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index f7e4586..df6a373 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -35,6 +35,7 @@
#include <asm/mem_sharing.h>
#include <asm/xstate.h>
#include <asm/debugger.h>
+#include <asm/pqos.h>
static int gdbsx_guest_mem_io(
domid_t domid, struct xen_domctl_gdbsx_memio *iop)
@@ -1223,6 +1224,71 @@ long arch_do_domctl(
}
break;
+ case XEN_DOMCTL_attach_pqos:
+ {
+ struct domain *d;
+
+ if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
+ {
+ ret = -EBUSY;
+ break;
+ }
+
+ if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
+ {
+ if ( !system_support_cqm() )
+ ret = -ENODEV;
+ else if ( d->arch.pqos_cqm_rmid > 0 )
+ ret = -EINVAL;
+ else
+ {
+ ret = 0;
+ d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id);
+ if ( d->arch.pqos_cqm_rmid <= 0 )
+ {
+ /* set to default value */
+ d->arch.pqos_cqm_rmid = 0;
+ ret = -ENODEV;
+ }
+ }
+ }
+ else
+ ret = -EINVAL;
+
+ rcu_unlock_domain(d);
+ }
+ break;
+
+ case XEN_DOMCTL_detach_pqos:
+ {
+ struct domain *d;
+
+ if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
+ {
+ ret = -EBUSY;
+ break;
+ }
+
+ if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
+ {
+ if ( !system_support_cqm() )
+ ret = -ENODEV;
+ else if ( d->arch.pqos_cqm_rmid > 0 )
+ {
+ free_cqm_resource(d->domain_id);
+ d->arch.pqos_cqm_rmid = 0;
+ ret = 0;
+ }
+ else
+ ret = -EINVAL;
+ }
+ else
+ ret = -EINVAL;
+
+ rcu_unlock_domain(d);
+ }
+ break;
+
default:
ret = iommu_do_domctl(domctl, d, u_domctl);
break;
diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index 47a850a..4fe21db 100644
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn {
typedef struct xen_domctl_set_max_evtchn xen_domctl_set_max_evtchn_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t);
+/* XEN_DOMCTL_attach_pqos */
+/* XEN_DOMCTL_detach_pqos */
+struct xen_domctl_qos_resource {
+ /* Attach or detach flag for cqm */
+#define _XEN_DOMCTL_ADF_pqos_cqm 0
+#define XEN_DOMCTL_ADF_pqos_cqm (1U<<_XEN_DOMCTL_ADF_pqos_cqm)
+ uint32_t flags;
+};
+typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
+
struct xen_domctl {
uint32_t cmd;
#define XEN_DOMCTL_createdomain 1
@@ -941,6 +952,9 @@ struct xen_domctl {
#define XEN_DOMCTL_setnodeaffinity 68
#define XEN_DOMCTL_getnodeaffinity 69
#define XEN_DOMCTL_set_max_evtchn 70
+#define XEN_DOMCTL_attach_pqos 71
+#define XEN_DOMCTL_detach_pqos 72
+#define XEN_DOMCTL_list_pqos 73
#define XEN_DOMCTL_gdbsx_guestmemio 1000
#define XEN_DOMCTL_gdbsx_pausevcpu 1001
#define XEN_DOMCTL_gdbsx_unpausevcpu 1002
@@ -1001,6 +1015,7 @@ struct xen_domctl {
struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
struct xen_domctl_gdbsx_domstatus gdbsx_domstatus;
+ struct xen_domctl_qos_resource qos_res;
uint8_t pad[128];
} u;
};
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
2013-11-20 3:27 ` [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest dongxiao.xu
@ 2013-11-20 11:44 ` Andrew Cooper
2013-11-20 13:19 ` Xu, Dongxiao
0 siblings, 1 reply; 17+ messages in thread
From: Andrew Cooper @ 2013-11-20 11:44 UTC (permalink / raw)
To: dongxiao.xu; +Cc: xen-devel
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> From: Dongxiao Xu <dongxiao.xu@intel.com>
>
> Add hypervisor side support for dynamically attach and detach CQM
> services for a certain guest.
>
> Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
> xen/arch/x86/domctl.c | 66 +++++++++++++++++++++++++++++++++++++++++++
> xen/include/public/domctl.h | 15 ++++++++++
> 2 files changed, 81 insertions(+)
>
> diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
> index f7e4586..df6a373 100644
> --- a/xen/arch/x86/domctl.c
> +++ b/xen/arch/x86/domctl.c
> @@ -35,6 +35,7 @@
> #include <asm/mem_sharing.h>
> #include <asm/xstate.h>
> #include <asm/debugger.h>
> +#include <asm/pqos.h>
>
> static int gdbsx_guest_mem_io(
> domid_t domid, struct xen_domctl_gdbsx_memio *iop)
> @@ -1223,6 +1224,71 @@ long arch_do_domctl(
> }
> break;
>
> + case XEN_DOMCTL_attach_pqos:
> + {
> + struct domain *d;
> +
> + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
> + {
> + ret = -EBUSY;
> + break;
> + }
do_domctl has done this for us. There is also no need to shadow 'd'.
> +
> + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
> + {
> + if ( !system_support_cqm() )
> + ret = -ENODEV;
Surely this is generic, rather than condition on the caller specifying
XEN_DOMCTL_ADF_pqos_cqm
> + else if ( d->arch.pqos_cqm_rmid > 0 )
> + ret = -EINVAL;
> + else
> + {
> + ret = 0;
> + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id);
> + if ( d->arch.pqos_cqm_rmid <= 0 )
> + {
> + /* set to default value */
> + d->arch.pqos_cqm_rmid = 0;
> + ret = -ENODEV;
Perhaps E2BIG or EBUSY ? This error can be fixed by detaching cqm from
other domains.
> + }
> + }
> + }
> + else
> + ret = -EINVAL;
> +
> + rcu_unlock_domain(d);
> + }
> + break;
> +
> + case XEN_DOMCTL_detach_pqos:
> + {
> + struct domain *d;
> +
> + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
> + {
> + ret = -EBUSY;
> + break;
> + }
> +
> + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
> + {
> + if ( !system_support_cqm() )
> + ret = -ENODEV;
> + else if ( d->arch.pqos_cqm_rmid > 0 )
> + {
> + free_cqm_resource(d->domain_id);
> + d->arch.pqos_cqm_rmid = 0;
> + ret = 0;
> + }
> + else
> + ret = -EINVAL;
> + }
> + else
> + ret = -EINVAL;
> +
> + rcu_unlock_domain(d);
> + }
> + break;
> +
> default:
> ret = iommu_do_domctl(domctl, d, u_domctl);
> break;
> diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> index 47a850a..4fe21db 100644
> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn {
> typedef struct xen_domctl_set_max_evtchn xen_domctl_set_max_evtchn_t;
> DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t);
>
> +/* XEN_DOMCTL_attach_pqos */
> +/* XEN_DOMCTL_detach_pqos */
> +struct xen_domctl_qos_resource {
> + /* Attach or detach flag for cqm */
> +#define _XEN_DOMCTL_ADF_pqos_cqm 0
> +#define XEN_DOMCTL_ADF_pqos_cqm (1U<<_XEN_DOMCTL_ADF_pqos_cqm)
> + uint32_t flags;
The implementation semantics of this flag is redundant. You
differentiate attach/detach both on the domctl subop, and the flag in
this structure, leaving two invalid states.
It might be better to have a XEN_DOMCTL_qpos_op and a sub command, if
more of these hypercalls are on their way.
> +};
> +typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
> +
> struct xen_domctl {
> uint32_t cmd;
> #define XEN_DOMCTL_createdomain 1
> @@ -941,6 +952,9 @@ struct xen_domctl {
> #define XEN_DOMCTL_setnodeaffinity 68
> #define XEN_DOMCTL_getnodeaffinity 69
> #define XEN_DOMCTL_set_max_evtchn 70
> +#define XEN_DOMCTL_attach_pqos 71
> +#define XEN_DOMCTL_detach_pqos 72
> +#define XEN_DOMCTL_list_pqos 73
You introduce this list_pqos domctl with no implementation.
> #define XEN_DOMCTL_gdbsx_guestmemio 1000
> #define XEN_DOMCTL_gdbsx_pausevcpu 1001
> #define XEN_DOMCTL_gdbsx_unpausevcpu 1002
> @@ -1001,6 +1015,7 @@ struct xen_domctl {
> struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
> struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
> struct xen_domctl_gdbsx_domstatus gdbsx_domstatus;
> + struct xen_domctl_qos_resource qos_res;
> uint8_t pad[128];
> } u;
> };
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
2013-11-20 11:44 ` Andrew Cooper
@ 2013-11-20 13:19 ` Xu, Dongxiao
0 siblings, 0 replies; 17+ messages in thread
From: Xu, Dongxiao @ 2013-11-20 13:19 UTC (permalink / raw)
To: Andrew Cooper; +Cc: xen-devel@lists.xen.org
> -----Original Message-----
> From: Andrew Cooper [mailto:andrew.cooper3@citrix.com]
> Sent: Wednesday, November 20, 2013 7:45 PM
> To: Xu, Dongxiao
> Cc: xen-devel@lists.xen.org
> Subject: Re: [Xen-devel] [PATCH 4/8] x86: dynamically attach/detach CQM service
> for a guest
>
> On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> > From: Dongxiao Xu <dongxiao.xu@intel.com>
> >
> > Add hypervisor side support for dynamically attach and detach CQM
> > services for a certain guest.
> >
> > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> > ---
> > xen/arch/x86/domctl.c | 66
> +++++++++++++++++++++++++++++++++++++++++++
> > xen/include/public/domctl.h | 15 ++++++++++
> > 2 files changed, 81 insertions(+)
> >
> > diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
> > index f7e4586..df6a373 100644
> > --- a/xen/arch/x86/domctl.c
> > +++ b/xen/arch/x86/domctl.c
> > @@ -35,6 +35,7 @@
> > #include <asm/mem_sharing.h>
> > #include <asm/xstate.h>
> > #include <asm/debugger.h>
> > +#include <asm/pqos.h>
> >
> > static int gdbsx_guest_mem_io(
> > domid_t domid, struct xen_domctl_gdbsx_memio *iop)
> > @@ -1223,6 +1224,71 @@ long arch_do_domctl(
> > }
> > break;
> >
> > + case XEN_DOMCTL_attach_pqos:
> > + {
> > + struct domain *d;
> > +
> > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
> > + {
> > + ret = -EBUSY;
> > + break;
> > + }
>
> do_domctl has done this for us. There is also no need to shadow 'd'.
OK, will remove the logic.
>
> > +
> > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
> > + {
> > + if ( !system_support_cqm() )
> > + ret = -ENODEV;
>
> Surely this is generic, rather than condition on the caller specifying
> XEN_DOMCTL_ADF_pqos_cqm
CQM is just one type of monitoring resource, there may be other types later.
>
> > + else if ( d->arch.pqos_cqm_rmid > 0 )
> > + ret = -EINVAL;
> > + else
> > + {
> > + ret = 0;
> > + d->arch.pqos_cqm_rmid =
> alloc_cqm_resource(d->domain_id);
> > + if ( d->arch.pqos_cqm_rmid <= 0 )
> > + {
> > + /* set to default value */
> > + d->arch.pqos_cqm_rmid = 0;
> > + ret = -ENODEV;
>
> Perhaps E2BIG or EBUSY ? This error can be fixed by detaching cqm from
> other domains.
OK, maybe EBUSY is a better choice.
>
> > + }
> > + }
> > + }
> > + else
> > + ret = -EINVAL;
> > +
> > + rcu_unlock_domain(d);
> > + }
> > + break;
> > +
> > + case XEN_DOMCTL_detach_pqos:
> > + {
> > + struct domain *d;
> > +
> > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
> > + {
> > + ret = -EBUSY;
> > + break;
> > + }
> > +
> > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm )
> > + {
> > + if ( !system_support_cqm() )
> > + ret = -ENODEV;
> > + else if ( d->arch.pqos_cqm_rmid > 0 )
> > + {
> > + free_cqm_resource(d->domain_id);
> > + d->arch.pqos_cqm_rmid = 0;
> > + ret = 0;
> > + }
> > + else
> > + ret = -EINVAL;
> > + }
> > + else
> > + ret = -EINVAL;
> > +
> > + rcu_unlock_domain(d);
> > + }
> > + break;
> > +
> > default:
> > ret = iommu_do_domctl(domctl, d, u_domctl);
> > break;
> > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> > index 47a850a..4fe21db 100644
> > --- a/xen/include/public/domctl.h
> > +++ b/xen/include/public/domctl.h
> > @@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn {
> > typedef struct xen_domctl_set_max_evtchn
> xen_domctl_set_max_evtchn_t;
> > DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t);
> >
> > +/* XEN_DOMCTL_attach_pqos */
> > +/* XEN_DOMCTL_detach_pqos */
> > +struct xen_domctl_qos_resource {
> > + /* Attach or detach flag for cqm */
> > +#define _XEN_DOMCTL_ADF_pqos_cqm 0
> > +#define XEN_DOMCTL_ADF_pqos_cqm
> (1U<<_XEN_DOMCTL_ADF_pqos_cqm)
> > + uint32_t flags;
>
> The implementation semantics of this flag is redundant. You
> differentiate attach/detach both on the domctl subop, and the flag in
> this structure, leaving two invalid states.
>
> It might be better to have a XEN_DOMCTL_qpos_op and a sub command, if
> more of these hypercalls are on their way.
Flag is to identify the type of QoS monitoring resource, and CQM is one example.
>
> > +};
> > +typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
> > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
> > +
> > struct xen_domctl {
> > uint32_t cmd;
> > #define XEN_DOMCTL_createdomain 1
> > @@ -941,6 +952,9 @@ struct xen_domctl {
> > #define XEN_DOMCTL_setnodeaffinity 68
> > #define XEN_DOMCTL_getnodeaffinity 69
> > #define XEN_DOMCTL_set_max_evtchn 70
> > +#define XEN_DOMCTL_attach_pqos 71
> > +#define XEN_DOMCTL_detach_pqos 72
> > +#define XEN_DOMCTL_list_pqos 73
>
> You introduce this list_pqos domctl with no implementation.
Yes, it should be in the patch 6/8.
Thanks,
Dongxiao
>
> > #define XEN_DOMCTL_gdbsx_guestmemio 1000
> > #define XEN_DOMCTL_gdbsx_pausevcpu 1001
> > #define XEN_DOMCTL_gdbsx_unpausevcpu 1002
> > @@ -1001,6 +1015,7 @@ struct xen_domctl {
> > struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
> > struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
> > struct xen_domctl_gdbsx_domstatus gdbsx_domstatus;
> > + struct xen_domctl_qos_resource qos_res;
> > uint8_t pad[128];
> > } u;
> > };
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 5/8] tools: dynamically attach/detach CQM service for a guest
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
` (3 preceding siblings ...)
2013-11-20 3:27 ` [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 3:27 ` [PATCH 6/8] x86: get per domain CQM information dongxiao.xu
` (2 subsequent siblings)
7 siblings, 0 replies; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Add two new xl instruction for attaching and detaching CQM services
for a running guest.
Above "qos_res" is the monitoring resource type, and current supported
type is "cqm".
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
tools/libxc/xc_domain.c | 18 ++++++++++
tools/libxc/xenctrl.h | 2 ++
tools/libxl/Makefile | 3 +-
tools/libxl/libxl.h | 3 ++
tools/libxl/libxl_pqos.c | 86 +++++++++++++++++++++++++++++++++++++++++++++
tools/libxl/xl.h | 2 ++
tools/libxl/xl_cmdimpl.c | 36 +++++++++++++++++++
tools/libxl/xl_cmdtable.c | 10 ++++++
8 files changed, 159 insertions(+), 1 deletion(-)
create mode 100644 tools/libxl/libxl_pqos.c
diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index 1ccafc5..85c2d4d 100644
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -1776,6 +1776,24 @@ int xc_domain_set_max_evtchn(xc_interface *xch, uint32_t domid,
return do_domctl(xch, &domctl);
}
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags)
+{
+ DECLARE_DOMCTL;
+ domctl.cmd = XEN_DOMCTL_attach_pqos;
+ domctl.domain = (domid_t)domid;
+ domctl.u.qos_res.flags = flags;
+ return do_domctl(xch, &domctl);
+}
+
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags)
+{
+ DECLARE_DOMCTL;
+ domctl.cmd = XEN_DOMCTL_detach_pqos;
+ domctl.domain = (domid_t)domid;
+ domctl.u.qos_res.flags = flags;
+ return do_domctl(xch, &domctl);
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index 4ac6b8a..a57f147 100644
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -2395,4 +2395,6 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch,
*/
int xc_kexec_unload(xc_interface *xch, int type);
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags);
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags);
#endif /* XENCTRL_H */
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index cf214bb..35f0b97 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -74,7 +74,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
libxl_internal.o libxl_utils.o libxl_uuid.o \
libxl_json.o libxl_aoutils.o libxl_numa.o \
libxl_save_callout.o _libxl_save_msgs_callout.o \
- libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
+ libxl_qmp.o libxl_event.o libxl_fork.o libxl_pqos.o \
+ $(LIBXL_OBJS-y)
LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
$(LIBXL_OBJS): CFLAGS += $(CFLAGS_LIBXL) -include $(XEN_ROOT)/tools/config.h
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index c7dceda..a9a506f 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1051,6 +1051,9 @@ int libxl_flask_getenforce(libxl_ctx *ctx);
int libxl_flask_setenforce(libxl_ctx *ctx, int mode);
int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size);
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres);
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres);
+
/* misc */
/* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c
new file mode 100644
index 0000000..6d8013a
--- /dev/null
+++ b/tools/libxl/libxl_pqos.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2013 Intel Corporation
+ * Author Jiongxi Li <jiongxi.li@intel.com>
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+#include "libxl_internal.h"
+
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres)
+{
+ int rc;
+ int flags = 0;
+
+ if (!strcmp(qosres, "cqm"))
+ flags |= XEN_DOMCTL_ADF_pqos_cqm;
+ else {
+ rc = -EINVAL;
+ fprintf(stderr, "Invalid QoS resource type! "
+ "Supported types are: \"cqm\".\n");
+ return rc;
+ }
+
+ rc = xc_domain_pqos_attach(ctx->xch, domid, flags);
+ if (rc < 0) {
+ fprintf(stderr, "Failed to attach CQM service to domain %d. ", domid);
+ if (errno == ENODEV)
+ fprintf(stderr, "CQM not supported, or no free CQM RMID available.\n");
+ else if (errno == EINVAL)
+ fprintf(stderr, "Domain %d is already attached with CQM service.\n", domid);
+ else if (errno == ESRCH)
+ fprintf(stderr, "Is Domain %d a valid domain?\n", domid);
+ else
+ fprintf(stderr, "errno: %d\n", errno);
+ }
+
+ return rc;
+}
+
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres)
+{
+ int rc;
+ int flags = 0;
+
+ if (!strcmp(qosres, "cqm"))
+ flags |= XEN_DOMCTL_ADF_pqos_cqm;
+ else {
+ rc = -EINVAL;
+ fprintf(stderr, "Invalid QoS resource type! "
+ "Supported types are: \"cqm\".\n");
+ return rc;
+ }
+
+ rc = xc_domain_pqos_detach(ctx->xch, domid, flags);
+ if (rc < 0) {
+ fprintf(stderr, "Failed to detach CQM service to domain %d. ", domid);
+ if (errno == ENODEV)
+ fprintf(stderr, "CQM not supported.\n");
+ else if (errno == EINVAL)
+ fprintf(stderr, "Domain %d is not attached with CQM service.\n", domid);
+ else if (errno == ESRCH)
+ fprintf(stderr, "Is Domain %d a valid domain?\n", domid);
+ else
+ fprintf(stderr, "errno: %d\n", errno);
+ }
+
+ return rc;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index e005c39..78738b9 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -105,6 +105,8 @@ int main_getenforce(int argc, char **argv);
int main_setenforce(int argc, char **argv);
int main_loadpolicy(int argc, char **argv);
int main_remus(int argc, char **argv);
+int main_pqosattach(int argc, char **argv);
+int main_pqosdetach(int argc, char **argv);
void help(const char *command);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 84a604f..1a12e8f 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -7195,6 +7195,42 @@ int main_remus(int argc, char **argv)
return -ERROR_FAIL;
}
+int main_pqosattach(int argc, char **argv)
+{
+ uint32_t domid;
+ int opt, rc;
+ const char *qosres = NULL;
+
+ SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-attach", 2) {
+ /* No options */
+ }
+
+ qosres = argv[optind];
+ domid = find_domain(argv[optind + 1]);
+
+ rc = libxl_pqos_attach(ctx, domid, qosres);
+
+ return rc;
+}
+
+int main_pqosdetach(int argc, char **argv)
+{
+ uint32_t domid;
+ int opt, rc;
+ const char *qosres = NULL;
+
+ SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-detach", 2) {
+ /* No options */
+ }
+
+ qosres = argv[optind];
+ domid = find_domain(argv[optind + 1]);
+
+ rc = libxl_pqos_detach(ctx, domid, qosres);
+
+ return rc;
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 326a660..02a2572 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -488,6 +488,16 @@ struct cmd_spec cmd_table[] = {
" of the domain."
},
+ { "pqos-attach",
+ &main_pqosattach, 0, 1,
+ "Allocate and map qos resource",
+ "<Resource> <Domain>",
+ },
+ { "pqos-detach",
+ &main_pqosdetach, 0, 1,
+ "Reliquish qos resource",
+ "<Resource> <Domain>",
+ },
};
int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 6/8] x86: get per domain CQM information
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
` (4 preceding siblings ...)
2013-11-20 3:27 ` [PATCH 5/8] tools: " dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 11:56 ` Andrew Cooper
2013-11-20 3:27 ` [PATCH 7/8] tools: " dongxiao.xu
2013-11-20 3:27 ` [PATCH 8/8] x86: enable CQM monitoring for each domain RMID dongxiao.xu
7 siblings, 1 reply; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Retrive CQM information for certain domain, which reflects the L3 cache
occupancy for a socket.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
xen/arch/x86/pqos.c | 60 ++++++++++++++++++++++++++++++++++++
xen/arch/x86/sysctl.c | 64 +++++++++++++++++++++++++++++++++++++++
xen/include/asm-x86/msr-index.h | 4 +++
xen/include/asm-x86/pqos.h | 14 +++++++++
xen/include/public/domctl.h | 14 +++++++++
xen/include/public/sysctl.h | 23 ++++++++++++++
6 files changed, 179 insertions(+)
diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
index 895d892..3699efe 100644
--- a/xen/arch/x86/pqos.c
+++ b/xen/arch/x86/pqos.c
@@ -19,13 +19,30 @@
* Place - Suite 330, Boston, MA 02111-1307 USA.
*/
#include <asm/processor.h>
+#include <asm/msr.h>
+#include <xen/cpumask.h>
#include <xen/init.h>
#include <xen/spinlock.h>
+#include <public/domctl.h>
#include <asm/pqos.h>
static bool_t pqos_enabled = 1;
boolean_param("pqos", pqos_enabled);
+static void read_qm_data(void *arg)
+{
+ struct qm_element *qm_element = arg;
+
+ wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid, qm_element->rmid);
+ rdmsrl(MSR_IA32_QMC, qm_element->qm_data);
+}
+
+static void get_generic_qm_info(struct qm_element *qm_element)
+{
+ int cpu = qm_element->cpu;
+ on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1);
+}
+
unsigned int cqm_res_count = 0;
unsigned int cqm_upscaling_factor = 0;
bool_t cqm_enabled = 0;
@@ -85,6 +102,25 @@ bool_t system_support_cqm(void)
return cqm_enabled;
}
+unsigned int get_cqm_count(void)
+{
+ return cqm_res_count;
+}
+
+unsigned int get_cqm_avail(void)
+{
+ unsigned int cqm_avail = 0;
+ int i;
+
+ for (i = 0; i < cqm_res_count; i++)
+ {
+ if ( !cqm_res_array[i].inuse )
+ cqm_avail++;
+ }
+
+ return cqm_avail;
+}
+
int alloc_cqm_resource(domid_t domain_id)
{
int i, rmid = -1;
@@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id)
spin_unlock_irqrestore(&cqm_lock, flags);
}
+void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
+ struct xen_domctl_getdomcqminfo *info)
+{
+ struct qm_element element;
+ int cpu, i;
+
+ for_each_cpu ( cpu, &cpu_cqmdata_map )
+ {
+ element.cpu = cpu;
+ element.rmid = rmid;
+ element.evtid = QOS_MONITOR_EVTID_L3;
+
+ get_generic_qm_info(&element);
+
+ i = cpu_to_socket(cpu);
+ info->socket_cqmdata[i].valid =
+ (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1;
+ if ( info->socket_cqmdata[i].valid )
+ info->socket_cqmdata[i].l3c_occupancy = element.qm_data * cqm_upscaling_factor;
+ else
+ info->socket_cqmdata[i].l3c_occupancy = 0;
+ }
+}
+
/*
* Local variables:
* mode: C
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index 15d4b91..a779fdc 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -28,6 +28,7 @@
#include <xen/nodemask.h>
#include <xen/cpu.h>
#include <xsm/xsm.h>
+#include <asm/pqos.h>
#define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0)
@@ -101,6 +102,69 @@ long arch_do_sysctl(
}
break;
+ case XEN_SYSCTL_getdomcqminfolist:
+ {
+ struct domain *d;
+ struct xen_domctl_getdomcqminfo info;
+ uint32_t resource_count;
+ uint32_t resource_avail;
+ uint32_t num_domains = 0;
+ cpumask_t cpu_cqmdata_map;
+ DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS);
+ int cpu;
+
+ if ( !system_support_cqm() )
+ {
+ ret = -EFAULT;
+ break;
+ }
+
+ resource_count = get_cqm_count();
+ resource_avail = get_cqm_avail();
+
+ cpumask_clear(&cpu_cqmdata_map);
+ bitmap_zero(sockets, QOS_MAX_SOCKETS);
+ for_each_online_cpu(cpu)
+ {
+ int i = cpu_to_socket(cpu);
+ if ( test_and_set_bit(i, sockets) )
+ continue;
+ cpumask_set_cpu(cpu, &cpu_cqmdata_map);
+ }
+
+ rcu_read_lock(&domlist_read_lock);
+ for_each_domain ( d )
+ {
+ if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain )
+ continue;
+ if ( num_domains == sysctl->u.getdomaininfolist.max_domains )
+ break;
+ if ( d->arch.pqos_cqm_rmid <= 0 )
+ continue;
+ memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo));
+ info.domain = d->domain_id;
+ get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map, &info);
+
+ if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer,
+ num_domains, &info, 1) )
+ {
+ ret = -EFAULT;
+ break;
+ }
+
+ num_domains++;
+ }
+ rcu_read_unlock(&domlist_read_lock);
+
+ sysctl->u.getdomcqminfolist.num_domains = num_domains;
+ sysctl->u.getdomcqminfolist.resource_count = resource_count;
+ sysctl->u.getdomcqminfolist.resource_avail = resource_avail;
+
+ if ( copy_to_guest(u_sysctl, sysctl, 1) )
+ ret = -EFAULT;
+ }
+ break;
+
default:
ret = -ENOSYS;
break;
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index e597a28..46ef165 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -488,4 +488,8 @@
/* Geode defined MSRs */
#define MSR_GEODE_BUSCONT_CONF0 0x00001900
+/* Platform QoS register */
+#define MSR_IA32_QOSEVTSEL 0x00000c8d
+#define MSR_IA32_QMC 0x00000c8e
+
#endif /* __ASM_MSR_INDEX_H */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
index 7e32fa5..6d1b1e8 100644
--- a/xen/include/asm-x86/pqos.h
+++ b/xen/include/asm-x86/pqos.h
@@ -27,15 +27,29 @@
/* QoS Monitoring Event ID */
#define QOS_MONITOR_EVTID_L3 0x1
+/* IA32_QM_CTR */
+#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62)
+
struct cqm_res_struct {
bool_t inuse;
uint16_t domain_id;
};
+struct qm_element {
+ uint16_t cpu;
+ uint32_t rmid;
+ uint8_t evtid;
+ uint64_t qm_data;
+};
+
void init_platform_qos(void);
bool_t system_support_cqm(void);
int alloc_cqm_resource(domid_t);
void free_cqm_resource(domid_t);
+unsigned int get_cqm_count(void);
+unsigned int get_cqm_avail(void);
+void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
+ struct xen_domctl_getdomcqminfo *info);
#endif
diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index 4fe21db..bdefc83 100644
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -883,6 +883,20 @@ struct xen_domctl_qos_resource {
typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
+struct xen_socket_cqmdata {
+ uint8_t valid;
+ uint64_t l3c_occupancy;
+};
+
+struct xen_domctl_getdomcqminfo {
+ /* OUT variables. */
+ domid_t domain;
+#define QOS_MAX_SOCKETS 128
+ struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS];
+};
+typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t);
+
struct xen_domctl {
uint32_t cmd;
#define XEN_DOMCTL_createdomain 1
diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
index 8437d31..91f206e 100644
--- a/xen/include/public/sysctl.h
+++ b/xen/include/public/sysctl.h
@@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op {
typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t);
+struct xen_sysctl_getcqminfo
+{
+ uint32_t resource_count;
+ uint32_t resource_avail;
+ struct xen_domctl_getdomcqminfo *dom_cqminfo;
+};
+typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t;
+
/* XEN_SYSCTL_getdomaininfolist */
struct xen_sysctl_getdomaininfolist {
/* IN variables. */
@@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op {
typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t);
+/* XEN_SYSCTL_getdomcqminfolist */
+struct xen_sysctl_getdomcqminfolist {
+ /* IN variables. */
+ domid_t first_domain;
+ uint32_t max_domains;
+ XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer;
+ /* OUT variables. */
+ uint32_t num_domains;
+ uint32_t resource_count;
+ uint32_t resource_avail;
+};
+typedef struct xen_sysctl_getdomcqminfolist xen_sysctl_getdomcqminfolist_t;
+DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t);
struct xen_sysctl {
uint32_t cmd;
@@ -654,6 +675,7 @@ struct xen_sysctl {
#define XEN_SYSCTL_cpupool_op 18
#define XEN_SYSCTL_scheduler_op 19
#define XEN_SYSCTL_coverage_op 20
+#define XEN_SYSCTL_getdomcqminfolist 21
uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
union {
struct xen_sysctl_readconsole readconsole;
@@ -675,6 +697,7 @@ struct xen_sysctl {
struct xen_sysctl_cpupool_op cpupool_op;
struct xen_sysctl_scheduler_op scheduler_op;
struct xen_sysctl_coverage_op coverage_op;
+ struct xen_sysctl_getdomcqminfolist getdomcqminfolist;
uint8_t pad[128];
} u;
};
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 6/8] x86: get per domain CQM information
2013-11-20 3:27 ` [PATCH 6/8] x86: get per domain CQM information dongxiao.xu
@ 2013-11-20 11:56 ` Andrew Cooper
2013-11-20 13:22 ` Xu, Dongxiao
0 siblings, 1 reply; 17+ messages in thread
From: Andrew Cooper @ 2013-11-20 11:56 UTC (permalink / raw)
To: dongxiao.xu; +Cc: xen-devel
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> From: Dongxiao Xu <dongxiao.xu@intel.com>
>
> Retrive CQM information for certain domain, which reflects the L3 cache
> occupancy for a socket.
>
> Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
> xen/arch/x86/pqos.c | 60 ++++++++++++++++++++++++++++++++++++
> xen/arch/x86/sysctl.c | 64 +++++++++++++++++++++++++++++++++++++++
> xen/include/asm-x86/msr-index.h | 4 +++
> xen/include/asm-x86/pqos.h | 14 +++++++++
> xen/include/public/domctl.h | 14 +++++++++
> xen/include/public/sysctl.h | 23 ++++++++++++++
> 6 files changed, 179 insertions(+)
>
> diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> index 895d892..3699efe 100644
> --- a/xen/arch/x86/pqos.c
> +++ b/xen/arch/x86/pqos.c
> @@ -19,13 +19,30 @@
> * Place - Suite 330, Boston, MA 02111-1307 USA.
> */
> #include <asm/processor.h>
> +#include <asm/msr.h>
> +#include <xen/cpumask.h>
> #include <xen/init.h>
> #include <xen/spinlock.h>
> +#include <public/domctl.h>
> #include <asm/pqos.h>
>
> static bool_t pqos_enabled = 1;
> boolean_param("pqos", pqos_enabled);
>
> +static void read_qm_data(void *arg)
> +{
> + struct qm_element *qm_element = arg;
> +
> + wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid, qm_element->rmid);
> + rdmsrl(MSR_IA32_QMC, qm_element->qm_data);
> +}
> +
> +static void get_generic_qm_info(struct qm_element *qm_element)
> +{
> + int cpu = qm_element->cpu;
> + on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1);
> +}
> +
> unsigned int cqm_res_count = 0;
> unsigned int cqm_upscaling_factor = 0;
> bool_t cqm_enabled = 0;
> @@ -85,6 +102,25 @@ bool_t system_support_cqm(void)
> return cqm_enabled;
> }
>
> +unsigned int get_cqm_count(void)
> +{
> + return cqm_res_count;
> +}
> +
> +unsigned int get_cqm_avail(void)
> +{
> + unsigned int cqm_avail = 0;
> + int i;
> +
> + for (i = 0; i < cqm_res_count; i++)
> + {
> + if ( !cqm_res_array[i].inuse )
> + cqm_avail++;
> + }
Style - extra spaces inside brackets for the for loop, and these braces
can go.
> +
> + return cqm_avail;
> +}
> +
> int alloc_cqm_resource(domid_t domain_id)
> {
> int i, rmid = -1;
> @@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id)
> spin_unlock_irqrestore(&cqm_lock, flags);
> }
>
> +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
> + struct xen_domctl_getdomcqminfo *info)
> +{
> + struct qm_element element;
> + int cpu, i;
> +
> + for_each_cpu ( cpu, &cpu_cqmdata_map )
> + {
> + element.cpu = cpu;
> + element.rmid = rmid;
> + element.evtid = QOS_MONITOR_EVTID_L3;
> +
> + get_generic_qm_info(&element);
> +
> + i = cpu_to_socket(cpu);
> + info->socket_cqmdata[i].valid =
> + (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1;
> + if ( info->socket_cqmdata[i].valid )
> + info->socket_cqmdata[i].l3c_occupancy = element.qm_data * cqm_upscaling_factor;
> + else
> + info->socket_cqmdata[i].l3c_occupancy = 0;
> + }
> +}
> +
> /*
> * Local variables:
> * mode: C
> diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
> index 15d4b91..a779fdc 100644
> --- a/xen/arch/x86/sysctl.c
> +++ b/xen/arch/x86/sysctl.c
> @@ -28,6 +28,7 @@
> #include <xen/nodemask.h>
> #include <xen/cpu.h>
> #include <xsm/xsm.h>
> +#include <asm/pqos.h>
>
> #define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0)
>
> @@ -101,6 +102,69 @@ long arch_do_sysctl(
> }
> break;
>
> + case XEN_SYSCTL_getdomcqminfolist:
> + {
> + struct domain *d;
> + struct xen_domctl_getdomcqminfo info;
> + uint32_t resource_count;
> + uint32_t resource_avail;
> + uint32_t num_domains = 0;
> + cpumask_t cpu_cqmdata_map;
> + DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS);
> + int cpu;
unsigned int.
> +
> + if ( !system_support_cqm() )
> + {
> + ret = -EFAULT;
ENODEV surely ?
> + break;
> + }
> +
> + resource_count = get_cqm_count();
> + resource_avail = get_cqm_avail();
> +
> + cpumask_clear(&cpu_cqmdata_map);
> + bitmap_zero(sockets, QOS_MAX_SOCKETS);
> + for_each_online_cpu(cpu)
> + {
> + int i = cpu_to_socket(cpu);
> + if ( test_and_set_bit(i, sockets) )
> + continue;
> + cpumask_set_cpu(cpu, &cpu_cqmdata_map);
> + }
> +
> + rcu_read_lock(&domlist_read_lock);
> + for_each_domain ( d )
> + {
> + if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain )
> + continue;
> + if ( num_domains == sysctl->u.getdomaininfolist.max_domains )
> + break;
> + if ( d->arch.pqos_cqm_rmid <= 0 )
> + continue;
> + memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo));
> + info.domain = d->domain_id;
> + get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map, &info);
> +
> + if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer,
> + num_domains, &info, 1) )
> + {
> + ret = -EFAULT;
> + break;
> + }
> +
> + num_domains++;
> + }
> + rcu_read_unlock(&domlist_read_lock);
> +
> + sysctl->u.getdomcqminfolist.num_domains = num_domains;
> + sysctl->u.getdomcqminfolist.resource_count = resource_count;
> + sysctl->u.getdomcqminfolist.resource_avail = resource_avail;
> +
> + if ( copy_to_guest(u_sysctl, sysctl, 1) )
> + ret = -EFAULT;
> + }
> + break;
> +
> default:
> ret = -ENOSYS;
> break;
> diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
> index e597a28..46ef165 100644
> --- a/xen/include/asm-x86/msr-index.h
> +++ b/xen/include/asm-x86/msr-index.h
> @@ -488,4 +488,8 @@
> /* Geode defined MSRs */
> #define MSR_GEODE_BUSCONT_CONF0 0x00001900
>
> +/* Platform QoS register */
> +#define MSR_IA32_QOSEVTSEL 0x00000c8d
> +#define MSR_IA32_QMC 0x00000c8e
> +
> #endif /* __ASM_MSR_INDEX_H */
> diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> index 7e32fa5..6d1b1e8 100644
> --- a/xen/include/asm-x86/pqos.h
> +++ b/xen/include/asm-x86/pqos.h
> @@ -27,15 +27,29 @@
> /* QoS Monitoring Event ID */
> #define QOS_MONITOR_EVTID_L3 0x1
>
> +/* IA32_QM_CTR */
> +#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62)
> +
> struct cqm_res_struct {
> bool_t inuse;
> uint16_t domain_id;
> };
>
> +struct qm_element {
> + uint16_t cpu;
> + uint32_t rmid;
> + uint8_t evtid;
> + uint64_t qm_data;
> +};
> +
The packing of this structure is rather poor. Please re-order to reduce
some of the holes.
Also, cpu parameters are unsigned int in Xen.
> void init_platform_qos(void);
>
> bool_t system_support_cqm(void);
> int alloc_cqm_resource(domid_t);
> void free_cqm_resource(domid_t);
> +unsigned int get_cqm_count(void);
> +unsigned int get_cqm_avail(void);
> +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
> + struct xen_domctl_getdomcqminfo *info);
>
> #endif
> diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> index 4fe21db..bdefc83 100644
> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -883,6 +883,20 @@ struct xen_domctl_qos_resource {
> typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
> DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
>
> +struct xen_socket_cqmdata {
> + uint8_t valid;
> + uint64_t l3c_occupancy;
> +};
> +
> +struct xen_domctl_getdomcqminfo {
> + /* OUT variables. */
> + domid_t domain;
> +#define QOS_MAX_SOCKETS 128
> + struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS];
> +};
> +typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t);
> +
> struct xen_domctl {
> uint32_t cmd;
> #define XEN_DOMCTL_createdomain 1
> diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
> index 8437d31..91f206e 100644
> --- a/xen/include/public/sysctl.h
> +++ b/xen/include/public/sysctl.h
> @@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op {
> typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t;
> DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t);
>
> +struct xen_sysctl_getcqminfo
> +{
> + uint32_t resource_count;
> + uint32_t resource_avail;
> + struct xen_domctl_getdomcqminfo *dom_cqminfo;
XEN_GUEST_HANDLE please.
> +};
> +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t;
> +
> /* XEN_SYSCTL_getdomaininfolist */
> struct xen_sysctl_getdomaininfolist {
> /* IN variables. */
> @@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op {
> typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t;
> DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t);
>
> +/* XEN_SYSCTL_getdomcqminfolist */
> +struct xen_sysctl_getdomcqminfolist {
> + /* IN variables. */
> + domid_t first_domain;
> + uint32_t max_domains;
> + XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer;
> + /* OUT variables. */
> + uint32_t num_domains;
> + uint32_t resource_count;
> + uint32_t resource_avail;
> +};
> +typedef struct xen_sysctl_getdomcqminfolist xen_sysctl_getdomcqminfolist_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t);
>
> struct xen_sysctl {
> uint32_t cmd;
> @@ -654,6 +675,7 @@ struct xen_sysctl {
> #define XEN_SYSCTL_cpupool_op 18
> #define XEN_SYSCTL_scheduler_op 19
> #define XEN_SYSCTL_coverage_op 20
> +#define XEN_SYSCTL_getdomcqminfolist 21
> uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
> union {
> struct xen_sysctl_readconsole readconsole;
> @@ -675,6 +697,7 @@ struct xen_sysctl {
> struct xen_sysctl_cpupool_op cpupool_op;
> struct xen_sysctl_scheduler_op scheduler_op;
> struct xen_sysctl_coverage_op coverage_op;
> + struct xen_sysctl_getdomcqminfolist getdomcqminfolist;
> uint8_t pad[128];
> } u;
> };
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [PATCH 6/8] x86: get per domain CQM information
2013-11-20 11:56 ` Andrew Cooper
@ 2013-11-20 13:22 ` Xu, Dongxiao
0 siblings, 0 replies; 17+ messages in thread
From: Xu, Dongxiao @ 2013-11-20 13:22 UTC (permalink / raw)
To: Andrew Cooper; +Cc: xen-devel@lists.xen.org
> -----Original Message-----
> From: Andrew Cooper [mailto:andrew.cooper3@citrix.com]
> Sent: Wednesday, November 20, 2013 7:57 PM
> To: Xu, Dongxiao
> Cc: xen-devel@lists.xen.org
> Subject: Re: [Xen-devel] [PATCH 6/8] x86: get per domain CQM information
>
> On 20/11/13 03:27, dongxiao.xu@intel.com wrote:
> > From: Dongxiao Xu <dongxiao.xu@intel.com>
> >
> > Retrive CQM information for certain domain, which reflects the L3 cache
> > occupancy for a socket.
> >
> > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> > ---
> > xen/arch/x86/pqos.c | 60
> ++++++++++++++++++++++++++++++++++++
> > xen/arch/x86/sysctl.c | 64
> +++++++++++++++++++++++++++++++++++++++
> > xen/include/asm-x86/msr-index.h | 4 +++
> > xen/include/asm-x86/pqos.h | 14 +++++++++
> > xen/include/public/domctl.h | 14 +++++++++
> > xen/include/public/sysctl.h | 23 ++++++++++++++
> > 6 files changed, 179 insertions(+)
> >
> > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> > index 895d892..3699efe 100644
> > --- a/xen/arch/x86/pqos.c
> > +++ b/xen/arch/x86/pqos.c
> > @@ -19,13 +19,30 @@
> > * Place - Suite 330, Boston, MA 02111-1307 USA.
> > */
> > #include <asm/processor.h>
> > +#include <asm/msr.h>
> > +#include <xen/cpumask.h>
> > #include <xen/init.h>
> > #include <xen/spinlock.h>
> > +#include <public/domctl.h>
> > #include <asm/pqos.h>
> >
> > static bool_t pqos_enabled = 1;
> > boolean_param("pqos", pqos_enabled);
> >
> > +static void read_qm_data(void *arg)
> > +{
> > + struct qm_element *qm_element = arg;
> > +
> > + wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid,
> qm_element->rmid);
> > + rdmsrl(MSR_IA32_QMC, qm_element->qm_data);
> > +}
> > +
> > +static void get_generic_qm_info(struct qm_element *qm_element)
> > +{
> > + int cpu = qm_element->cpu;
> > + on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1);
> > +}
> > +
> > unsigned int cqm_res_count = 0;
> > unsigned int cqm_upscaling_factor = 0;
> > bool_t cqm_enabled = 0;
> > @@ -85,6 +102,25 @@ bool_t system_support_cqm(void)
> > return cqm_enabled;
> > }
> >
> > +unsigned int get_cqm_count(void)
> > +{
> > + return cqm_res_count;
> > +}
> > +
> > +unsigned int get_cqm_avail(void)
> > +{
> > + unsigned int cqm_avail = 0;
> > + int i;
> > +
> > + for (i = 0; i < cqm_res_count; i++)
> > + {
> > + if ( !cqm_res_array[i].inuse )
> > + cqm_avail++;
> > + }
>
> Style - extra spaces inside brackets for the for loop, and these braces
> can go.
OK.
>
> > +
> > + return cqm_avail;
> > +}
> > +
> > int alloc_cqm_resource(domid_t domain_id)
> > {
> > int i, rmid = -1;
> > @@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id)
> > spin_unlock_irqrestore(&cqm_lock, flags);
> > }
> >
> > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
> > + struct xen_domctl_getdomcqminfo *info)
> > +{
> > + struct qm_element element;
> > + int cpu, i;
> > +
> > + for_each_cpu ( cpu, &cpu_cqmdata_map )
> > + {
> > + element.cpu = cpu;
> > + element.rmid = rmid;
> > + element.evtid = QOS_MONITOR_EVTID_L3;
> > +
> > + get_generic_qm_info(&element);
> > +
> > + i = cpu_to_socket(cpu);
> > + info->socket_cqmdata[i].valid =
> > + (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1;
> > + if ( info->socket_cqmdata[i].valid )
> > + info->socket_cqmdata[i].l3c_occupancy = element.qm_data *
> cqm_upscaling_factor;
> > + else
> > + info->socket_cqmdata[i].l3c_occupancy = 0;
> > + }
> > +}
> > +
> > /*
> > * Local variables:
> > * mode: C
> > diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
> > index 15d4b91..a779fdc 100644
> > --- a/xen/arch/x86/sysctl.c
> > +++ b/xen/arch/x86/sysctl.c
> > @@ -28,6 +28,7 @@
> > #include <xen/nodemask.h>
> > #include <xen/cpu.h>
> > #include <xsm/xsm.h>
> > +#include <asm/pqos.h>
> >
> > #define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0)
> >
> > @@ -101,6 +102,69 @@ long arch_do_sysctl(
> > }
> > break;
> >
> > + case XEN_SYSCTL_getdomcqminfolist:
> > + {
> > + struct domain *d;
> > + struct xen_domctl_getdomcqminfo info;
> > + uint32_t resource_count;
> > + uint32_t resource_avail;
> > + uint32_t num_domains = 0;
> > + cpumask_t cpu_cqmdata_map;
> > + DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS);
> > + int cpu;
>
> unsigned int.
OK.
>
> > +
> > + if ( !system_support_cqm() )
> > + {
> > + ret = -EFAULT;
>
> ENODEV surely ?
OK.
>
> > + break;
> > + }
> > +
> > + resource_count = get_cqm_count();
> > + resource_avail = get_cqm_avail();
> > +
> > + cpumask_clear(&cpu_cqmdata_map);
> > + bitmap_zero(sockets, QOS_MAX_SOCKETS);
> > + for_each_online_cpu(cpu)
> > + {
> > + int i = cpu_to_socket(cpu);
> > + if ( test_and_set_bit(i, sockets) )
> > + continue;
> > + cpumask_set_cpu(cpu, &cpu_cqmdata_map);
> > + }
> > +
> > + rcu_read_lock(&domlist_read_lock);
> > + for_each_domain ( d )
> > + {
> > + if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain )
> > + continue;
> > + if ( num_domains == sysctl->u.getdomaininfolist.max_domains )
> > + break;
> > + if ( d->arch.pqos_cqm_rmid <= 0 )
> > + continue;
> > + memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo));
> > + info.domain = d->domain_id;
> > + get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map,
> &info);
> > +
> > + if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer,
> > + num_domains, &info, 1) )
> > + {
> > + ret = -EFAULT;
> > + break;
> > + }
> > +
> > + num_domains++;
> > + }
> > + rcu_read_unlock(&domlist_read_lock);
> > +
> > + sysctl->u.getdomcqminfolist.num_domains = num_domains;
> > + sysctl->u.getdomcqminfolist.resource_count = resource_count;
> > + sysctl->u.getdomcqminfolist.resource_avail = resource_avail;
> > +
> > + if ( copy_to_guest(u_sysctl, sysctl, 1) )
> > + ret = -EFAULT;
> > + }
> > + break;
> > +
> > default:
> > ret = -ENOSYS;
> > break;
> > diff --git a/xen/include/asm-x86/msr-index.h
> b/xen/include/asm-x86/msr-index.h
> > index e597a28..46ef165 100644
> > --- a/xen/include/asm-x86/msr-index.h
> > +++ b/xen/include/asm-x86/msr-index.h
> > @@ -488,4 +488,8 @@
> > /* Geode defined MSRs */
> > #define MSR_GEODE_BUSCONT_CONF0 0x00001900
> >
> > +/* Platform QoS register */
> > +#define MSR_IA32_QOSEVTSEL 0x00000c8d
> > +#define MSR_IA32_QMC 0x00000c8e
> > +
> > #endif /* __ASM_MSR_INDEX_H */
> > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> > index 7e32fa5..6d1b1e8 100644
> > --- a/xen/include/asm-x86/pqos.h
> > +++ b/xen/include/asm-x86/pqos.h
> > @@ -27,15 +27,29 @@
> > /* QoS Monitoring Event ID */
> > #define QOS_MONITOR_EVTID_L3 0x1
> >
> > +/* IA32_QM_CTR */
> > +#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62)
> > +
> > struct cqm_res_struct {
> > bool_t inuse;
> > uint16_t domain_id;
> > };
> >
> > +struct qm_element {
> > + uint16_t cpu;
> > + uint32_t rmid;
> > + uint8_t evtid;
> > + uint64_t qm_data;
> > +};
> > +
>
> The packing of this structure is rather poor. Please re-order to reduce
> some of the holes.
>
> Also, cpu parameters are unsigned int in Xen.
Both OK.
>
> > void init_platform_qos(void);
> >
> > bool_t system_support_cqm(void);
> > int alloc_cqm_resource(domid_t);
> > void free_cqm_resource(domid_t);
> > +unsigned int get_cqm_count(void);
> > +unsigned int get_cqm_avail(void);
> > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
> > + struct xen_domctl_getdomcqminfo *info);
> >
> > #endif
> > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> > index 4fe21db..bdefc83 100644
> > --- a/xen/include/public/domctl.h
> > +++ b/xen/include/public/domctl.h
> > @@ -883,6 +883,20 @@ struct xen_domctl_qos_resource {
> > typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t;
> > DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t);
> >
> > +struct xen_socket_cqmdata {
> > + uint8_t valid;
> > + uint64_t l3c_occupancy;
> > +};
> > +
> > +struct xen_domctl_getdomcqminfo {
> > + /* OUT variables. */
> > + domid_t domain;
> > +#define QOS_MAX_SOCKETS 128
> > + struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS];
> > +};
> > +typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t;
> > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t);
> > +
> > struct xen_domctl {
> > uint32_t cmd;
> > #define XEN_DOMCTL_createdomain 1
> > diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
> > index 8437d31..91f206e 100644
> > --- a/xen/include/public/sysctl.h
> > +++ b/xen/include/public/sysctl.h
> > @@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op {
> > typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t;
> > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t);
> >
> > +struct xen_sysctl_getcqminfo
> > +{
> > + uint32_t resource_count;
> > + uint32_t resource_avail;
> > + struct xen_domctl_getdomcqminfo *dom_cqminfo;
>
> XEN_GUEST_HANDLE please.
OK.
Will reflect above suggestions in next version changes.
Thanks,
Dongxiao
>
> > +};
> > +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t;
> > +
> > /* XEN_SYSCTL_getdomaininfolist */
> > struct xen_sysctl_getdomaininfolist {
> > /* IN variables. */
> > @@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op {
> > typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t;
> > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t);
> >
> > +/* XEN_SYSCTL_getdomcqminfolist */
> > +struct xen_sysctl_getdomcqminfolist {
> > + /* IN variables. */
> > + domid_t first_domain;
> > + uint32_t max_domains;
> > + XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer;
> > + /* OUT variables. */
> > + uint32_t num_domains;
> > + uint32_t resource_count;
> > + uint32_t resource_avail;
> > +};
> > +typedef struct xen_sysctl_getdomcqminfolist
> xen_sysctl_getdomcqminfolist_t;
> > +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t);
> >
> > struct xen_sysctl {
> > uint32_t cmd;
> > @@ -654,6 +675,7 @@ struct xen_sysctl {
> > #define XEN_SYSCTL_cpupool_op 18
> > #define XEN_SYSCTL_scheduler_op 19
> > #define XEN_SYSCTL_coverage_op 20
> > +#define XEN_SYSCTL_getdomcqminfolist 21
> > uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
> > union {
> > struct xen_sysctl_readconsole readconsole;
> > @@ -675,6 +697,7 @@ struct xen_sysctl {
> > struct xen_sysctl_cpupool_op cpupool_op;
> > struct xen_sysctl_scheduler_op scheduler_op;
> > struct xen_sysctl_coverage_op coverage_op;
> > + struct xen_sysctl_getdomcqminfolist getdomcqminfolist;
> > uint8_t pad[128];
> > } u;
> > };
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 7/8] tools: get per domain CQM information
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
` (5 preceding siblings ...)
2013-11-20 3:27 ` [PATCH 6/8] x86: get per domain CQM information dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
2013-11-20 3:27 ` [PATCH 8/8] x86: enable CQM monitoring for each domain RMID dongxiao.xu
7 siblings, 0 replies; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
Retrive CQM information for certain domain, which reflects the L3 cache
occupancy for a socket.
The xl command is "xl pqos-list cqm (domid)".
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
tools/libxc/xc_domain.c | 33 ++++++++++++++++
tools/libxc/xenctrl.h | 5 +++
tools/libxl/libxl.h | 5 +++
tools/libxl/libxl_pqos.c | 14 +++++++
tools/libxl/xl.h | 1 +
tools/libxl/xl_cmdimpl.c | 91 +++++++++++++++++++++++++++++++++++++++++++++
tools/libxl/xl_cmdtable.c | 5 +++
7 files changed, 154 insertions(+)
diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index 85c2d4d..96a1d50 100644
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -1794,6 +1794,39 @@ int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags)
return do_domctl(xch, &domctl);
}
+int xc_domain_getcqminfolist(xc_interface *xch,
+ uint32_t first_domain,
+ unsigned int max_domains,
+ sysctl_cqminfo_t *info)
+{
+ int ret = 0;
+ xen_domctl_getdomcqminfo_t *d_info = info->dom_cqminfo;
+ DECLARE_SYSCTL;
+
+ DECLARE_HYPERCALL_BOUNCE(d_info, max_domains*sizeof(*d_info), XC_HYPERCALL_BUFFER_BOUNCE_OUT);
+
+ if ( xc_hypercall_bounce_pre(xch, d_info) )
+ return -1;
+
+ sysctl.cmd = XEN_SYSCTL_getdomcqminfolist;
+ sysctl.u.getdomcqminfolist.first_domain = first_domain;
+ sysctl.u.getdomcqminfolist.max_domains = max_domains;
+ set_xen_guest_handle(sysctl.u.getdomcqminfolist.buffer, d_info);
+
+ if ( xc_sysctl(xch, &sysctl) < 0 )
+ ret = -1;
+ else
+ {
+ ret = sysctl.u.getdomcqminfolist.num_domains;
+ info->resource_count = sysctl.u.getdomcqminfolist.resource_count;
+ info->resource_avail = sysctl.u.getdomcqminfolist.resource_avail;
+ }
+
+ xc_hypercall_bounce_post(xch, d_info);
+
+ return ret;
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index a57f147..b0b1259 100644
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -2395,6 +2395,11 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch,
*/
int xc_kexec_unload(xc_interface *xch, int type);
+typedef xen_sysctl_getcqminfo_t sysctl_cqminfo_t;
int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags);
int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags);
+int xc_domain_getcqminfolist(xc_interface *xch,
+ uint32_t first_domain,
+ unsigned int max_domains,
+ sysctl_cqminfo_t *info);
#endif /* XENCTRL_H */
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index a9a506f..530363c 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -285,6 +285,7 @@
#include <libxl_uuid.h>
#include <_libxl_list.h>
+#include <xenctrl.h>
/* API compatibility. */
#ifdef LIBXL_API_VERSION
@@ -1053,6 +1054,10 @@ int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size);
int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres);
int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres);
+int libxl_list_domain_cqm(libxl_ctx *ctx,
+ uint32_t first_domain,
+ unsigned int max_domains,
+ sysctl_cqminfo_t *info);
/* misc */
diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c
index 6d8013a..e43c706 100644
--- a/tools/libxl/libxl_pqos.c
+++ b/tools/libxl/libxl_pqos.c
@@ -77,6 +77,20 @@ int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres)
return rc;
}
+int libxl_list_domain_cqm(libxl_ctx *ctx,
+ uint32_t first_domain,
+ unsigned int max_domains,
+ sysctl_cqminfo_t *info)
+{
+ int ret;
+
+ ret = xc_domain_getcqminfolist(ctx->xch, first_domain, max_domains, info);
+ if (ret < 0)
+ return -EINVAL;
+
+ return ret;
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 78738b9..994d3be 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -107,6 +107,7 @@ int main_loadpolicy(int argc, char **argv);
int main_remus(int argc, char **argv);
int main_pqosattach(int argc, char **argv);
int main_pqosdetach(int argc, char **argv);
+int main_pqoslist(int argc, char **argv);
void help(const char *command);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 1a12e8f..52c1321 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -7231,6 +7231,97 @@ int main_pqosdetach(int argc, char **argv)
return rc;
}
+static void print_cqm_info(const sysctl_cqminfo_t *info, int nb_domain)
+{
+ int i, j;
+ int socket_num = 0;
+ xen_domctl_getdomcqminfo_t *dom_cqminfo;
+
+ printf("RMID count %5d\tRMID available %5d\n", info->resource_count, info->resource_avail);
+ if (info->resource_count == 0)
+ printf("System doesn't supoort CQM\n");
+ else if (info->resource_count - info->resource_avail == 1)
+ printf("No RMID is assigned to domains\n");
+ else if (!nb_domain)
+ printf("No RMID is assgined to your monitoring domains\n");
+ else {
+ printf("Name ID");
+ do {
+ printf("\tSocketID\tL3C_Usage");
+ socket_num++;
+ } while (info->dom_cqminfo->socket_cqmdata[socket_num].valid);
+ printf("\n");
+ for (i = 0; i < nb_domain; i++) {
+ char *domname;
+ dom_cqminfo = info->dom_cqminfo + i;
+ domname = libxl_domid_to_name(ctx, dom_cqminfo->domain);
+ printf("%-40s %5d",
+ domname,
+ dom_cqminfo->domain);
+ free(domname);
+ for(j = 0; j < socket_num; j++)
+ printf("%10u %16lu ", j, dom_cqminfo->socket_cqmdata[j].l3c_occupancy);
+ printf("\n");
+ }
+ }
+}
+int main_pqoslist(int argc, char **argv)
+{
+ int opt;
+ const char *qosres = NULL;
+ uint32_t first_domain;
+ unsigned int max_domains;
+ int num_domains;
+ sysctl_cqminfo_t info;
+
+ SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-list", 1) {
+ /* No options */
+ }
+
+ qosres = argv[optind];
+
+ if (!strcmp(qosres, "cqm")) {
+ if (optind + 1 >= argc) {
+ first_domain = 0;
+ max_domains = 1024;
+ } else if (optind + 1 == argc - 1) {
+ first_domain = find_domain(argv[optind + 1]);
+ max_domains = 1;
+ } else {
+ help("pqos-list");
+ return 2;
+ }
+
+ info.dom_cqminfo = calloc(max_domains,
+ sizeof(xen_domctl_getdomcqminfo_t));
+ if (!info.dom_cqminfo) {
+ fprintf(stderr, "allocating domain cqminfo failed.\n");
+ return ERROR_FAIL;
+ }
+
+ num_domains = libxl_list_domain_cqm(ctx, first_domain,
+ max_domains, &info);
+
+ if (num_domains < 0) {
+ fprintf(stderr, "Failed to get domain CQM info, "
+ "check whether CQM feature is supported.\n");
+ if (info.dom_cqminfo)
+ free(info.dom_cqminfo);
+ return 1;
+ }
+ print_cqm_info(&info, num_domains);
+
+ if (info.dom_cqminfo)
+ free(info.dom_cqminfo);
+ } else {
+ fprintf(stderr, "qos resource type supported is: cqm\n");
+ help("pqos-list");
+ return 2;
+ }
+
+ return 0;
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 02a2572..6ced416 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -498,6 +498,11 @@ struct cmd_spec cmd_table[] = {
"Reliquish qos resource",
"<Resource> <Domain>",
},
+ { "pqos-list",
+ &main_pqoslist, 0, 0,
+ "List qos information about all/some domains",
+ "<Resource> [Domain]",
+ },
};
int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 8/8] x86: enable CQM monitoring for each domain RMID
2013-11-20 3:27 [PATCH 0/8] enable Cache QoS Monitoring (CQM) feature dongxiao.xu
` (6 preceding siblings ...)
2013-11-20 3:27 ` [PATCH 7/8] tools: " dongxiao.xu
@ 2013-11-20 3:27 ` dongxiao.xu
7 siblings, 0 replies; 17+ messages in thread
From: dongxiao.xu @ 2013-11-20 3:27 UTC (permalink / raw)
To: xen-devel
From: Dongxiao Xu <dongxiao.xu@intel.com>
If the CQM service is attached to a domain, its related RMID will be set
to hardware for monitoring when the domain's vcpu is scheduled in. When
the domain's vcpu is scheduled out, RMID 0 (system reserved) will be set
for monitoring.
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
xen/arch/x86/domain.c | 5 +++++
xen/arch/x86/pqos.c | 21 ++++++++++++++++++++-
xen/include/asm-x86/msr-index.h | 1 +
xen/include/asm-x86/pqos.h | 1 +
4 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 5f920cd..77d3b12 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1375,6 +1375,8 @@ static void __context_switch(void)
{
memcpy(&p->arch.user_regs, stack_regs, CTXT_SWITCH_STACK_BYTES);
vcpu_save_fpu(p);
+ if ( system_support_cqm() )
+ cqm_assoc_rmid(0);
p->arch.ctxt_switch_from(p);
}
@@ -1399,6 +1401,9 @@ static void __context_switch(void)
}
vcpu_restore_fpu_eager(n);
n->arch.ctxt_switch_to(n);
+
+ if ( system_support_cqm() && n->domain->arch.pqos_cqm_rmid > 0 )
+ cqm_assoc_rmid(n->domain->arch.pqos_cqm_rmid);
}
gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
index 3699efe..4b01a7a 100644
--- a/xen/arch/x86/pqos.c
+++ b/xen/arch/x86/pqos.c
@@ -27,6 +27,7 @@
#include <asm/pqos.h>
static bool_t pqos_enabled = 1;
+static unsigned int rmid_bitwidth;
boolean_param("pqos", pqos_enabled);
static void read_qm_data(void *arg)
@@ -43,6 +44,14 @@ static void get_generic_qm_info(struct qm_element *qm_element)
on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1);
}
+void cqm_assoc_rmid(uint16_t rmid)
+{
+ uint64_t val;
+ uint64_t rmid_mask = ~(~0ull << rmid_bitwidth);
+ rdmsrl(MSR_IA32_PQR_ASSOC, val);
+ wrmsrl(MSR_IA32_PQR_ASSOC, (val & ~(rmid_mask)) | (rmid & rmid_mask));
+}
+
unsigned int cqm_res_count = 0;
unsigned int cqm_upscaling_factor = 0;
bool_t cqm_enabled = 0;
@@ -78,13 +87,23 @@ static void init_cqm(void)
static void init_qos_monitor(void)
{
unsigned int qm_features;
- unsigned int eax, ebx, ecx;
+ unsigned int eax, ebx, ecx, i = 0;
if ( !(boot_cpu_has(X86_FEATURE_QOSM)) )
return;
cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
+ while (1)
+ {
+ if ( (1 << i) - 1 >= ebx )
+ {
+ rmid_bitwidth = i;
+ break;
+ }
+ i++;
+ }
+
if ( qm_features & QOS_MONITOR_TYPE_L3 )
init_cqm();
}
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 46ef165..45f4918 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -491,5 +491,6 @@
/* Platform QoS register */
#define MSR_IA32_QOSEVTSEL 0x00000c8d
#define MSR_IA32_QMC 0x00000c8e
+#define MSR_IA32_PQR_ASSOC 0x00000c8f
#endif /* __ASM_MSR_INDEX_H */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
index 6d1b1e8..d5c5e37 100644
--- a/xen/include/asm-x86/pqos.h
+++ b/xen/include/asm-x86/pqos.h
@@ -51,5 +51,6 @@ unsigned int get_cqm_count(void);
unsigned int get_cqm_avail(void);
void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map,
struct xen_domctl_getdomcqminfo *info);
+void cqm_assoc_rmid(uint16_t rmid);
#endif
--
1.7.9.5
^ permalink raw reply related [flat|nested] 17+ messages in thread