[PATCH v4 5/7] x86: enable CQM monitoring for each domain RMID

[Dongxiao Xu <dongxiao.xu@intel.com>]

If the CQM service is attached to a domain, its related RMID will be set
to hardware for monitoring when the domain's vcpu is scheduled in. When
the domain's vcpu is scheduled out, RMID 0 (system reserved) will be set
for monitoring.

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
 xen/arch/x86/domain.c           |    5 +++++
 xen/arch/x86/pqos.c             |   12 ++++++++++++
 xen/include/asm-x86/msr-index.h |    1 +
 xen/include/asm-x86/pqos.h      |    1 +
 4 files changed, 19 insertions(+)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 90e52a2..79bc83c 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1366,6 +1366,8 @@ static void __context_switch(void)
     {
         memcpy(&p->arch.user_regs, stack_regs, CTXT_SWITCH_STACK_BYTES);
         vcpu_save_fpu(p);
+        if ( system_supports_cqm() )
+            cqm_assoc_rmid(0);
         p->arch.ctxt_switch_from(p);
     }
 
@@ -1390,6 +1392,9 @@ static void __context_switch(void)
         }
         vcpu_restore_fpu_eager(n);
         n->arch.ctxt_switch_to(n);
+
+        if ( system_supports_cqm() && n->domain->arch.pqos_cqm_rmid > 0 )
+            cqm_assoc_rmid(n->domain->arch.pqos_cqm_rmid);
     }
 
     gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
index dc0f0fc..c510bba 100644
--- a/xen/arch/x86/pqos.c
+++ b/xen/arch/x86/pqos.c
@@ -58,6 +58,8 @@ static void __init parse_pqos_param(char *s)
 }
 custom_param("pqos", parse_pqos_param);
 
+static uint64_t rmid_mask;
+
 struct pqos_cqm *cqm;
 static DEFINE_SPINLOCK(cqm_lock);
 
@@ -105,6 +107,8 @@ static void __init init_qos_monitor(void)
 
     cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
 
+    rmid_mask = ~(~0ull << get_count_order(ebx));
+
     if ( opt_cqm && (qm_features & QOS_MONITOR_TYPE_L3) )
         init_cqm();
 }
@@ -211,6 +215,14 @@ void get_cqm_info(cpumask_t *cpu_cqmdata_map, struct xen_socket_cqmdata *data)
     on_selected_cpus(cpu_cqmdata_map, read_cqm_data, data, 1);
 }
 
+void cqm_assoc_rmid(unsigned int rmid)
+{
+    uint64_t val;
+
+    rdmsrl(MSR_IA32_PQR_ASSOC, val);
+    wrmsrl(MSR_IA32_PQR_ASSOC, (val & ~(rmid_mask)) | (rmid & rmid_mask));
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 46ef165..45f4918 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -491,5 +491,6 @@
 /* Platform QoS register */
 #define MSR_IA32_QOSEVTSEL             0x00000c8d
 #define MSR_IA32_QMC                   0x00000c8e
+#define MSR_IA32_PQR_ASSOC             0x00000c8f
 
 #endif /* __ASM_MSR_INDEX_H */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
index 49f2302..04dcdee 100644
--- a/xen/include/asm-x86/pqos.h
+++ b/xen/include/asm-x86/pqos.h
@@ -50,5 +50,6 @@ int alloc_cqm_rmid(struct domain *d);
 void free_cqm_rmid(struct domain *d);
 unsigned int get_cqm_count(void);
 void get_cqm_info(cpumask_t *cpu_cqmdata_map, struct xen_socket_cqmdata *data);
+void cqm_assoc_rmid(unsigned int rmid);
 
 #endif
-- 
1.7.9.5