[PATCH v10 4/6] x86: enable CQM monitoring for each domain RMID

[Dongxiao Xu <dongxiao.xu@intel.com>]

If the CQM service is attached to a domain, its related RMID will be set
to hardware for monitoring when the domain's vcpu is scheduled in. When
the domain's vcpu is scheduled out, RMID 0 (system reserved) will be set
for monitoring.

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
---
 xen/arch/x86/domain.c           |    5 +++++
 xen/arch/x86/pqos/cqm.c         |   21 +++++++++++++++++++++
 xen/include/asm-x86/msr-index.h |    1 +
 xen/include/asm-x86/pqos.h      |    6 ++++++
 4 files changed, 33 insertions(+)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 3ea9402..e256cbb 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1380,6 +1380,8 @@ static void __context_switch(void)
     {
         memcpy(&p->arch.user_regs, stack_regs, CTXT_SWITCH_STACK_BYTES);
         vcpu_save_fpu(p);
+        if ( system_supports_cqm() && cqm->rmid_inuse )
+            cqm_assoc_rmid(0);
         p->arch.ctxt_switch_from(p);
     }
 
@@ -1404,6 +1406,9 @@ static void __context_switch(void)
         }
         vcpu_restore_fpu_eager(n);
         n->arch.ctxt_switch_to(n);
+
+        if ( system_supports_cqm() && n->domain->arch.pqos_cqm_rmid > 0 )
+            cqm_assoc_rmid(n->domain->arch.pqos_cqm_rmid);
     }
 
     gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
diff --git a/xen/arch/x86/pqos/cqm.c b/xen/arch/x86/pqos/cqm.c
index 4913975..7a936e0 100644
--- a/xen/arch/x86/pqos/cqm.c
+++ b/xen/arch/x86/pqos/cqm.c
@@ -24,6 +24,7 @@
 #include <asm/pqos.h>
 
 struct pqos_cqm *__read_mostly cqm = NULL;
+static DEFINE_PER_CPU(struct pqr_assoc, pqr_assoc);
 
 static int cqm_add_socket(int socket)
 {
@@ -260,6 +261,26 @@ void get_cqm_info(const cpumask_t *cpu_cqmdata_map)
     on_selected_cpus(cpu_cqmdata_map, read_cqm_data, NULL, 1);
 }
 
+void cqm_assoc_rmid(unsigned int rmid)
+{
+    uint64_t val;
+    uint64_t new_val;
+
+    if ( !this_cpu(pqr_assoc).initialized )
+    {
+        rdmsrl(MSR_IA32_PQR_ASSOC, this_cpu(pqr_assoc).val);
+        this_cpu(pqr_assoc).initialized = 1;
+    }
+    val = this_cpu(pqr_assoc).val;
+
+    new_val = (val & ~cqm->rmid_mask) | (rmid & cqm->rmid_mask);
+    if ( val != new_val )
+    {
+        wrmsrl(MSR_IA32_PQR_ASSOC, new_val);
+        this_cpu(pqr_assoc).val = new_val;
+    }
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 847aeb9..1fc90a6 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -485,5 +485,6 @@
 /* Platform QoS register */
 #define MSR_IA32_QOSEVTSEL             0x00000c8d
 #define MSR_IA32_QMC                   0x00000c8e
+#define MSR_IA32_PQR_ASSOC             0x00000c8f
 
 #endif /* __ASM_MSR_INDEX_H */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
index 55b5187..83c6ac6 100644
--- a/xen/include/asm-x86/pqos.h
+++ b/xen/include/asm-x86/pqos.h
@@ -47,6 +47,11 @@ struct pqos_cqm {
 };
 extern struct pqos_cqm *cqm;
 
+struct pqr_assoc {
+    uint64_t val;
+    bool_t initialized;
+};
+
 static inline bool_t system_supports_cqm(void)
 {
     return !!cqm;
@@ -58,5 +63,6 @@ void __init init_cqm(unsigned int rmid_max, unsigned long rmid_mask);
 int alloc_cqm_rmid(struct domain *d);
 void free_cqm_rmid(struct domain *d);
 void get_cqm_info(const cpumask_t *cpu_cqmdata_map);
+void cqm_assoc_rmid(unsigned int rmid);
 
 #endif
-- 
1.7.9.5