Re: [PATCH v4 4/7] x86: collect CQM information from all sockets

[Andrew Cooper <andrew.cooper3@citrix.com>]

On 03/12/13 08:47, Dongxiao Xu wrote:
> Collect CQM information (L3 cache occupancy) from all sockets.
> Upper layer application can parse the data structure to get the
> information of guest's L3 cache occupancy on certain sockets.
>
> Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
>  xen/arch/x86/pqos.c             |   46 +++++++++++++++++++++++++++++
>  xen/arch/x86/sysctl.c           |   62 +++++++++++++++++++++++++++++++++++++++
>  xen/include/asm-x86/msr-index.h |    4 +++
>  xen/include/asm-x86/pqos.h      |    7 +++++
>  xen/include/public/domctl.h     |    9 ++++++
>  xen/include/public/sysctl.h     |   10 +++++++
>  6 files changed, 138 insertions(+)
>
> diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c
> index dd22cae..dc0f0fc 100644
> --- a/xen/arch/x86/pqos.c
> +++ b/xen/arch/x86/pqos.c
> @@ -19,6 +19,7 @@
>   * Place - Suite 330, Boston, MA 02111-1307 USA.
>   */
>  #include <asm/processor.h>
> +#include <asm/msr.h>
>  #include <xen/init.h>
>  #include <xen/spinlock.h>
>  #include <asm/pqos.h>
> @@ -121,6 +122,12 @@ bool_t system_supports_cqm(void)
>      return !!cqm;
>  }
>  
> +unsigned int get_cqm_count(void)
> +{
> +    ASSERT(system_supports_cqm());
> +    return cqm->max_rmid + 1;
> +}
> +
>  int alloc_cqm_rmid(struct domain *d)
>  {
>      int rc = 0;
> @@ -165,6 +172,45 @@ void free_cqm_rmid(struct domain *d)
>      d->arch.pqos_cqm_rmid = 0;
>  }
>  
> +static void read_cqm_data(void *arg)
> +{
> +    uint64_t cqm_data;
> +    unsigned int rmid;
> +    int socket = cpu_to_socket(smp_processor_id());
> +    struct xen_socket_cqmdata *data = arg;
> +    unsigned long flags, i;
> +
> +    ASSERT(system_supports_cqm());
> +
> +    if ( socket < 0 )
> +        return;
> +
> +    spin_lock_irqsave(&cqm_lock, flags);
> +    for ( rmid = cqm->min_rmid; rmid <= cqm->max_rmid; rmid++ )
> +    {
> +        if ( cqm->rmid_to_dom[rmid] == DOMID_INVALID )
> +            continue;
> +
> +        wrmsr(MSR_IA32_QOSEVTSEL, QOS_MONITOR_EVTID_L3, rmid);
> +        rdmsrl(MSR_IA32_QMC, cqm_data);
> +
> +        i = socket * (cqm->max_rmid + 1) + rmid;
> +        data[i].valid = !(cqm_data & IA32_QM_CTR_ERROR_MASK);
> +        if ( data[i].valid )
> +        {
> +            data[i].l3c_occupancy = cqm_data * cqm->upscaling_factor;
> +            data[i].socket = socket;
> +            data[i].domid = cqm->rmid_to_dom[rmid];
> +        }
> +    }
> +    spin_unlock_irqrestore(&cqm_lock, flags);
> +}
> +
> +void get_cqm_info(cpumask_t *cpu_cqmdata_map, struct xen_socket_cqmdata *data)
> +{
> +    on_selected_cpus(cpu_cqmdata_map, read_cqm_data, data, 1);
> +}
> +
>  /*
>   * Local variables:
>   * mode: C
> diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
> index 15d4b91..3977e7d 100644
> --- a/xen/arch/x86/sysctl.c
> +++ b/xen/arch/x86/sysctl.c
> @@ -28,6 +28,7 @@
>  #include <xen/nodemask.h>
>  #include <xen/cpu.h>
>  #include <xsm/xsm.h>
> +#include <asm/pqos.h>
>  
>  #define get_xen_guest_handle(val, hnd)  do { val = (hnd).p; } while (0)
>  
> @@ -66,6 +67,21 @@ void arch_do_physinfo(xen_sysctl_physinfo_t *pi)
>          pi->capabilities |= XEN_SYSCTL_PHYSCAP_hvm_directio;
>  }
>  
> +/* Select one random CPU for each socket */
> +static void select_socket_cpu(cpumask_t *cpu_bitmap)
> +{
> +    int i;
> +    unsigned int cpu;
> +
> +    cpumask_clear(cpu_bitmap);
> +    for ( i = 0; i < MAX_NUM_SOCKETS; i++ )
> +    {
> +        cpu = cpumask_any(&socket_cpu_map[i]);
> +        if ( cpu < nr_cpu_ids )
> +            cpumask_set_cpu(cpu, cpu_bitmap);
> +    }
> +}
> +
>  long arch_do_sysctl(
>      struct xen_sysctl *sysctl, XEN_GUEST_HANDLE_PARAM(xen_sysctl_t) u_sysctl)
>  {
> @@ -101,6 +117,52 @@ long arch_do_sysctl(
>      }
>      break;
>  
> +    case XEN_SYSCTL_getcqminfo:
> +    {
> +        struct xen_socket_cqmdata *info;
> +        uint32_t num_sockets;
> +        uint32_t num_rmid;
> +        cpumask_t cpu_cqmdata_map;
> +
> +        if ( !system_supports_cqm() )
> +        {
> +            ret = -ENODEV;
> +            break;
> +        }
> +
> +        select_socket_cpu(&cpu_cqmdata_map);

This should exclude the current socket.  There is no point IPI'ing a
different cpu on the same socket for an action this cpu can perform.

~Andrew

> +
> +        num_sockets = min((unsigned int)cpumask_weight(&cpu_cqmdata_map),
> +                          sysctl->u.getcqminfo.num_sockets);
> +        num_rmid = get_cqm_count();
> +        info = xzalloc_array(struct xen_socket_cqmdata,
> +                             num_rmid * num_sockets);
> +        if ( !info )
> +        {
> +            ret = -ENOMEM;
> +            break;
> +        }
> +
> +        get_cqm_info(&cpu_cqmdata_map, info);
> +
> +        if ( copy_to_guest_offset(sysctl->u.getcqminfo.buffer,
> +                                  0, info, num_rmid * num_sockets) )
> +        {
> +            ret = -EFAULT;
> +            xfree(info);
> +            break;
> +        }
> +
> +        sysctl->u.getcqminfo.num_rmid = num_rmid;
> +        sysctl->u.getcqminfo.num_sockets = num_sockets;
> +
> +        if ( copy_to_guest(u_sysctl, sysctl, 1) )
> +            ret = -EFAULT;
> +
> +        xfree(info);
> +    }
> +    break;
> +
>      default:
>          ret = -ENOSYS;
>          break;
> diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
> index e597a28..46ef165 100644
> --- a/xen/include/asm-x86/msr-index.h
> +++ b/xen/include/asm-x86/msr-index.h
> @@ -488,4 +488,8 @@
>  /* Geode defined MSRs */
>  #define MSR_GEODE_BUSCONT_CONF0		0x00001900
>  
> +/* Platform QoS register */
> +#define MSR_IA32_QOSEVTSEL             0x00000c8d
> +#define MSR_IA32_QMC                   0x00000c8e
> +
>  #endif /* __ASM_MSR_INDEX_H */
> diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
> index 9807485..49f2302 100644
> --- a/xen/include/asm-x86/pqos.h
> +++ b/xen/include/asm-x86/pqos.h
> @@ -21,6 +21,8 @@
>  #ifndef ASM_PQOS_H
>  #define ASM_PQOS_H
>  #include <xen/sched.h>
> +#include <xen/cpumask.h>
> +#include <public/domctl.h>
>  
>  #include <public/xen.h>
>  
> @@ -38,10 +40,15 @@ struct pqos_cqm {
>  };
>  extern struct pqos_cqm *cqm;
>  
> +/* IA32_QM_CTR */
> +#define IA32_QM_CTR_ERROR_MASK         (0x3ul << 62)
> +
>  void init_platform_qos(void);
>  
>  bool_t system_supports_cqm(void);
>  int alloc_cqm_rmid(struct domain *d);
>  void free_cqm_rmid(struct domain *d);
> +unsigned int get_cqm_count(void);
> +void get_cqm_info(cpumask_t *cpu_cqmdata_map, struct xen_socket_cqmdata *data);
>  
>  #endif
> diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> index d53e216..563aeaf 100644
> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -877,6 +877,15 @@ struct xen_domctl_qos_type {
>  typedef struct xen_domctl_qos_type xen_domctl_qos_type_t;
>  DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_type_t);
>  
> +struct xen_socket_cqmdata {
> +    uint64_t l3c_occupancy;
> +    uint32_t socket;
> +    domid_t  domid;
> +    uint8_t  valid;
> +};
> +typedef struct xen_socket_cqmdata xen_socket_cqmdata_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_socket_cqmdata_t);
> +
>  struct xen_domctl {
>      uint32_t cmd;
>  #define XEN_DOMCTL_createdomain                   1
> diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
> index 8437d31..8b2844e 100644
> --- a/xen/include/public/sysctl.h
> +++ b/xen/include/public/sysctl.h
> @@ -632,6 +632,14 @@ struct xen_sysctl_coverage_op {
>  typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t;
>  DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t);
>  
> +/* XEN_SYSCTL_getcqminfo */
> +struct xen_sysctl_getcqminfo {
> +    XEN_GUEST_HANDLE_64(xen_socket_cqmdata_t) buffer; /* OUT */
> +    uint32_t              num_sockets;    /* IN/OUT */
> +    uint32_t              num_rmid;       /* OUT */
> +};
> +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getcqminfo_t);
>  
>  struct xen_sysctl {
>      uint32_t cmd;
> @@ -654,6 +662,7 @@ struct xen_sysctl {
>  #define XEN_SYSCTL_cpupool_op                    18
>  #define XEN_SYSCTL_scheduler_op                  19
>  #define XEN_SYSCTL_coverage_op                   20
> +#define XEN_SYSCTL_getcqminfo                    21
>      uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
>      union {
>          struct xen_sysctl_readconsole       readconsole;
> @@ -675,6 +684,7 @@ struct xen_sysctl {
>          struct xen_sysctl_cpupool_op        cpupool_op;
>          struct xen_sysctl_scheduler_op      scheduler_op;
>          struct xen_sysctl_coverage_op       coverage_op;
> +        struct xen_sysctl_getcqminfo        getcqminfo;
>          uint8_t                             pad[128];
>      } u;
>  };