Re: [PATCH v16 10/10] tools: CMDs and APIs for Cache Monitoring Technology

[Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>]

On Thu, Sep 25, 2014 at 06:19:10PM +0800, Chao Peng wrote:
> Introduced some new xl commands to enable/disable Cache Monitoring
> Technology(CMT) feature.
> 
> The following two commands is to attach/detach the CMT feature
> to/from a certain domain.
> 
> $ xl psr-cmt-attach domid
> $ xl psr-cmt-detach domid
> 
> This command is to display the CMT information, such as L3 cache
> occupancy.
> 
> $ xl psr-cmt-show cache_occupancy <domid>
> 
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> ---
>  docs/man/xl.pod.1           |   25 +++++
>  tools/libxc/Makefile        |    1 +
>  tools/libxc/xc_msr_x86.h    |   36 ++++++++
>  tools/libxc/xc_psr.c        |  215 +++++++++++++++++++++++++++++++++++++++++++
>  tools/libxc/xenctrl.h       |   17 ++++
>  tools/libxl/Makefile        |    2 +-
>  tools/libxl/libxl.h         |   19 ++++
>  tools/libxl/libxl_psr.c     |  184 ++++++++++++++++++++++++++++++++++++
>  tools/libxl/libxl_types.idl |    4 +
>  tools/libxl/libxl_utils.c   |   28 ++++++
>  tools/libxl/xl.h            |    3 +
>  tools/libxl/xl_cmdimpl.c    |  131 ++++++++++++++++++++++++++
>  tools/libxl/xl_cmdtable.c   |   17 ++++
>  13 files changed, 681 insertions(+), 1 deletion(-)
>  create mode 100644 tools/libxc/xc_msr_x86.h
>  create mode 100644 tools/libxc/xc_psr.c
>  create mode 100644 tools/libxl/libxl_psr.c
> 
> diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
> index f1e95db..ef8cd24 100644
> --- a/docs/man/xl.pod.1
> +++ b/docs/man/xl.pod.1
> @@ -1387,6 +1387,31 @@ Load FLASK policy from the given policy file. The initial policy is provided to
>  the hypervisor as a multiboot module; this command allows runtime updates to the
>  policy. Loading new security policy will reset runtime changes to device labels.
>  
> +=head1 Cache Monitoring Technology
> +
> +Some new hardware may offer monitoring capability in each logical processor to
> +measure specific platform shared resource metric, for example, L3 cache
> +occupancy. In Xen implementation, the monitoring granularity is domain level.
> +To monitor a specific domain, just attach the domain id with the monitoring
> +service. When the domain doesn't need to be monitored any more, detach the
> +domain id from the monitoring service.
> +
> +=over 4
> +
> +=item B<psr-cmt-attach> [I<domain-id>]
> +
> +attach: Attach the platform shared resource monitoring service to a domain.
> +
> +=item B<psr-cmt-detach> [I<domain-id>]
> +
> +detach: Detach the platform shared resource monitoring service from a domain.
> +
> +=item B<psr-cmt-show> [I<psr-monitor-type>] [I<domain-id>]
> +
> +Show monitoring data for a certain domain or all domains. Current supported
> +monitor types are:
> + - "cache-occupancy": showing the L3 cache occupancy.
> +
>  =back
>  
>  =head1 TO BE DOCUMENTED
> diff --git a/tools/libxc/Makefile b/tools/libxc/Makefile
> index dde6109..d8cc21b 100644
> --- a/tools/libxc/Makefile
> +++ b/tools/libxc/Makefile
> @@ -35,6 +35,7 @@ CTRL_SRCS-y       += xc_kexec.c
>  CTRL_SRCS-y       += xtl_core.c
>  CTRL_SRCS-y       += xtl_logger_stdio.c
>  CTRL_SRCS-y       += xc_resource.c
> +CTRL_SRCS-y       += xc_psr.c
>  CTRL_SRCS-$(CONFIG_X86) += xc_pagetab.c
>  CTRL_SRCS-$(CONFIG_Linux) += xc_linux.c xc_linux_osdep.c
>  CTRL_SRCS-$(CONFIG_FreeBSD) += xc_freebsd.c xc_freebsd_osdep.c
> diff --git a/tools/libxc/xc_msr_x86.h b/tools/libxc/xc_msr_x86.h
> new file mode 100644
> index 0000000..1e0ee99
> --- /dev/null
> +++ b/tools/libxc/xc_msr_x86.h
> @@ -0,0 +1,36 @@
> +/*
> + * xc_msr_x86.h
> + *
> + * MSR definition macros
> + *
> + * Copyright (C) 2014      Intel Corporation
> + * Author Dongxiao Xu <dongxiao.xu@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#ifndef XC_MSR_X86_H
> +#define XC_MSR_X86_H
> +
> +#define MSR_IA32_QOSEVTSEL      0x00000c8d
> +#define MSR_IA32_QMC            0x00000c8e
> +
> +#endif
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-file-style: "BSD"
> + * c-basic-offset: 4
> + * tab-width: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
> new file mode 100644
> index 0000000..0b5a227
> --- /dev/null
> +++ b/tools/libxc/xc_psr.c
> @@ -0,0 +1,215 @@
> +/*
> + * xc_psr.c
> + *
> + * platform shared resource related API functions.
> + *
> + * Copyright (C) 2014      Intel Corporation
> + * Author Dongxiao Xu <dongxiao.xu@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "xc_private.h"
> +#include "xc_msr_x86.h"
> +
> +#define IA32_QM_CTR_ERROR_MASK         (0x3ull << 62)
> +
> +#define EVTID_L3_OCCUPANCY             0x1
> +
> +int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid)
> +{
> +    DECLARE_DOMCTL;
> +
> +    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
> +    domctl.domain = (domid_t)domid;
> +    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_ATTACH;
> +
> +    return do_domctl(xch, &domctl);
> +}
> +
> +int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid)
> +{
> +    DECLARE_DOMCTL;
> +
> +    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
> +    domctl.domain = (domid_t)domid;
> +    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_DETACH;
> +
> +    return do_domctl(xch, &domctl);
> +}
> +
> +int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
> +                                    uint32_t *rmid)
> +{
> +    int rc;
> +    DECLARE_DOMCTL;
> +
> +    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
> +    domctl.domain = (domid_t)domid;
> +    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_QUERY_RMID;
> +
> +    rc = do_domctl(xch, &domctl);
> +
> +    if ( !rc )
> +        *rmid = domctl.u.psr_cmt_op.data;
> +
> +    return rc;
> +}
> +
> +int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid)
> +{
> +    static int val = 0;
> +    int rc;
> +    DECLARE_SYSCTL;
> +
> +    if ( val )
> +    {
> +        *total_rmid = val;
> +        return 0;
> +    }
> +
> +    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
> +    sysctl.u.psr_cmt_op.cmd = XEN_SYSCTL_PSR_CMT_get_total_rmid;
> +    sysctl.u.psr_cmt_op.flags = 0;
> +
> +    rc = xc_sysctl(xch, &sysctl);
> +    if ( !rc )
> +        val = *total_rmid = sysctl.u.psr_cmt_op.data;
> +
> +    return rc;
> +}
> +
> +int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
> +                                            uint32_t *upscaling_factor)
> +{
> +    static int val = 0;
> +    int rc;
> +    DECLARE_SYSCTL;
> +
> +    if ( val )
> +    {
> +        *upscaling_factor = val;
> +        return 0;
> +    }
> +
> +    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
> +    sysctl.u.psr_cmt_op.cmd =
> +        XEN_SYSCTL_PSR_CMT_get_l3_upscaling_factor;
> +    sysctl.u.psr_cmt_op.flags = 0;
> +
> +    rc = xc_sysctl(xch, &sysctl);
> +    if ( !rc )
> +        val = *upscaling_factor = sysctl.u.psr_cmt_op.data;
> +
> +    return rc;
> +}
> +
> +int xc_psr_cmt_get_l3_cache_size(xc_interface *xch,
> +                                      uint32_t *l3_cache_size)
> +{
> +    static int val = 0;
> +    int rc;
> +    DECLARE_SYSCTL;
> +
> +    if ( val )
> +    {
> +        *l3_cache_size = val;
> +        return 0;
> +    }
> +
> +    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
> +    sysctl.u.psr_cmt_op.cmd =
> +        XEN_SYSCTL_PSR_CMT_get_l3_cache_size;
> +    sysctl.u.psr_cmt_op.flags = 0;
> +
> +    rc = xc_sysctl(xch, &sysctl);
> +    if ( !rc )
> +        val = *l3_cache_size= sysctl.u.psr_cmt_op.data;
> +
> +    return rc;
> +}
> +
> +int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid,
> +    uint32_t cpu, xc_psr_cmt_type type, uint64_t *monitor_data)
> +{
> +    xc_resource_op_t op;
> +    xc_resource_data_t entries[2];
> +    uint32_t evtid;
> +    int rc;
> +
> +    switch ( type )
> +    {
> +    case XC_PSR_CMT_L3_OCCUPANCY:
> +        evtid = EVTID_L3_OCCUPANCY;
> +        break;
> +    default:
> +        return -1;
> +    }
> +
> +    entries[0].cmd = XEN_RESOURCE_OP_MSR_WRITE;
> +    entries[0].idx = MSR_IA32_QOSEVTSEL;
> +    entries[0].val = (uint64_t)rmid << 32 | evtid;
> +    entries[0].rsvd = 0;
> +
> +    entries[1].cmd = XEN_RESOURCE_OP_MSR_READ;
> +    entries[1].idx = MSR_IA32_QMC;
> +    entries[1].val = 0;
> +    entries[1].rsvd = 0;
> +
> +    op.result = 0;
> +    op.cpu = cpu;
> +    op.nr_entries = 2;
> +    op.entries = entries;
> +
> +    rc = xc_resource_op(xch, 1, &op);
> +    if ( rc )
> +        return rc;
> +
> +    if ( op.result || entries[1].val & IA32_QM_CTR_ERROR_MASK )
> +        return -1;
> +
> +    *monitor_data = entries[1].val;
> +
> +    return 0;
> +}
> +
> +int xc_psr_cmt_enabled(xc_interface *xch)
> +{
> +    static int val = -1;
> +    int rc;
> +    DECLARE_SYSCTL;
> +
> +    if ( val >= 0 )
> +        return val;
> +
> +    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
> +    sysctl.u.psr_cmt_op.cmd = XEN_SYSCTL_PSR_CMT_enabled;
> +    sysctl.u.psr_cmt_op.flags = 0;
> +
> +    rc = do_sysctl(xch, &sysctl);
> +    if ( !rc )
> +    {
> +        val = sysctl.u.psr_cmt_op.data;
> +        return val;
> +    }
> +
> +    return 0;
> +}
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-file-style: "BSD"
> + * c-basic-offset: 4
> + * tab-width: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
> index fb58c44..d5a3c0a 100644
> --- a/tools/libxc/xenctrl.h
> +++ b/tools/libxc/xenctrl.h
> @@ -2668,6 +2668,23 @@ struct xc_resource_op {
>  typedef struct xc_resource_op xc_resource_op_t;
>  int xc_resource_op(xc_interface *xch, uint32_t nr_ops, xc_resource_op_t *ops);
>  
> +enum xc_psr_cmt_type {
> +    XC_PSR_CMT_L3_OCCUPANCY,
> +};
> +typedef enum xc_psr_cmt_type xc_psr_cmt_type;
> +int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid);
> +int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid);
> +int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
> +    uint32_t *rmid);
> +int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid);
> +int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
> +    uint32_t *upscaling_factor);
> +int xc_psr_cmt_get_l3_cache_size(xc_interface *xch,
> +    uint32_t *l3_cache_size);
> +int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid,
> +    uint32_t cpu, uint32_t psr_cmt_type, uint64_t *monitor_data);
> +int xc_psr_cmt_enabled(xc_interface *xch);
> +
>  #endif /* XENCTRL_H */
>  
>  /*
> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
> index 990414b..fd9ae28 100644
> --- a/tools/libxl/Makefile
> +++ b/tools/libxl/Makefile
> @@ -43,7 +43,7 @@ LIBXL_OBJS-y += libxl_blktap2.o
>  else
>  LIBXL_OBJS-y += libxl_noblktap2.o
>  endif
> -LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
> +LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
>  LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
>  
>  ifeq ($(CONFIG_NetBSD),y)
> diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
> index bc68cac..5acc933 100644
> --- a/tools/libxl/libxl.h
> +++ b/tools/libxl/libxl.h
> @@ -640,6 +640,13 @@ typedef uint8_t libxl_mac[6];
>  #define LIBXL_MAC_BYTES(mac) mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
>  void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src);
>  
> +/*
> + * LIBXL_HAVE_PSR_CMT
> + *
> + * If this is defined, the Cache Monitoring Technology feature is supported.
> + */
> +#define LIBXL_HAVE_PSR_CMT 1
> +
>  typedef char **libxl_string_list;
>  void libxl_string_list_dispose(libxl_string_list *sl);
>  int libxl_string_list_length(const libxl_string_list *sl);
> @@ -1380,6 +1387,18 @@ bool libxl_ms_vm_genid_is_zero(const libxl_ms_vm_genid *id);
>  void libxl_ms_vm_genid_copy(libxl_ctx *ctx, libxl_ms_vm_genid *dst,
>                              libxl_ms_vm_genid *src);
>  
> +int libxl_get_socket_cpu(libxl_ctx *ctx, uint32_t socketid);
> +
> +int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid);
> +int libxl_psr_cmt_detach(libxl_ctx *ctx, uint32_t domid);
> +int libxl_psr_cmt_domain_attached(libxl_ctx *ctx, uint32_t domid);
> +int libxl_psr_cmt_enabled(libxl_ctx *ctx);
> +int libxl_psr_cmt_get_total_rmid(libxl_ctx *ctx, uint32_t *total_rmid);
> +int libxl_psr_cmt_get_l3_cache_size(libxl_ctx *ctx,
> +    uint32_t *l3_cache_size);
> +int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, uint32_t domid,
> +    uint32_t socketid, uint32_t *l3_cache_occupancy);
> +
>  /* misc */
>  
>  /* Each of these sets or clears the flag according to whether the
> diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
> new file mode 100644
> index 0000000..5551be8
> --- /dev/null
> +++ b/tools/libxl/libxl_psr.c
> @@ -0,0 +1,184 @@
> +/*
> + * Copyright (C) 2014      Intel Corporation
> + * Author Dongxiao Xu <dongxiao.xu@intel.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "libxl_osdeps.h" /* must come before any other headers */
> +#include "libxl_internal.h"
> +
> +
> +#define IA32_QM_CTR_ERROR_MASK         (0x3ul << 62)
> +
> +static void libxl_psr_cmt_err_msg(libxl_ctx *ctx, int err)
> +{
> +    GC_INIT(ctx);
> +
> +    char *msg;
> +
> +    switch (err) {
> +    case ENOSYS:
> +        msg = "unsupported operation";
> +        break;
> +    case ENODEV:
> +        msg = "Cache Monitoring Technology is not supported in this system";
> +        break;
> +    case EEXIST:
> +        msg = "Cache Monitoring Technology is already attached to this domain";
> +        break;
> +    case ENOENT:
> +        msg = "Cache Monitoring Technology is not attached to this domain";
> +        break;
> +    case EUSERS:
> +        msg = "there is no free RMID available";
> +        break;
> +    case ESRCH:
> +        msg = "is this Domain ID valid?";
> +        break;
> +    case EFAULT:
> +        msg = "failed to exchange data with Xen";
> +        break;
> +    default:
> +        msg = "unknown error";
> +        break;
> +    }
> +
> +    LOGE(ERROR, "%s", msg);
> +
> +    GC_FREE;
> +}
> +
> +int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid)
> +{
> +    int rc;
> +
> +    rc = xc_psr_cmt_attach(ctx->xch, domid);
> +    if (rc < 0) {
> +        libxl_psr_cmt_err_msg(ctx, errno);
> +        return ERROR_FAIL;
> +    }
> +
> +    return 0;
> +}
> +
> +int libxl_psr_cmt_detach(libxl_ctx *ctx, uint32_t domid)
> +{
> +    int rc;
> +
> +    rc = xc_psr_cmt_detach(ctx->xch, domid);
> +    if (rc < 0) {
> +        libxl_psr_cmt_err_msg(ctx, errno);
> +        return ERROR_FAIL;
> +    }
> +
> +    return 0;
> +}
> +
> +int libxl_psr_cmt_domain_attached(libxl_ctx *ctx, uint32_t domid)
> +{
> +    int rc;
> +    uint32_t rmid;
> +
> +    rc = xc_psr_cmt_get_domain_rmid(ctx->xch, domid, &rmid);
> +    if (rc < 0)
> +        return 0;
> +
> +    return !!rmid;
> +}
> +
> +int libxl_psr_cmt_enabled(libxl_ctx *ctx)
> +{
> +    return xc_psr_cmt_enabled(ctx->xch);
> +}
> +
> +int libxl_psr_cmt_get_total_rmid(libxl_ctx *ctx, uint32_t *total_rmid)
> +{
> +    int rc;
> +
> +    rc = xc_psr_cmt_get_total_rmid(ctx->xch, total_rmid);
> +    if (rc < 0) {
> +        libxl_psr_cmt_err_msg(ctx, errno);
> +        return ERROR_FAIL;
> +    }
> +
> +    return 0;
> +}
> +
> +int libxl_psr_cmt_get_l3_cache_size(libxl_ctx *ctx,
> +                                         uint32_t *l3_cache_size)
> +{
> +    int rc;
> +
> +    rc = xc_psr_cmt_get_l3_cache_size(ctx->xch, l3_cache_size);
> +    if (rc < 0) {
> +        libxl_psr_cmt_err_msg(ctx, errno);
> +        return ERROR_FAIL;
> +    }
> +
> +    return 0;
> +}
> +
> +int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, uint32_t domid,
> +    uint32_t socketid, uint32_t *l3_cache_occupancy)
> +{
> +    GC_INIT(ctx);
> +
> +    unsigned int rmid;
> +    uint32_t upscaling_factor;
> +    uint64_t monitor_data;
> +    int cpu, rc;
> +    xc_psr_cmt_type type;
> +
> +    rc = xc_psr_cmt_get_domain_rmid(ctx->xch, domid, &rmid);
> +    if (rc < 0 || rmid == 0) {
> +        LOGE(ERROR, "fail to get the domain rmid, "
> +            "or domain is not attached with platform QoS monitoring service");
> +        rc = ERROR_FAIL;
> +        goto out;
> +    }
> +
> +    cpu = libxl_get_socket_cpu(ctx, socketid);
> +    if (cpu < 0) {
> +        LOGE(ERROR, "failed to get socket cpu");
> +        rc = ERROR_FAIL;
> +        goto out;
> +    }
> +
> +    type = XC_PSR_CMT_L3_OCCUPANCY;
> +    rc = xc_psr_cmt_get_data(ctx->xch, rmid, cpu, type, &monitor_data);
> +    if (rc < 0) {
> +        LOGE(ERROR, "failed to get monitoring data");
> +        rc = ERROR_FAIL;
> +        goto out;
> +    }
> +
> +    rc = xc_psr_cmt_get_l3_upscaling_factor(ctx->xch, &upscaling_factor);
> +    if (rc < 0) {
> +        LOGE(ERROR, "failed to get L3 upscaling factor");
> +        rc = ERROR_FAIL;
> +        goto out;
> +    }
> +
> +    *l3_cache_occupancy = upscaling_factor * monitor_data / 1024;
> +    rc = 0;
> +out:
> +    GC_FREE;
> +    return 0;
> +}
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-basic-offset: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
> index f1fcbc3..27a5022 100644
> --- a/tools/libxl/libxl_types.idl
> +++ b/tools/libxl/libxl_types.idl
> @@ -635,3 +635,7 @@ libxl_event = Struct("event",[
>                                   ])),
>             ("domain_create_console_available", None),
>             ]))])
> +
> +libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
> +    (1, "CACHE_OCCUPANCY"),
> +    ])
> diff --git a/tools/libxl/libxl_utils.c b/tools/libxl/libxl_utils.c
> index 58df4f3..8ec822d 100644
> --- a/tools/libxl/libxl_utils.c
> +++ b/tools/libxl/libxl_utils.c
> @@ -1065,6 +1065,34 @@ int libxl__random_bytes(libxl__gc *gc, uint8_t *buf, size_t len)
>      return ret;
>  }
>  
> +int libxl_get_socket_cpu(libxl_ctx *ctx, uint32_t socketid)
> +{
> +    int i, j, cpu, nr_cpus;
> +    libxl_cputopology *topology;
> +    int *socket_cpus;
> +
> +    topology = libxl_get_cpu_topology(ctx, &nr_cpus);
> +    if (!topology)
> +        return ERROR_FAIL;
> +
> +    socket_cpus = malloc(sizeof(int) * nr_cpus);
> +    if (!socket_cpus) {
> +        free(topology);

Not libxl_cputopology_list_free(topology, nr_cpus) ?
That is how for example 'libxl_nodemap_to_cpumap' does it.


> +        return ERROR_FAIL;
> +    }
> +
> +    for (i = 0, j = 0; i < nr_cpus; i++)
> +        if (topology[i].socket == socketid)
> +            socket_cpus[j++] = i;
> +
> +    cpu = socket_cpus[rand() % j];


Could you describe the reasoning behind this please?

> +
> +    free(socket_cpus);
> +    free(topology);
> +
> +    return cpu;
> +}
> +
>  /*
>   * Local variables:
>   * mode: C
> diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
> index 10a2e66..abb1fac 100644
> --- a/tools/libxl/xl.h
> +++ b/tools/libxl/xl.h
> @@ -110,6 +110,9 @@ int main_loadpolicy(int argc, char **argv);
>  int main_remus(int argc, char **argv);
>  #endif
>  int main_devd(int argc, char **argv);
> +int main_psr_cmt_attach(int argc, char **argv);
> +int main_psr_cmt_detach(int argc, char **argv);
> +int main_psr_cmt_show(int argc, char **argv);
>  
>  void help(const char *command);
>  
> diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
> index 698b3bc..1fa8755 100644
> --- a/tools/libxl/xl_cmdimpl.c
> +++ b/tools/libxl/xl_cmdimpl.c
> @@ -7428,6 +7428,137 @@ out:
>      return ret;
>  }
>  
> +static int psr_cmt_show_cache_occupancy(uint32_t domid)
> +{
> +    uint32_t i, socketid, nr_sockets, total_rmid;
> +    uint32_t l3_cache_size, l3_cache_occupancy;
> +    libxl_physinfo info;
> +    char *domain_name;
> +    int rc, print_header, nr_domains;
> +    libxl_dominfo *dominfo;
> +
> +    if (!libxl_psr_cmt_enabled(ctx)) {
> +        printf("CMT is not supported in the system\n");

s/not supported/not support (or disabled)/

> +        return -1;
> +    }
> +
> +    rc = libxl_get_physinfo(ctx, &info);
> +    if (rc < 0) {
> +        printf("failed to get system socket number\n");

That is not exactly what this hypercall gets you. I would just
say: "Failed getting physinfo, rc: %d."

> +        return -1;
> +    }
> +    nr_sockets = info.nr_cpus / info.threads_per_core / info.cores_per_socket;
> +
> +    rc = libxl_psr_cmt_get_total_rmid(ctx, &total_rmid);
> +    if (rc < 0) {
> +        printf("failed to get system total rmid number\n");

Failed to get max RMID value.
> +        return -1;
> +    }
> +
> +    rc = libxl_psr_cmt_get_l3_cache_size(ctx, &l3_cache_size);
> +    if (rc < 0) {
> +        printf("failed to get system l3 cache size\n");

Missing full stop.

> +        return -1;
> +    }
> +
> +    printf("Total RMID: %d\n", total_rmid);
> +    printf("Per-Socket L3 Cache Size: %d KB\n", l3_cache_size);
> +
> +    print_header = 1;
> +    if (!(dominfo = libxl_list_domain(ctx, &nr_domains))) {
> +        fprintf(stderr, "libxl_list_domain failed.\n");
> +        return -1;
> +    }
> +    for (i = 0; i < nr_domains; i++) {
> +        if (domid != ~0 && dominfo[i].domid != domid)
> +            continue;
> +        if (!libxl_psr_cmt_domain_attached(ctx, dominfo[i].domid))
> +            continue;
> +        if (print_header) {
> +            printf("%-40s %5s", "Name", "ID");
> +            for (socketid = 0; socketid < nr_sockets; socketid++)
> +                printf("%14s %d", "Socket", socketid);
> +            printf("\n");
> +            print_header = 0;
> +        }
> +        domain_name = libxl_domid_to_name(ctx, dominfo[i].domid);
> +        printf("%-40s %5d", domain_name, dominfo[i].domid);
> +        free(domain_name);
> +        for (socketid = 0; socketid < nr_sockets; socketid++) {
> +            rc = libxl_psr_cmt_get_cache_occupancy(ctx, dominfo[i].domid,
> +                     socketid, &l3_cache_occupancy);

Not checking the 'rc' ?

> +            printf("%13u KB", l3_cache_occupancy);
> +        }
> +        printf("\n");
> +    }
> +    libxl_dominfo_list_free(dominfo, nr_domains);
> +
> +    return 0;
> +}
> +
> +int main_psr_cmt_attach(int argc, char **argv)
> +{
> +    uint32_t domid;
> +    int opt, ret = 0;
> +
> +    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-attach", 1) {
> +        /* No options */
> +    }
> +
> +    domid = find_domain(argv[optind]);
> +    ret = libxl_psr_cmt_attach(ctx, domid);
> +
> +    return ret;
> +}
> +
> +int main_psr_cmt_detach(int argc, char **argv)
> +{
> +    uint32_t domid;
> +    int opt, ret = 0;
> +
> +    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-detach", 1) {
> +        /* No options */
> +    }
> +
> +    domid = find_domain(argv[optind]);
> +    ret = libxl_psr_cmt_detach(ctx, domid);
> +
> +    return ret;
> +}
> +
> +int main_psr_cmt_show(int argc, char **argv)
> +{
> +    int opt, ret = 0;
> +    uint32_t domid;
> +    libxl_psr_cmt_type type;
> +
> +    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-show", 1) {
> +        /* No options */
> +    }
> +
> +    libxl_psr_cmt_type_from_string(argv[optind], &type);
> +
> +    if (optind + 1 >= argc)
> +        domid = ~0;
> +    else if (optind + 1 == argc - 1)
> +        domid = find_domain(argv[optind + 1]);
> +    else {
> +        help("psr-cmt-show");
> +        return 2;
> +    }
> +
> +    switch (type) {
> +    case LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY:
> +        ret = psr_cmt_show_cache_occupancy(domid);
> +        break;
> +    default:
> +        help("psr-cmt-show");
> +        return 2;
> +    }
> +
> +    return ret;
> +}
> +
>  /*
>   * Local variables:
>   * mode: C
> diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
> index 35f02c4..da25c3f 100644
> --- a/tools/libxl/xl_cmdtable.c
> +++ b/tools/libxl/xl_cmdtable.c
> @@ -502,6 +502,23 @@ struct cmd_spec cmd_table[] = {
>        "[options]",
>        "-F                      Run in the foreground",
>      },
> +    { "psr-cmt-attach",
> +      &main_psr_cmt_attach, 0, 1,
> +      "Attach Cache Monitoring Technology service to a domain",
> +      "<Domain>",
> +    },
> +    { "psr-cmt-detach",
> +      &main_psr_cmt_detach, 0, 1,
> +      "Detach Cache Monitoring Technology service from a domain",
> +      "<Domain>",
> +    },
> +    { "psr-cmt-show",
> +      &main_psr_cmt_show, 0, 1,
> +      "Show Cache Monitoring Technology information",
> +      "<PSR-CMT-Type> <Domain>",
> +      "Available monitor types:\n"
> +      "\"cache_occupancy\":         Show L3 cache occupancy\n",
> +    },
>  };
>  
>  int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
> -- 
> 1.7.9.5
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel