Re: [PATCH v3 8/8] tools: add tools support for Intel CAT

[Ian Campbell <ian.campbell@citrix.com>]

On Thu, 2015-03-26 at 20:38 +0800, Chao Peng wrote:
> This is the xc/xl changes to support Intel Cache Allocation
> Technology(CAT). Two commands are introduced:
> - xl psr-cat-cbm-set [-s socket] <domain> <cbm>
>   Set cache capacity bitmasks(CBM) for a domain.
> - xl psr-cat-show <domain>
>   Show Cache Allocation Technology information.

Please could you show an example of the output from this one.

> 
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> ---
> Changes in v3:
> * Add manpage.
> * libxl_psr_cat_set/get_domain_data => libxl_psr_cat_set/get_cbm.
> * Move libxl_count_physical_sockets into seperate patch.
> * Support LIBXL_PSR_TARGET_ALL for libxl_psr_cat_set_cbm.
> * Clean up the print codes.
> ---
>  docs/man/xl.pod.1             |  31 +++++++++
>  tools/libxc/include/xenctrl.h |  15 +++++
>  tools/libxc/xc_psr.c          |  74 +++++++++++++++++++++
>  tools/libxl/libxl.h           |  20 ++++++
>  tools/libxl/libxl_psr.c       | 126 ++++++++++++++++++++++++++++++++++--
>  tools/libxl/libxl_types.idl   |   5 ++
>  tools/libxl/xl.h              |   4 ++
>  tools/libxl/xl_cmdimpl.c      | 146 ++++++++++++++++++++++++++++++++++++++++++
>  tools/libxl/xl_cmdtable.c     |  12 ++++
>  9 files changed, 426 insertions(+), 7 deletions(-)
> 
> diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
> index b016272..99979a5 100644
> --- a/docs/man/xl.pod.1
> +++ b/docs/man/xl.pod.1
> @@ -1492,6 +1492,37 @@ monitor types are:
>  
>  =back
>  
> +=head1 CACHE ALLOCATION TECHNOLOGY
> +
> +Intel Broadwell and later server platforms offer capabilities to configure and
> +make use of the Cache Allocation Technology (CAT) mechanisms, which enable more
> +cache resources (i.e. L3 cache) to be made available for high priority
> +applications. In Xen implementation, CAT is used to control cache allocation
> +on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
> +(CBM) for the domain.
> +
> +=over 4
> +
> +=item B<psr-cat-cbm-set> [I<OPTIONS>] [I<domain-id>]
> +
> +Set cache capacity bitmasks(CBM) for a domain.

What is the syntax of these bitmaps, and where do I pass them?

I think there is also a bunch of terminology (CBM, COS) which need
explaining, otherwise no one will know how to use it. Perhaps that
belongs in a separate document or the wiki though?

> diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
> index e367a80..6c670d5 100644
> --- a/tools/libxc/xc_psr.c
> +++ b/tools/libxc/xc_psr.c
> @@ -248,6 +248,80 @@ int xc_psr_cmt_enabled(xc_interface *xch)
>  
>      return 0;
>  }
> +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
> +                               xc_psr_cat_type type, uint32_t target,
> +                               uint64_t data)
> +{
> +    DECLARE_DOMCTL;
> +    uint32_t cmd;
> +
> +    switch ( type )
> +    {
> +    case XC_PSR_CAT_L3_CBM:
> +        cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
> +        break;
> +    default:
> +        return -1;

You should also set errno to an appropriate value, since calling code
will try and use it to log with.

There were several instances of this I think.

> +    }


> @@ -1513,6 +1520,19 @@ int libxl_psr_cmt_get_sample(libxl_ctx *ctx,
>                               uint64_t *tsc_r);
>  #endif
>  
> +#ifdef LIBXL_HAVE_PSR_CAT
> +
> +#define LIBXL_PSR_TARGET_ALL (~0U)
> +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
> +                          libxl_psr_cat_type type, uint32_t target,
> +                          uint64_t cbm);
> +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
> +                          libxl_psr_cat_type type, uint32_t target,
> +                          uint64_t *cbm_r);

What are the units of the various cbm*

If they are now more precisely typed (i.e. not the opaque data from last
time) then is the type parameter still needed?

> +int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, uint32_t socket,
> +                              uint32_t *cos_max_r, uint32_t *cbm_len_r);

Is there going to be any user documentation regarding what cos and cbm
are and how to interpret them and set them?

> @@ -247,6 +290,75 @@ out:
>      return rc;
>  }
>  
> +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
> +                          libxl_psr_cat_type type, uint32_t target,
> +                          uint64_t cbm)
> +{
> +    GC_INIT(ctx);
> +    int rc;
> +
> +    uint32_t i, nr_sockets;
> +
> +    if (target != LIBXL_PSR_TARGET_ALL) {
> +        rc = xc_psr_cat_set_domain_data(ctx->xch, domid, type, target, cbm);
> +        if (rc < 0) {
> +            libxl__psr_cat_log_err_msg(gc, errno);
> +            rc = ERROR_FAIL;
> +        }
> +    } else {
> +        nr_sockets = libxl_count_physical_sockets(ctx);
> +        if (nr_sockets == 0) {
> +            LOGE(ERROR, "failed to get system socket count");
> +            rc = ERROR_FAIL;
> +            goto out;
> +        }
> +        for (i = 0; i < nr_sockets; i++) {
> +            rc = xc_psr_cat_set_domain_data(ctx->xch, domid, type, i, cbm);
> +            if (rc < 0) {
> +                libxl__psr_cat_log_err_msg(gc, errno);
> +                rc = ERROR_FAIL;

You neither error out nor latch this value, so you only return the
status of the final socket.

I think you probably want to exit on first error. And depending on the
semantics of this stuff you may also need to unwind any work already
done.

> diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
> index f26a02d..ff963df 100644
> --- a/tools/libxl/xl_cmdimpl.c
> +++ b/tools/libxl/xl_cmdimpl.c
> @@ -8038,6 +8038,152 @@ int main_psr_cmt_show(int argc, char **argv)
>  }
>  #endif
>  
> +#ifdef LIBXL_HAVE_PSR_CAT
> +static void psr_cat_print_one_domain_cbm(libxl_dominfo *dominfo,
> +                                         uint32_t socket)
> +{
> +    char *domain_name;
> +    uint64_t cbm;
> +
> +    domain_name = libxl_domid_to_name(ctx, dominfo->domid);

AFAICT you use dominfo here for domifo->domid, but you looked it up with
libxl_domain_info(,... domid) so you already had that in your hand.

IOW I think you can just pass the domid to this function, and omit the
call to libxl_domain_info in the callers, or use list->domid in the case
you have that in hand.

> +    printf("%5d%25s", dominfo->domid, domain_name);
> +    free(domain_name);
> +
> +    if (!libxl_psr_cat_get_cbm(ctx, dominfo->domid,
> +                               LIBXL_PSR_CAT_TYPE_L3_CBM, socket, &cbm))
> +         printf("%#16"PRIx64, cbm);
> +
> +    printf("\n");
> +}
> +
> +static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socket)
> +{
> +    int i, nr_domains;
> +
> +    if (domid != INVALID_DOMID) {
> +        libxl_dominfo dominfo;
> +
> +        if (libxl_domain_info(ctx, &dominfo, domid)) {
> +            fprintf(stderr, "Failed to get domain info for %d\n", domid);
> +            return -1;
> +        }
> +        psr_cat_print_one_domain_cbm(&dominfo, socket);
> +    } else {
> +        libxl_dominfo *list;
> +
> +        if (!(list = libxl_list_domain(ctx, &nr_domains))) {
> +            fprintf(stderr, "Failed to get domain info for domain list.\n");

"...for cbm something something."

> +            return -1;
> +        }
> +        for (i = 0; i < nr_domains; i++)
> +            psr_cat_print_one_domain_cbm(list + i, socket);
> +        libxl_dominfo_list_free(list, nr_domains);
> +    }
> +
> +    return 0;
> +}
> +
> +static int psr_cat_print_socket(uint32_t domid, uint32_t socket)
> +{
> +    uint32_t l3_cache_size, cos_max, cbm_len;
> +    int rc;
> +
> +    rc = libxl_psr_cmt_get_l3_cache_size(ctx, socket, &l3_cache_size);
> +    if (rc) {
> +        fprintf(stderr, "Failed to get l3 cache size for socket:%d\n", socket);
> +        return -1;
> +    }
> +
> +    rc = libxl_psr_cat_get_l3_info(ctx, socket, &cos_max, &cbm_len);

Would an interface which returns a list with information for all sockets
not be more convenient?

> +    if (rc) {
> +        fprintf(stderr, "Failed to get cat info for socket:%d\n", socket);
> +        return -1;
> +    }
> +
> +    /* Header */
> +    printf("SocketID:\t%u\n", socket);
> +    printf("L3 Cache:\t%uKB\n", l3_cache_size);
> +    printf("Maximum COS:\t%u\n", cos_max);
> +    printf("CBM length:\t%u\n", cbm_len);
> +    printf("Default CBM:\t%#"PRIx64"\n", (1ul << cbm_len) - 1);
> +    printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
> +
> +    return psr_cat_print_domain_cbm(domid, socket);
> +}
> +
> +static int psr_cat_show(uint32_t domid)
> +{
> +    uint32_t socket, nr_sockets;
> +    int rc;
> +
> +    nr_sockets = libxl_count_physical_sockets(ctx);
> +    if (nr_sockets == 0) {
> +        fprintf(stderr, "Failed getting physinfo\n");

That wasn't a physinfo call.

Ian.