[PATCH v10 6/6] tools: enable Cache QoS Monitoring feature for libxl/libxc

[Dongxiao Xu <dongxiao.xu@intel.com>]

Introduced two new xl commands to attach/detach CQM service for a guest
$ xl pqos-attach cqm domid
$ xl pqos-detach cqm domid

Introduce one new xl command to retrieve guest CQM information
$ xl pqos-list cqm

Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
---
 docs/man/xl.pod.1           |   23 ++++++
 tools/libxc/xc_domain.c     |   41 ++++++++++
 tools/libxc/xenctrl.h       |   13 +++
 tools/libxl/Makefile        |    3 +-
 tools/libxl/libxl.h         |   12 +++
 tools/libxl/libxl_pqos.c    |  190 +++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_types.idl |    9 ++
 tools/libxl/xl.h            |    3 +
 tools/libxl/xl_cmdimpl.c    |  109 +++++++++++++++++++++++++
 tools/libxl/xl_cmdtable.c   |   15 ++++
 10 files changed, 417 insertions(+), 1 deletion(-)
 create mode 100644 tools/libxl/libxl_pqos.c

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f7ceaa8..daa191a 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1340,6 +1340,29 @@ Load FLASK policy from the given policy file. The initial policy is provided to
 the hypervisor as a multiboot module; this command allows runtime updates to the
 policy. Loading new security policy will reset runtime changes to device labels.
 
+=head1 PLATFORM QOS
+
+New Intel processor may offer monitoring capability in each logical processor to
+measure specific quality-of-service metric, for example, Cache QoS Monitoring to
+get L3 cache occupancy.
+
+=over 4
+
+=item B<pqos-attach> [I<qos-type>] [I<domain-id>]
+
+Attach certain platform QoS service for a domain.
+Current supported I<qos-type> is: "cqm".
+
+=item B<pqos-detach> [I<qos-type>] [I<domain-id>]
+
+Detach certain platform QoS service from a domain.
+Current supported I<qos-type> is: "cqm".
+
+=item B<pqos-list> [I<qos-type>]
+
+List platform QoS information for QoS attached domains.
+Current supported I<qos-type> is: "cqm".
+
 =back
 
 =head1 TO BE DOCUMENTED
diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index 369c3f3..80f0458 100644
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -1814,6 +1814,47 @@ int xc_domain_set_max_evtchn(xc_interface *xch, uint32_t domid,
     return do_domctl(xch, &domctl);
 }
 
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, uint64_t flags)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_attach_pqos;
+    domctl.domain = (domid_t)domid;
+    domctl.u.qos_type.flags = flags;
+
+    return do_domctl(xch, &domctl);
+}
+
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, uint64_t flags)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_detach_pqos;
+    domctl.domain = (domid_t)domid;
+    domctl.u.qos_type.flags = flags;
+
+    return do_domctl(xch, &domctl);
+}
+
+int xc_domain_getcqminfo(xc_interface *xch, xc_cqminfo_t *info)
+{
+    int ret;
+    DECLARE_SYSCTL;
+
+    sysctl.cmd = XEN_SYSCTL_getcqminfo;
+    ret = xc_sysctl(xch, &sysctl);
+    if ( ret >= 0 )
+    {
+        info->socket_l3c_mfn = sysctl.u.getcqminfo.socket_l3c_mfn;
+        info->rmid_dom_mfn = sysctl.u.getcqminfo.rmid_dom_mfn;
+        info->nr_rmids = sysctl.u.getcqminfo.nr_rmids;
+        info->nr_sockets = sysctl.u.getcqminfo.nr_sockets;
+        info->l3c_total = sysctl.u.getcqminfo.l3c_total;
+    }
+
+    return ret;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index e3a32f2..f1e8b9b 100644
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -2424,4 +2424,17 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch,
  */
 int xc_kexec_unload(xc_interface *xch, int type);
 
+struct xc_cqminfo
+{
+    uint64_aligned_t socket_l3c_mfn;
+    uint64_aligned_t rmid_dom_mfn;
+    uint32_t nr_rmids;
+    uint32_t nr_sockets;
+    uint32_t l3c_total;
+};
+typedef struct xc_cqminfo xc_cqminfo_t;
+
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, uint64_t flags);
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, uint64_t flags);
+int xc_domain_getcqminfo(xc_interface *xch, xc_cqminfo_t *info);
 #endif /* XENCTRL_H */
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 755b666..7b54720 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -76,7 +76,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_internal.o libxl_utils.o libxl_uuid.o \
 			libxl_json.o libxl_aoutils.o libxl_numa.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
-			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
+			libxl_qmp.o libxl_event.o libxl_fork.o libxl_pqos.o \
+			$(LIBXL_OBJS-y)
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
 LIBXL_TESTS += timedereg
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index b2c3015..b330633 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -467,6 +467,13 @@ typedef uint8_t libxl_mac[6];
 #define LIBXL_MAC_FMTLEN ((2*6)+5) /* 6 hex bytes plus 5 colons */
 #define LIBXL_MAC_BYTES(mac) mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
 
+/*
+ * LIBXL_HAVE_PQOS_CQM
+ *
+ * If this is defined, the CQM feature is suported.
+ */
+#define LIBXL_HAVE_PQOS_CQM 1
+
 typedef char **libxl_string_list;
 void libxl_string_list_dispose(libxl_string_list *sl);
 int libxl_string_list_length(const libxl_string_list *sl);
@@ -1151,6 +1158,11 @@ int libxl_flask_getenforce(libxl_ctx *ctx);
 int libxl_flask_setenforce(libxl_ctx *ctx, int mode);
 int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size);
 
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, libxl_pqos_type qos_type);
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, libxl_pqos_type qos_type);
+libxl_cqminfo * libxl_getcqminfo(libxl_ctx *ctx, unsigned int *l3c_total,
+    unsigned int *nr_domains, unsigned int *nr_rmids, unsigned int *nr_sockets);
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c
new file mode 100644
index 0000000..d26aa4c
--- /dev/null
+++ b/tools/libxl/libxl_pqos.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2014      Intel Corporation
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ * Author Jiongxi Li <jiongxi.li@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+#include "libxl_internal.h"
+
+static const char * const msg[] = {
+    [EINVAL] = "invalid QoS resource type! Supported types: \"cqm\"",
+    [ENODEV] = "CQM is not supported in this system.",
+    [EEXIST] = "CQM is already attached to this domain.",
+    [ENOENT] = "CQM is not attached to this domain.",
+    [EUSERS] = "there is no free CQM RMID available.",
+    [ESRCH]  = "is this Domain ID valid?",
+};
+
+static void libxl_pqos_err_msg(libxl_ctx *ctx, int err)
+{
+    GC_INIT(ctx);
+
+    switch (err) {
+    case EINVAL:
+    case ENODEV:
+    case EEXIST:
+    case EUSERS:
+    case ESRCH:
+    case ENOENT:
+        LOGE(ERROR, "%s", msg[err]);
+        break;
+    default:
+        LOGE(ERROR, "errno: %d", err);
+    }
+
+    GC_FREE;
+}
+
+static int libxl_pqos_type2flags(libxl_pqos_type qos_type, uint64_t *flags)
+{
+    int rc = 0;
+
+    if (qos_type == LIBXL_PQOS_TYPE_CQM)
+        *flags |= XEN_DOMCTL_pqos_cqm;
+    else
+        rc = -1;
+
+    return rc;
+}
+
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, libxl_pqos_type qos_type)
+{
+    int rc;
+    uint64_t flags = 0;
+
+    rc = libxl_pqos_type2flags(qos_type, &flags);
+    if (rc < 0) {
+        libxl_pqos_err_msg(ctx, EINVAL);
+        return ERROR_FAIL;
+    }
+
+    rc = xc_domain_pqos_attach(ctx->xch, domid, flags);
+    if (rc < 0) {
+        libxl_pqos_err_msg(ctx, errno);
+        return ERROR_FAIL;
+    }
+
+    return 0;
+}
+
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, libxl_pqos_type qos_type)
+{
+    int rc;
+    uint64_t flags = 0;
+
+    rc = libxl_pqos_type2flags(qos_type, &flags);
+    if (rc < 0) {
+        libxl_pqos_err_msg(ctx, EINVAL);
+        return ERROR_FAIL;
+    }
+
+    rc = xc_domain_pqos_detach(ctx->xch, domid, flags);
+    if (rc < 0) {
+        libxl_pqos_err_msg(ctx, errno);
+        return ERROR_FAIL;
+    }
+
+    return 0;
+}
+
+libxl_cqminfo * libxl_getcqminfo(libxl_ctx *ctx, unsigned int *l3c_total,
+    unsigned int *nr_domains, unsigned int *nr_rmids, unsigned int *nr_sockets)
+{
+    libxl_cqminfo *xl_cqminfo;
+    unsigned long *socket_l3c, *l3c;
+    uint16_t *rmid_to_dom;
+    xc_cqminfo_t xcinfo;
+    int i, j, idx;
+    unsigned int nr_rmids_size, nr_sockets_size;
+    GC_INIT(ctx);
+
+    if (xc_domain_getcqminfo(ctx->xch, &xcinfo) < 0) {
+        LOGE(ERROR, "getting domain cqm info");
+        goto err;
+    }
+
+    rmid_to_dom = xc_map_foreign_range(ctx->xch, DOMID_XEN,
+                      XC_PAGE_SIZE, PROT_READ, xcinfo.rmid_dom_mfn);
+    if (rmid_to_dom == 0) {
+        LOGE(ERROR, "Failed to map cqm rmid_to_dom buffers");
+        goto err;
+    }
+
+    *l3c_total = xcinfo.l3c_total;
+    *nr_sockets = xcinfo.nr_sockets;
+    *nr_rmids = xcinfo.nr_rmids;
+    *nr_domains = 1024;
+
+    nr_sockets_size = xcinfo.nr_sockets * sizeof(unsigned long);
+    socket_l3c = xc_map_foreign_range(ctx->xch, DOMID_XEN,
+                     nr_sockets_size, PROT_READ, xcinfo.socket_l3c_mfn);
+    if (socket_l3c == 0) {
+        LOGE(ERROR, "Failed to map cqm rmid_to_dom buffers");
+        munmap(rmid_to_dom, XC_PAGE_SIZE);
+        goto err;
+    }
+
+    xl_cqminfo = calloc(*nr_domains * xcinfo.nr_sockets,
+                        sizeof(libxl_cqminfo));
+    if (!xl_cqminfo) {
+        LOGE(ERROR, "allocating domain cqm data structure");
+        munmap(socket_l3c, nr_sockets_size);
+        munmap(rmid_to_dom, XC_PAGE_SIZE);
+        goto err;
+    }
+
+    for (i = 0; i < xcinfo.nr_sockets; i++) {
+        nr_rmids_size = xcinfo.nr_rmids * sizeof(unsigned long);
+        l3c = xc_map_foreign_range(ctx->xch, DOMID_XEN,
+                  nr_rmids_size, PROT_READ, socket_l3c[i]);
+        if (l3c == 0) {
+            LOGE(ERROR, "Failed to map cqm rmid_to_dom buffers");
+            free(xl_cqminfo);
+            munmap(socket_l3c, nr_sockets_size);
+            munmap(rmid_to_dom, XC_PAGE_SIZE);
+            goto err;
+        }
+
+        for (j = 0; j < xcinfo.nr_rmids; j++) {
+            if (rmid_to_dom[j] > *nr_domains)
+                continue;
+            idx = i + rmid_to_dom[j] * xcinfo.nr_sockets;
+            xl_cqminfo[idx].valid = 1;
+            xl_cqminfo[idx].l3c = l3c[j];
+        }
+        munmap(l3c, nr_rmids_size);
+    }
+
+    munmap(socket_l3c, nr_sockets_size);
+    munmap(rmid_to_dom, XC_PAGE_SIZE);
+    GC_FREE;
+
+    return xl_cqminfo;
+
+err:
+    *nr_domains = 0;
+    *nr_rmids = 0;
+    *nr_sockets = 0;
+    GC_FREE;
+
+    return NULL;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 612645c..73b9ab3 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -598,3 +598,12 @@ libxl_event = Struct("event",[
                                  ])),
            ("domain_create_console_available", Struct(None, [])),
            ]))])
+
+libxl_pqos_type = Enumeration("pqos_type", [
+    (1, "CQM"),
+    ])
+
+libxl_cqminfo = Struct("cqminfo", [
+    ("valid",         uint32),
+    ("l3c",           uint64),
+    ])
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 10a2e66..9b4f668 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -110,6 +110,9 @@ int main_loadpolicy(int argc, char **argv);
 int main_remus(int argc, char **argv);
 #endif
 int main_devd(int argc, char **argv);
+int main_pqosattach(int argc, char **argv);
+int main_pqosdetach(int argc, char **argv);
+int main_pqoslist(int argc, char **argv);
 
 void help(const char *command);
 
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 8389468..664a34d 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -7381,6 +7381,115 @@ out:
     return ret;
 }
 
+int main_pqosattach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, rc;
+    libxl_pqos_type qos_type;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-attach", 2) {
+        /* No options */
+    }
+
+    /* libxl_pqos_attach will handle the parameter check. */
+    libxl_pqos_type_from_string(argv[optind], &qos_type);
+    domid = find_domain(argv[optind + 1]);
+
+    rc = libxl_pqos_attach(ctx, domid, qos_type);
+
+    return rc;
+}
+
+int main_pqosdetach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, rc;
+    libxl_pqos_type qos_type;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-detach", 2) {
+        /* No options */
+    }
+
+    /* libxl_pqos_detach will handle the parameter check. */
+    libxl_pqos_type_from_string(argv[optind], &qos_type);
+    domid = find_domain(argv[optind + 1]);
+
+    rc = libxl_pqos_detach(ctx, domid, qos_type);
+
+    return rc;
+}
+
+static void print_cqm_info(libxl_cqminfo *info, unsigned int l3c_total,
+    unsigned int nr_domains, unsigned int nr_rmids, unsigned int nr_sockets)
+{
+    unsigned int i, j;
+    char *domname;
+    int print_header;
+    int used_rmid = 0;
+
+    if (nr_rmids == 0) {
+        printf("System doesn't support CQM.\n");
+        return;
+    }
+
+    print_header = 1;
+    for (i = 0; i < nr_domains; i++) {
+        if (!info[i * nr_sockets].valid)
+            continue;
+        if (print_header) {
+            printf("Name                                        ID");
+            for (j = 0; j < nr_sockets; j++)
+                printf("\tSocketID\tL3C_Usage");
+            print_header = 0;
+        }
+
+        domname = libxl_domid_to_name(ctx, i);
+        printf("\n%-40s %5d", domname, i);
+        free(domname);
+        for (j = 0; j < nr_sockets; j++) {
+            printf("%10u %13lu KB     ", j, info[i * nr_sockets + j].l3c/1024);
+        }
+        used_rmid++;
+    }
+    if (!used_rmid)
+        printf("No RMID is assigned to domains.\n");
+    else
+        printf("\n");
+
+    printf("\nSocket L3 Cache Size: %d KB\n", l3c_total);
+    printf("RMID count %5d\tRMID available %5d\n",
+           nr_rmids, nr_rmids - used_rmid - 1);
+}
+
+int main_pqoslist(int argc, char **argv)
+{
+    int opt;
+    const char *qos_type = NULL;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-list", 1) {
+        /* No options */
+    }
+
+    qos_type = argv[optind];
+
+    if (!strcmp(qos_type, "cqm")) {
+        libxl_cqminfo *info;
+        unsigned int l3c_total, nr_domains, nr_rmids, nr_sockets;
+        info = libxl_getcqminfo(ctx, &l3c_total,
+                                &nr_domains, &nr_rmids, &nr_sockets);
+        if (!info)
+            return 1;
+        print_cqm_info(info, l3c_total, nr_domains, nr_rmids, nr_sockets);
+        free(info);
+    } else {
+        fprintf(stderr, "QoS resource type supported is: cqm.\n");
+        help("pqos-list");
+        return 2;
+    }
+
+    return 0;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index e8ab93a..0c45427 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -497,6 +497,21 @@ struct cmd_spec cmd_table[] = {
       "[options]",
       "-F                      Run in the foreground",
     },
+    { "pqos-attach",
+      &main_pqosattach, 0, 1,
+      "Allocate and map qos resources",
+      "<Resource> <Domain>",
+    },
+    { "pqos-detach",
+      &main_pqosdetach, 0, 1,
+      "Relinquish qos resources",
+      "<Resource> <Domain>",
+    },
+    { "pqos-list",
+      &main_pqoslist, 0, 0,
+      "List qos information for all domains",
+      "<Resource>",
+    },
 };
 
 int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
-- 
1.7.9.5