[PATCH v3 7/7] tools: enable Cache QoS Monitoring feature for libxl/libxc

[dongxiao.xu@intel.com]

From: Dongxiao Xu <dongxiao.xu@intel.com>

Introduced a new config parameter "pqos_cqm", if it is set to 1, guest
will be created with CQM feature enabled.

Introduced two new xl commands to attach/detach CQM service for a guest
$ xl pqos-attach cqm domid
$ xl pqos-detach cqm domid

Introduce one new xl command to retrive guest CQM information
$ xl pqos-list cqm (domid)

Signed-off-by: Jiongxi Li <jiongxi.li@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
 tools/libxc/xc_domain.c     |   48 +++++++++++++++
 tools/libxc/xenctrl.h       |   12 ++++
 tools/libxl/Makefile        |    3 +-
 tools/libxl/libxl.h         |    5 ++
 tools/libxl/libxl_create.c  |    3 +
 tools/libxl/libxl_pqos.c    |  108 +++++++++++++++++++++++++++++++++
 tools/libxl/libxl_types.idl |    1 +
 tools/libxl/xl.h            |    3 +
 tools/libxl/xl_cmdimpl.c    |  138 +++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/xl_cmdtable.c   |   15 +++++
 10 files changed, 335 insertions(+), 1 deletion(-)
 create mode 100644 tools/libxl/libxl_pqos.c

diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index 1ccafc5..6b71559 100644
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -1776,6 +1776,54 @@ int xc_domain_set_max_evtchn(xc_interface *xch, uint32_t domid,
     return do_domctl(xch, &domctl);
 }
 
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, uint32_t flags)
+{
+    DECLARE_DOMCTL;
+    domctl.cmd = XEN_DOMCTL_attach_pqos;
+    domctl.domain = (domid_t)domid;
+    domctl.u.qos_type.flags = flags;
+    return do_domctl(xch, &domctl);
+}
+
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, uint32_t flags)
+{
+    DECLARE_DOMCTL;
+    domctl.cmd = XEN_DOMCTL_detach_pqos;
+    domctl.domain = (domid_t)domid;
+    domctl.u.qos_type.flags = flags;
+    return do_domctl(xch, &domctl);
+}
+
+int xc_domain_getcqminfolist(xc_interface *xch, sysctl_cqminfo_t *info)
+{
+    int ret = 0;
+    xen_socket_cqmdata_t *data = info->cqmdata;
+    DECLARE_SYSCTL;
+
+    DECLARE_HYPERCALL_BOUNCE(data,
+        info->num_rmid * info->num_sockets * sizeof(*data),
+        XC_HYPERCALL_BUFFER_BOUNCE_OUT);
+
+    if ( xc_hypercall_bounce_pre(xch, data) )
+        return -1;
+
+    sysctl.cmd = XEN_SYSCTL_getcqminfo;
+    set_xen_guest_handle(sysctl.u.getcqminfo.buffer, data);
+
+    if ( xc_sysctl(xch, &sysctl) < 0 )
+        ret = -1;
+    else
+    {
+        info->num_sockets = sysctl.u.getcqminfo.num_sockets;
+        info->num_rmid = sysctl.u.getcqminfo.num_rmid;
+        info->num_rmid_avail = sysctl.u.getcqminfo.num_rmid_avail;
+    }
+
+    xc_hypercall_bounce_post(xch, data);
+
+    return ret;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index 4ac6b8a..894e186 100644
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -2395,4 +2395,16 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch,
  */
 int xc_kexec_unload(xc_interface *xch, int type);
 
+struct xc_sysctl_getcqminfo
+{
+    uint32_t num_rmid;
+    uint32_t num_rmid_avail;
+    uint32_t num_sockets;
+    xen_socket_cqmdata_t *cqmdata;
+};
+typedef struct xc_sysctl_getcqminfo sysctl_cqminfo_t;
+
+int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, uint32_t flags);
+int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, uint32_t flags);
+int xc_domain_getcqminfolist(xc_interface *xch, sysctl_cqminfo_t *info);
 #endif /* XENCTRL_H */
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index cf214bb..35f0b97 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -74,7 +74,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_internal.o libxl_utils.o libxl_uuid.o \
 			libxl_json.o libxl_aoutils.o libxl_numa.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
-			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
+			libxl_qmp.o libxl_event.o libxl_fork.o libxl_pqos.o \
+			$(LIBXL_OBJS-y)
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
 $(LIBXL_OBJS): CFLAGS += $(CFLAGS_LIBXL) -include $(XEN_ROOT)/tools/config.h
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index c7dceda..fdca92d 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -285,6 +285,7 @@
 
 #include <libxl_uuid.h>
 #include <_libxl_list.h>
+#include <xenctrl.h>
 
 /* API compatibility. */
 #ifdef LIBXL_API_VERSION
@@ -1051,6 +1052,10 @@ int libxl_flask_getenforce(libxl_ctx *ctx);
 int libxl_flask_setenforce(libxl_ctx *ctx, int mode);
 int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size);
 
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qos_type);
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qos_type);
+int libxl_get_cqm_info(libxl_ctx *ctx, sysctl_cqminfo_t *info);
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 5e9cdcc..454c69d 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -40,6 +40,8 @@ int libxl__domain_create_info_setdefault(libxl__gc *gc,
 
     libxl_defbool_setdefault(&c_info->run_hotplug_scripts, true);
 
+    libxl_defbool_setdefault(&c_info->pqos_cqm, false);
+
     return 0;
 }
 
@@ -454,6 +456,7 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_create_info *info,
         }
         flags |= XEN_DOMCTL_CDF_hap;
     }
+    flags |= libxl_defbool_val(info->pqos_cqm) ? XEN_DOMCTL_CDF_pqos_cqm : 0;
     *domid = -1;
 
     /* Ultimately, handle is an array of 16 uint8_t, same as uuid */
diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c
new file mode 100644
index 0000000..bf7593a
--- /dev/null
+++ b/tools/libxl/libxl_pqos.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2013      Intel Corporation
+ * Author Jiongxi Li <jiongxi.li@intel.com>
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+#include "libxl_internal.h"
+
+static const char * const msg[] = {
+    [EINVAL] = "invalid QoS resource type! Supported types: \"cqm\"",
+    [ENODEV] = "CQM is not supported in this system.",
+    [EEXIST] = "CQM is already attached to this domain.",
+    [ENOENT] = "CQM is not attached to this domain.",
+    [EUSERS] = "there is no free CQM RMID available.",
+    [ESRCH]  = "is this Domain ID valid?",
+};
+
+int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qos_type)
+{
+    int rc;
+    uint32_t flags = 0;
+
+    if (!strncmp(qos_type, "cqm", 3))
+        flags |= XEN_DOMCTL_pqos_cqm;
+    else {
+        rc = -EINVAL;
+        LIBXL__LOG(ctx, XTL_ERROR, "%s", msg[EINVAL]);
+        return rc;
+    }
+
+    rc = xc_domain_pqos_attach(ctx->xch, domid, flags);
+    if (rc < 0) {
+        switch(errno) {
+        case EINVAL:
+        case ENODEV:
+        case EEXIST:
+        case EUSERS:
+        case ESRCH:
+            LIBXL__LOG(ctx, XTL_ERROR, "%s", msg[errno]);
+            break;
+        default:
+            LIBXL__LOG(ctx, XTL_ERROR, "errno: %d", errno);
+        }
+    }
+
+    return rc;
+}
+
+int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qos_type)
+{
+    int rc;
+    uint32_t flags = 0;
+
+    if (!strncmp(qos_type, "cqm", 3))
+        flags |= XEN_DOMCTL_pqos_cqm;
+    else {
+        rc = -EINVAL;
+        LIBXL__LOG(ctx, XTL_ERROR, "%s", msg[EINVAL]);
+        return rc;
+    }
+
+    rc = xc_domain_pqos_detach(ctx->xch, domid, flags);
+    if (rc < 0) {
+        switch(errno) {
+        case EINVAL:
+        case ENODEV:
+        case ENOENT:
+        case ESRCH:
+            LIBXL__LOG(ctx, XTL_ERROR, "%s", msg[errno]);
+            break;
+        default:
+            LIBXL__LOG(ctx, XTL_ERROR, "errno: %d", errno);
+        }
+    }
+
+    return rc;
+}
+
+int libxl_get_cqm_info(libxl_ctx *ctx,
+                       sysctl_cqminfo_t *info)
+{
+    int ret;
+
+    ret = xc_domain_getcqminfolist(ctx->xch, info);
+    if (ret < 0)
+        return -EINVAL;
+
+    return ret;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index de5bac3..22688d8 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -275,6 +275,7 @@ libxl_domain_create_info = Struct("domain_create_info",[
     ("poolid",       uint32),
     ("run_hotplug_scripts",libxl_defbool),
     ("pvh",          libxl_defbool),
+    ("pqos_cqm",     libxl_defbool),
     ], dir=DIR_IN)
 
 libxl_domain_restore_params = Struct("domain_restore_params", [
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index e005c39..994d3be 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -105,6 +105,9 @@ int main_getenforce(int argc, char **argv);
 int main_setenforce(int argc, char **argv);
 int main_loadpolicy(int argc, char **argv);
 int main_remus(int argc, char **argv);
+int main_pqosattach(int argc, char **argv);
+int main_pqosdetach(int argc, char **argv);
+int main_pqoslist(int argc, char **argv);
 
 void help(const char *command);
 
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 8690ec7..cc2a095 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -670,6 +670,8 @@ static void parse_config_data(const char *config_source,
         exit(1);
     }
 
+    xlu_cfg_get_defbool(config, "pqos_cqm", &c_info->pqos_cqm, 0);
+
     libxl_domain_build_info_init_type(b_info, c_info->type);
     if (blkdev_start)
         b_info->blkdev_start = strdup(blkdev_start);
@@ -7193,6 +7195,142 @@ int main_remus(int argc, char **argv)
     return -ERROR_FAIL;
 }
 
+int main_pqosattach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, rc;
+    const char *qos_type = NULL;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-attach", 2) {
+        /* No options */
+    }
+
+    qos_type = argv[optind];
+    domid = find_domain(argv[optind + 1]);
+
+    rc = libxl_pqos_attach(ctx, domid, qos_type);
+
+    return rc;
+}
+
+int main_pqosdetach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, rc;
+    const char *qos_type = NULL;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-detach", 2) {
+        /* No options */
+    }
+
+    qos_type = argv[optind];
+    domid = find_domain(argv[optind + 1]);
+
+    rc = libxl_pqos_detach(ctx, domid, qos_type);
+
+    return rc;
+}
+
+static void print_cqm_info(const sysctl_cqminfo_t *info, uint32_t first_domain,
+                           unsigned int num_domains)
+{
+    unsigned long i, j;
+    xen_socket_cqmdata_t *cqmdata;
+    char *domname;
+    int found = 0;
+
+    if (info->num_rmid == 0)
+        printf("System doesn't supoort CQM.\n");
+    else if (info->num_rmid - info->num_rmid_avail == 1)
+        printf("No RMID is assigned to domains.\n");
+    else {
+        printf("RMID count %5d\tRMID available %5d\n",
+               info->num_rmid, info->num_rmid_avail);
+        printf("Name                                        ID");
+        for (i = 0; i < info->num_sockets; i++)
+            printf("\tSocketID\tL3C_Usage");
+        for (i = first_domain; i < (first_domain + num_domains); i++) {
+            found = 0;
+            for (j = 0; j < (info->num_rmid * info->num_sockets); j++) {
+                cqmdata = info->cqmdata + j;
+                if (!cqmdata->valid || cqmdata->domid != i)
+                    continue;
+                if (!found) {
+                    domname = libxl_domid_to_name(ctx, cqmdata->domid);
+                    printf("\n%-40s %5d", domname, cqmdata->domid);
+                    free(domname);
+                    found = 1;
+                }
+                printf("%10u %16lu     ", cqmdata->socket, cqmdata->l3c_occupancy);
+            }
+        }
+        printf("\n");
+    }
+}
+
+int main_pqoslist(int argc, char **argv)
+{
+    int opt;
+    const char *qos_type = NULL;
+    uint32_t first_domain;
+    unsigned int num_domains;
+    int rc = 0;
+    sysctl_cqminfo_t info;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-list", 1) {
+        /* No options */
+    }
+
+    qos_type = argv[optind];
+
+    if (!strncmp(qos_type, "cqm", 3)) {
+        if (optind + 1 >= argc) {
+            first_domain = 0;
+            num_domains = 1024;
+        } else if (optind + 1 == argc - 1) {
+            first_domain = find_domain(argv[optind + 1]);
+            num_domains = 1;
+            if (!libxl_domid_to_name(ctx, first_domain))
+            {
+                fprintf(stderr, "Invalid domain id: %d.\n", first_domain);
+                return 1;
+            }
+        } else {
+            help("pqos-list");
+            return 2;
+        }
+
+        info.num_rmid= 256;
+        info.num_sockets = 128;
+        info.cqmdata = calloc(info.num_rmid * info.num_sockets,
+                              sizeof(xen_socket_cqmdata_t));
+        if (!info.cqmdata) {
+            fprintf(stderr, "Allocating domain cqminfo failed.\n");
+            return ERROR_FAIL;
+        }
+
+        rc = libxl_get_cqm_info(ctx, &info);
+
+        if (rc < 0) {
+            fprintf(stderr, "Failed to get domain CQM info, "
+                            "check whether CQM feature is supported.\n");
+            if (info.cqmdata)
+                free(info.cqmdata);
+            return 1;
+        }
+        print_cqm_info(&info, first_domain, num_domains);
+
+        if (info.cqmdata)
+            free(info.cqmdata);
+    } else {
+        fprintf(stderr, "QoS resource type supported is: cqm.\n");
+        help("pqos-list");
+        return 2;
+    }
+
+    return 0;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 326a660..6ced416 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -488,6 +488,21 @@ struct cmd_spec cmd_table[] = {
       "                        of the domain."
 
     },
+    { "pqos-attach",
+      &main_pqosattach, 0, 1,
+      "Allocate and map qos resource",
+      "<Resource> <Domain>",
+    },
+    { "pqos-detach",
+      &main_pqosdetach, 0, 1,
+      "Reliquish qos resource",
+      "<Resource> <Domain>",
+    },
+    { "pqos-list",
+      &main_pqoslist, 0, 0,
+      "List qos information about all/some domains",
+      "<Resource> [Domain]",
+    },
 };
 
 int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
-- 
1.7.9.5