[PATCH v8 08/13] xl: enable getting and setting soft affinity

[Dario Faggioli <dario.faggioli@citrix.com>]

Getting happens via `xl vcpu-list', which now looks like this:

 # xl vcpu-list -s
 Name       ID VCPU CPU State Time(s) Affinity (Hard / Soft)
 Domain-0   0     0  11  -b-     5.4  8-15 / all
 Domain-0   0     1  11  -b-     1.0  8-15 / all
 Domain-0   0    14  13  -b-     1.4  8-15 / all
 Domain-0   0    15   8  -b-     1.6  8-15 / all
 vm-test    3     0   4  -b-     2.5  0-12 / 0-7
 vm-test    3     1   0  -b-     3.2  0-12 / 0-7

Setting happens by specifying two pCPU masks to the `xl vcpu-pin'
command, the first one will be hard affinity, the second soft
affinity. If only one mask is specified, it is only hard affinity
that is affected. To change only soft affinity, '-' can be used
as the hard affinity mask parameter, and it will be left alone.

xl manual page is updated accordingly.

Signed-off-by: Dario Faggioli <dario.faggioli@citrix.com>
---
Changes from v7:
 * 'const char *' used instead of #define/#undef for string
   constants, as requested during review;
 * more consistently use the 'hard' and 'soft' pointers rather
   than the actual cpumap variables, as suggested during review;
 * bitmap holding hard affinity renamed from 'cpumap' to
   'cpumap_hard'.

Changes from v6:
 * removed a remnant from previous iteration, when `vcpu-pin'
   was tacking a porameter;
 * tried to simplify parsing inside of `xl vcpu-pin', as suggested
   during review. Could not take the exact approach described in the
   emails, but still did my best for making this more clear and easy
   to read;
 * fixed a few typos and leftovers.

Changes from v5:
 * change command line interface for 'vcpu-pin', as suggested during
   review.

Changes from v4:
 * fix and rephrased the manual entry, as suggested during review;
 * more refactoring to remove some leftover special casing, as
   suggested during review.

Changes from v3:
 * fix typos in doc, rephrased the help message and changed
   the title of the column for hard/soft affinity, as suggested
   during review.

Changes from v2:
 * this patch folds what in v2 were patches 13 and 14;
 * `xl vcpu-pin' always shows both had and soft affinity,
   without the need of passing '-s'.
---
 docs/man/xl.pod.1         |   32 +++++++++++---
 tools/libxl/xl_cmdimpl.c  |  101 ++++++++++++++++++++++++++++++---------------
 tools/libxl/xl_cmdtable.c |    2 -
 3 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index 30bd4bf..9d1c2a5 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -651,16 +651,32 @@ after B<vcpu-set>, go to B<SEE ALSO> section for information.
 Lists VCPU information for a specific domain.  If no domain is
 specified, VCPU information for all domains will be provided.
 
-=item B<vcpu-pin> I<domain-id> I<vcpu> I<cpus>
+=item B<vcpu-pin> I<domain-id> I<vcpu> I<cpus hard> I<cpus soft>
 
-Pins the VCPU to only run on the specific CPUs.  The keyword
-B<all> can be used to apply the I<cpus> list to all VCPUs in the
-domain.
+Set hard and soft affinity for a I<vcpu> of <domain-id>. Normally VCPUs
+can float between available CPUs whenever Xen deems a different run state
+is appropriate.
 
-Normally VCPUs can float between available CPUs whenever Xen deems a
-different run state is appropriate.  Pinning can be used to restrict
-this, by ensuring certain VCPUs can only run on certain physical
-CPUs.
+Hard affinity can be used to restrict this, by ensuring certain VCPUs
+can only run on certain physical CPUs. Soft affinity specifies a I<preferred>
+set of CPUs. Soft affinity needs special support in the scheduler, which is
+only provided in credit1.
+
+The keyword B<all> can be used to apply the hard and soft affinity masks to
+all the VCPUs in the domain. The symbol '-' can be used to leave either
+hard or soft affinity alone.
+
+For example:
+
+ xl vcpu-pin 0 3 - 6-9
+
+will set soft affinity for vCPU 3 of domain 0 to pCPUs 6,7,8 and 9,
+leaving its hard affinity untouched. On the othe hand:
+
+ xl vcpu-pin 0 3 3,4 6-9
+
+will set both hard and soft affinity, the former to pCPUs 3 and 4, the
+latter to pCPUs 6,7,8, and 9.
 
 =item B<vm-list>
 
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 860ec04..516a4d7 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -656,7 +656,7 @@ static int update_cpumap_range(const char *str, libxl_bitmap *cpumap)
  * single cpus or as eintire NUMA nodes) and turns it into the
  * corresponding libxl_bitmap (in cpumap).
  */
-static int vcpupin_parse(char *cpu, libxl_bitmap *cpumap)
+static int vcpupin_parse(const char *cpu, libxl_bitmap *cpumap)
 {
     char *ptr, *saveptr = NULL;
     int rc = 0;
@@ -4547,8 +4547,10 @@ static void print_vcpuinfo(uint32_t tdomid,
     }
     /*      TIM */
     printf("%9.1f  ", ((float)vcpuinfo->vcpu_time / 1e9));
-    /* CPU AFFINITY */
+    /* CPU HARD AND SOFT AFFINITY */
     print_bitmap(vcpuinfo->cpumap.map, nr_cpus, stdout);
+    printf(" / ");
+    print_bitmap(vcpuinfo->cpumap_soft.map, nr_cpus, stdout);
     printf("\n");
 }
 
@@ -4583,7 +4585,8 @@ static void vcpulist(int argc, char **argv)
     }
 
     printf("%-32s %5s %5s %5s %5s %9s %s\n",
-           "Name", "ID", "VCPU", "CPU", "State", "Time(s)", "CPU Affinity");
+           "Name", "ID", "VCPU", "CPU", "State", "Time(s)",
+           "Affinity (Hard / Soft)");
     if (!argc) {
         if (!(dominfo = libxl_list_domain(ctx, &nb_domain))) {
             fprintf(stderr, "libxl_list_domain failed.\n");
@@ -4616,17 +4619,29 @@ int main_vcpulist(int argc, char **argv)
     return 0;
 }
 
-static int vcpupin(uint32_t domid, const char *vcpu, char *cpu)
+int main_vcpupin(int argc, char **argv)
 {
     libxl_vcpuinfo *vcpuinfo;
-    libxl_bitmap cpumap;
-
-    uint32_t vcpuid;
+    libxl_bitmap cpumap_hard, cpumap_soft;;
+    libxl_bitmap *soft = &cpumap_soft, *hard = &cpumap_hard;
+    uint32_t vcpuid, domid;
+    const char *vcpu, *hard_str, *soft_str;
     char *endptr;
-    int i, nb_cpu, nb_vcpu, rc = -1;
+    int opt, nb_cpu, nb_vcpu, rc = -1;
+
+    libxl_bitmap_init(&cpumap_hard);
+    libxl_bitmap_init(&cpumap_soft);
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "vcpu-pin", 3) {
+        /* No options */
+    }
 
-    libxl_bitmap_init(&cpumap);
+    domid = find_domain(argv[optind]);
+    vcpu = argv[optind+1];
+    hard_str = argv[optind+2];
+    soft_str = (argc > optind+3) ? argv[optind+3] : NULL;
 
+    /* Figure out with which vCPU we are dealing with */
     vcpuid = strtoul(vcpu, &endptr, 10);
     if (vcpu == endptr) {
         if (strcmp(vcpu, "all")) {
@@ -4636,10 +4651,35 @@ static int vcpupin(uint32_t domid, const char *vcpu, char *cpu)
         vcpuid = -1;
     }
 
-    if (libxl_cpu_bitmap_alloc(ctx, &cpumap, 0))
+    if (libxl_cpu_bitmap_alloc(ctx, &cpumap_hard, 0) ||
+        libxl_cpu_bitmap_alloc(ctx, &cpumap_soft, 0))
         goto out;
 
-    if (vcpupin_parse(cpu, &cpumap))
+    /*
+     * Syntax is: xl vcpu-pin <domid> <vcpu> <hard> <soft>
+     * We want to handle all the following cases ('-' means
+     * "leave it alone"):
+     *  xl vcpu-pin 0 3 3,4
+     *  xl vcpu-pin 0 3 3,4 -
+     *  xl vcpu-pin 0 3 - 6-9
+     *  xl vcpu-pin 0 3 3,4 6-9
+     */
+
+    /*
+     * Hard affinity is always present. However, if it's "-", all we need
+     * is passing a NULL pointer to the libxl_set_vcpuaffinity() call below.
+     */
+    if (!strcmp(hard_str, "-"))
+        hard = NULL;
+    else if (vcpupin_parse(hard_str, hard))
+        goto out;
+    /*
+     * Soft affinity is handled similarly. Only difference: we also want
+     * to pass NULL to libxl_set_vcpuaffinity() if it is not specified.
+     */
+    if (argc <= optind+3 || !strcmp(soft_str, "-"))
+        soft = NULL;
+    else if (vcpupin_parse(soft_str, soft))
         goto out;
 
     if (dryrun_only) {
@@ -4650,7 +4690,14 @@ static int vcpupin(uint32_t domid, const char *vcpu, char *cpu)
         }
 
         fprintf(stdout, "cpumap: ");
-        print_bitmap(cpumap.map, nb_cpu, stdout);
+        if (hard)
+            print_bitmap(hard->map, nb_cpu, stdout);
+        else
+            fprintf(stdout, "-");
+        if (soft) {
+            fprintf(stdout, " ");
+            print_bitmap(soft->map, nb_cpu, stdout);
+        }
         fprintf(stdout, "\n");
 
         if (ferror(stdout) || fflush(stdout)) {
@@ -4663,43 +4710,29 @@ static int vcpupin(uint32_t domid, const char *vcpu, char *cpu)
     }
 
     if (vcpuid != -1) {
-        if (libxl_set_vcpuaffinity(ctx, domid, vcpuid, &cpumap, NULL)) {
-            fprintf(stderr, "Could not set affinity for vcpu `%u'.\n", vcpuid);
+        if (libxl_set_vcpuaffinity(ctx, domid, vcpuid, hard, soft)) {
+            fprintf(stderr, "Could not set affinity for vcpu `%u'.\n",
+                    vcpuid);
             goto out;
         }
     }
     else {
-        if (!(vcpuinfo = libxl_list_vcpu(ctx, domid, &nb_vcpu, &i))) {
+        if (!(vcpuinfo = libxl_list_vcpu(ctx, domid, &nb_vcpu, &nb_cpu))) {
             fprintf(stderr, "libxl_list_vcpu failed.\n");
             goto out;
         }
-        for (i = 0; i < nb_vcpu; i++) {
-            if (libxl_set_vcpuaffinity(ctx, domid, vcpuinfo[i].vcpuid,
-                                       &cpumap, NULL)) {
-                fprintf(stderr, "libxl_set_vcpuaffinity failed"
-                                " on vcpu `%u'.\n", vcpuinfo[i].vcpuid);
-            }
-        }
+        if (libxl_set_vcpuaffinity_all(ctx, domid, nb_vcpu, hard, soft))
+            fprintf(stderr, "Could not set affinity.\n");
         libxl_vcpuinfo_list_free(vcpuinfo, nb_vcpu);
     }
 
     rc = 0;
  out:
-    libxl_bitmap_dispose(&cpumap);
+    libxl_bitmap_dispose(&cpumap_soft);
+    libxl_bitmap_dispose(&cpumap_hard);
     return rc;
 }
 
-int main_vcpupin(int argc, char **argv)
-{
-    int opt;
-
-    SWITCH_FOREACH_OPT(opt, "", NULL, "vcpu-pin", 3) {
-        /* No options */
-    }
-
-    return vcpupin(find_domain(argv[optind]), argv[optind+1] , argv[optind+2]);
-}
-
 static void vcpuset(uint32_t domid, const char* nr_vcpus, int check_host)
 {
     char *endptr;
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 4279b9f..7b7fa92 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -218,7 +218,7 @@ struct cmd_spec cmd_table[] = {
     { "vcpu-pin",
       &main_vcpupin, 1, 1,
       "Set which CPUs a VCPU can use",
-      "<Domain> <VCPU|all> <CPUs|all>",
+      "<Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
     },
     { "vcpu-set",
       &main_vcpuset, 0, 1,