[RFC PATCH v4] xen: credit2: provide custom option to create runqueue

[Praveen Kumar <kpraveen.lkml@gmail.com>]

The patch introduces a new, very flexible way of arranging runqueues in Credit2.
It allows to specify, explicitly and precisely, what pCPUs should belong to
which runqueue.

Signed-off-by: Praveen Kumar <kpraveen.lkml@gmail.com>
---
 docs/misc/xen-command-line.markdown |  10 ++-
 xen/common/sched_credit2.c          | 167 +++++++++++++++++++++++++++++++++++-
 2 files changed, 174 insertions(+), 3 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
index 33e54aef63..f2ee4ad972 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -525,7 +525,7 @@ also slow in responding to load changes.
 The default value of `1 sec` is rather long.
 
 ### credit2\_runqueue
-> `= cpu | core | socket | node | all`
+> `= cpu | core | socket | node | all | <custom>`
 
 > Default: `socket`
 
@@ -543,6 +543,14 @@ Available alternatives, with their meaning, are:
 * `node`: one runqueue per each NUMA node of the host;
 * `all`: just one runqueue shared by all the logical pCPUs of
          the host
+* `<custom>`: one runqueue per mentioned subset. The subset can be defined as
+              as shown in below example:
+              credit2_runqueue=[[0,1,][2,6][3,5][4,7]] , or 0,1\;2,6\;3,5\;4,7
+              which means :
+                  - pCPUs 0 and 1 belong to runqueue 0
+                  - pCPUs 2 and 6 belong to runqueue 1
+                  - pCPUs 3 and 5 belong to runqueue 2
+                  - pCPUs 4 and 7 belong to runqueue 3
 
 ### dbgp
 > `= ehci[ <integer> | @pci<bus>:<slot>.<func> ]`
diff --git a/xen/common/sched_credit2.c b/xen/common/sched_credit2.c
index b9b928347f..ebec33f450 100644
--- a/xen/common/sched_credit2.c
+++ b/xen/common/sched_credit2.c
@@ -321,6 +321,16 @@ integer_param("credit2_balance_over", opt_overload_balance_tolerance);
  *           (logical) processors of the host belong. This will happen if
  *           the opt_runqueue parameter is set to 'all'.
  *
+ * - custom: meaning that there will be one runqueue per subset being passed as
+ *           parameter to credit2_runqueue as shown in below example.
+ *           Example:
+ *           credit2_runqueue=[[cpu0,cpu1][cpu3][cpu4,cpu5]] or
+ *           credit2_runqueue=0,1\;3\;4,5
+ *           The example mentioned states :
+ *               cpu0 and cpu1 belongs to runqueue 0
+ *               cpu3 belongs to runqueue 1
+ *               cpu4 and cpu5 belongs to runqueue 2
+ *
  * Depending on the value of opt_runqueue, therefore, cpus that are part of
  * either the same physical core, the same physical socket, the same NUMA
  * node, or just all of them, will be put together to form runqueues.
@@ -330,15 +340,138 @@ integer_param("credit2_balance_over", opt_overload_balance_tolerance);
 #define OPT_RUNQUEUE_SOCKET 2
 #define OPT_RUNQUEUE_NODE   3
 #define OPT_RUNQUEUE_ALL    4
+#define OPT_RUNQUEUE_CUSTOM 5
 static const char *const opt_runqueue_str[] = {
     [OPT_RUNQUEUE_CPU] = "cpu",
     [OPT_RUNQUEUE_CORE] = "core",
     [OPT_RUNQUEUE_SOCKET] = "socket",
     [OPT_RUNQUEUE_NODE] = "node",
-    [OPT_RUNQUEUE_ALL] = "all"
+    [OPT_RUNQUEUE_ALL] = "all",
+    [OPT_RUNQUEUE_CUSTOM] = "custom"
 };
 static int __read_mostly opt_runqueue = OPT_RUNQUEUE_SOCKET;
 
+static unsigned long __read_mostly custom_cpu_runqueue[NR_CPUS];
+
+static inline int getlen(const char *start, const char *end)
+{
+    if ( ( start ) && ( end ) && ( end > start ) )
+        return end-start;
+    else
+        return -1;
+}
+
+static int parse_custom_runqueue_option(const char *s)
+{
+    const char *parse = NULL, *s_end = NULL;
+    const char *start = NULL, *end = NULL;
+    char delimiter[2] = {0};
+    int cpu_added_to_runqueue = 0;
+    int runqueue = 0;
+
+    /* Format supported :
+     * [[0,1,4,5][2,3,6,7][8,9,12,13][10,11,14,15]]
+     *                  or
+     * 0,1,4,5\;2,3,6,7\;8,9,12,13\;10,11,14,15
+     */
+    parse = s;
+    s_end = s + strlen(s);
+    /* The start and should always be in format of '[..]' */
+    if ( ( '[' == *parse ) && ( ']' == *(s_end-1)) )
+    {
+        delimiter[0] = '[';
+        delimiter[1] = '\0';
+        parse++;
+    }
+    else
+    {
+        delimiter[0] = ';';
+        delimiter[1] = '\0';
+    }
+
+    while ( ( parse != NULL ) && ( parse < s_end ) )
+    {
+        const char *token_sub_str = NULL;
+
+        while ( *parse == '[' )
+            parse++;
+
+        start = parse;
+        end = strstr(parse, delimiter);
+
+        /* Check if we don't have the delimiter */
+        if ( !end )
+        {
+            /* If we don't have delimiter, then break, if start is greater than
+             * or equal to s_end, as we have reached the end.
+             */
+            if ( start >= s_end )
+                break;
+
+            /* We need to parse till s_end, as we have the last set */
+            end = s_end;
+        }
+
+        /* Just move to next, as we have empty set like [] or ;; */
+        if ( getlen ( start, end ) < 1 )
+            goto next;
+
+        /*
+         * find token within the subset
+         */
+        do
+        {
+            unsigned long token = 0;
+
+            /* Get cpu ids separated by ',' within each set */
+            token_sub_str = strpbrk(start, ",");
+
+            /* Basic checks to validate the last entry in subset */
+            if ( ( !token_sub_str && start < end ) ||
+                ( token_sub_str > end && token_sub_str > start ) )
+            {
+                if ( ( delimiter[0] == '[' ) && ( start == s_end - 1 ) )
+                    goto next;
+
+                token_sub_str = end;
+            }
+
+            /* Just move to next, as we have empty set like [] or ;; */
+            if ( getlen(start, token_sub_str) < 1 )
+                goto next;
+
+            token = simple_strtoul(start ,&token_sub_str, 0);
+
+            if ( token >= nr_cpu_ids)
+                return -1;
+
+            /* If not set already */
+            if ( custom_cpu_runqueue[token] == -1 )
+            {
+                custom_cpu_runqueue[token] = runqueue;
+                cpu_added_to_runqueue = 1;
+            }
+            else
+                return -1;
+
+            if ( !token_sub_str || token_sub_str > end )
+                goto next;
+
+            start = ++token_sub_str;
+        } while ( start < end );
+next:
+        if ( cpu_added_to_runqueue )
+        {
+            runqueue++;
+            cpu_added_to_runqueue = 0;
+        }
+
+        parse = ++end;
+    }
+    opt_runqueue = OPT_RUNQUEUE_CUSTOM;
+    return 0;
+}
+
 static void parse_credit2_runqueue(const char *s)
 {
     unsigned int i;
@@ -351,8 +484,29 @@ static void parse_credit2_runqueue(const char *s)
             return;
         }
     }
+    /*
+     * At this stage we are either unknown value of credit2_runqueue or we can
+     * consider it to be custom cpu. Lets try parsing the same.
+     * Resetting the custom_cpu_runqueue for future use. Only the non-negative
+     * entries will be valid. The index 'i' in custom_cpu_runqueue will store
+     * the specific runqueue it belongs to.
+     * Example:
+     *     If custom_cpu_runqueue[3] == 2
+     *     Then, it means that cpu 3 belong to runqueue 2.
+     *     If custom_cpu_runqueue[4] == -1
+     *     Then, it means that cpu 4 doesn't belong to any runqueue.
+     */
+    for ( i = 0; i < nr_cpu_ids; i++ )
+        custom_cpu_runqueue[i] = -1;
 
-    printk("WARNING, unrecognized value of credit2_runqueue option!\n");
+    if ( parse_custom_runqueue_option(s) != 0 )
+    {
+        /* Resetting in case of failure, so that we don't mess-up during any failure
+         * due to wrong or spurious pattern passed by user.
+         */
+        opt_runqueue = OPT_RUNQUEUE_SOCKET;
+        printk("WARNING, unrecognized value of credit2_runqueue option!\n");
+    }
 }
 custom_param("credit2_runqueue", parse_credit2_runqueue);
 
@@ -662,6 +816,15 @@ cpu_to_runqueue(struct csched2_private *prv, unsigned int cpu)
     struct csched2_runqueue_data *rqd;
     unsigned int rqi;
 
+    if ( opt_runqueue == OPT_RUNQUEUE_CUSTOM )
+    {
+        if ( custom_cpu_runqueue[cpu] != -1 )
+        {
+            BUG_ON(custom_cpu_runqueue[cpu] >= nr_cpu_ids);
+            return custom_cpu_runqueue[cpu];
+        }
+    }
+
     for ( rqi = 0; rqi < nr_cpu_ids; rqi++ )
     {
         unsigned int peer_cpu;
-- 
2.12.0


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel