[PATCH v2 0/4] Allow schedulers to be selectable through Kconfig

[PATCH v2 0/4] Allow schedulers to be selectable through Kconfig

[Jonathan Creekmore]

Add machinery to allow the schedulers to be individually selectable
through the Kconfig interface. The first patch in the series sets up
the Kconfig options for the schedulers and places the appropriate CONFIG_
options around the scheduler list. The second, third, and fourth patches
rework the scheduler list from being manually curated into a model
where just compiling any schedulers into the hypervisor causes the
scheduler list to be built up.

---
Changed since v1:
  * Marked credit2 as EXPERIMENTAL
  * Removed confusing language from the commit message
  * Alphabetize the schedulers
  * rename the __start and __end symbols to better match
    the rest of the file
  * Simplify the calculation of the number of schedulers
  * Make the scheduler ops structures static to their files

Jonathan Creekmore (4):
  build: Hook the schedulers into Kconfig
  build: Alloc space for sched list in the link file
  sched: Register the schedulers into the list
  sched: Use the auto-generated list of schedulers

 xen/arch/arm/xen.lds.S      |  4 +++
 xen/arch/x86/xen.lds.S      |  4 +++
 xen/common/Kconfig          | 64 +++++++++++++++++++++++++++++++++++++++++++++
 xen/common/Makefile         |  8 +++---
 xen/common/sched_arinc653.c |  4 ++-
 xen/common/sched_credit.c   |  4 ++-
 xen/common/sched_credit2.c  |  4 ++-
 xen/common/sched_rt.c       |  4 ++-
 xen/common/schedule.c       | 20 +++++++-------
 xen/include/xen/sched-if.h  |  7 ++---
 10 files changed, 99 insertions(+), 24 deletions(-)

-- 
2.6.4

[PATCH v2 1/4] build: Hook the schedulers into Kconfig

[Jonathan Creekmore]

Allow the schedulers to be independently enabled or disabled at
compile-time. To match existing behavior, all four schedulers are
compiled in by default, although the Credit2, RTDS, and ARINC653 are
marked EXPERIMENTAL to match their not currently supported status.

CC: George Dunlap <george.dunlap@eu.citrix.com>
CC: Dario Faggioli <dario.faggioli@citrix.com>
Signed-off-by: Jonathan Creekmore <jonathan.creekmore@gmail.com>
Acked-by: Dario Faggioli <dario.faggioli@citrix.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

---
Changed since v1:

  * Marked credit2 as EXPERIMENTAL
  * Removed confusing language from the commit message
  * Alphabetize the schedulers
---
 xen/common/Kconfig    | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++
 xen/common/Makefile   |  8 +++----
 xen/common/schedule.c | 12 ++++++++--
 3 files changed, 78 insertions(+), 6 deletions(-)

diff --git a/xen/common/Kconfig b/xen/common/Kconfig
index 7d0e9a9..7f02738 100644
--- a/xen/common/Kconfig
+++ b/xen/common/Kconfig
@@ -44,4 +44,68 @@ config KEXEC
 
 	  If unsure, say Y.
 
+# Enable schedulers
+menu "Schedulers"
+config SCHED_CREDIT
+	bool "Credit scheduler support"
+	default y
+	---help---
+	  The traditional credit scheduler is a general purpose scheduler.
+
+	  If unsure, say Y.
+
+config SCHED_CREDIT2
+	bool "Credit2 scheduler support (EXPERIMENTAL)"
+	default y
+	---help---
+	  The credit2 scheduler is a general purpose scheduler that is
+	  optimized for lower latency and higher VM density.
+
+	  If unsure, say Y.
+
+config SCHED_RTDS
+	bool "RTDS scheduler support (EXPERIMENTAL)"
+	default y
+	---help---
+	  The RTDS scheduler is a soft and firm real-time scheduler for
+	  multicore, targeted for embedded, automotive, graphics and gaming
+	  in the cloud, and general low-latency workloads.
+
+	  If unsure, say N.
+
+config SCHED_ARINC653
+	bool "ARINC653 scheduler support (EXPERIMENTAL)"
+	default y
+	---help---
+	  The ARINC653 scheduler is a hard real-time scheduler for single
+	  cores, targeted for avionics, drones, and medical devices.
+
+	  If unsure, say N.
+
+choice
+	prompt "Default Scheduler?"
+	default SCHED_CREDIT_DEFAULT if SCHED_CREDIT
+	default SCHED_CREDIT2_DEFAULT if SCHED_CREDIT2
+	default SCHED_RTDS_DEFAULT if SCHED_RTDS
+	default SCHED_ARINC653_DEFAULT if SCHED_ARINC653
+
+	config SCHED_CREDIT_DEFAULT
+		bool "Credit Scheduler" if SCHED_CREDIT
+	config SCHED_CREDIT2_DEFAULT
+		bool "Credit2 Scheduler" if SCHED_CREDIT2
+	config SCHED_RTDS_DEFAULT
+		bool "RT Scheduler" if SCHED_RTDS
+	config SCHED_ARINC653_DEFAULT
+		bool "ARINC653 Scheduler" if SCHED_ARINC653
+endchoice
+
+config SCHED_DEFAULT
+	string
+	default "credit" if SCHED_CREDIT_DEFAULT
+	default "credit2" if SCHED_CREDIT2_DEFAULT
+	default "rtds" if SCHED_RTDS_DEFAULT
+	default "arinc653" if SCHED_ARINC653_DEFAULT
+
+endmenu
+
 endmenu
diff --git a/xen/common/Makefile b/xen/common/Makefile
index 8ab15ba..29a5916 100644
--- a/xen/common/Makefile
+++ b/xen/common/Makefile
@@ -30,10 +30,10 @@ obj-y += rangeset.o
 obj-y += radix-tree.o
 obj-y += rbtree.o
 obj-y += rcupdate.o
-obj-y += sched_credit.o
-obj-y += sched_credit2.o
-obj-y += sched_arinc653.o
-obj-y += sched_rt.o
+obj-$(CONFIG_SCHED_ARINC653) += sched_arinc653.o
+obj-$(CONFIG_SCHED_CREDIT) += sched_credit.o
+obj-$(CONFIG_SCHED_CREDIT2) += sched_credit2.o
+obj-$(CONFIG_SCHED_RTDS) += sched_rt.o
 obj-y += schedule.o
 obj-y += shutdown.o
 obj-y += softirq.o
diff --git a/xen/common/schedule.c b/xen/common/schedule.c
index d121896..2f98a48 100644
--- a/xen/common/schedule.c
+++ b/xen/common/schedule.c
@@ -38,8 +38,8 @@
 #include <public/sched.h>
 #include <xsm/xsm.h>
 
-/* opt_sched: scheduler - default to credit */
-static char __initdata opt_sched[10] = "credit";
+/* opt_sched: scheduler - default to configured value */
+static char __initdata opt_sched[10] = CONFIG_SCHED_DEFAULT;
 string_param("sched", opt_sched);
 
 /* if sched_smt_power_savings is set,
@@ -65,10 +65,18 @@ DEFINE_PER_CPU(struct schedule_data, schedule_data);
 DEFINE_PER_CPU(struct scheduler *, scheduler);
 
 static const struct scheduler *schedulers[] = {
+#ifdef CONFIG_SCHED_CREDIT
     &sched_credit_def,
+#endif
+#ifdef CONFIG_SCHED_CREDIT2
     &sched_credit2_def,
+#endif
+#ifdef CONFIG_SCHED_ARINC653
     &sched_arinc653_def,
+#endif
+#ifdef CONFIG_SCHED_RTDS
     &sched_rtds_def,
+#endif
 };
 
 static struct scheduler __read_mostly ops;
-- 
2.6.4

[PATCH v2 2/4] build: Alloc space for sched list in the link file

[Jonathan Creekmore]

Creates a section to contain scheduler entry pointers that are gathered
together into an array. This will allow, in a follow-on patch, scheduler
entries to be automatically gathered together into the array for
automatic parsing.

CC: Ian Campbell <ian.campbell@citrix.com>
CC: Stefano Stabellini <stefano.stabellini@citrix.com>
CC: Keir Fraser <keir@xen.org>
CC: Jan Beulich <jbeulich@suse.com>
CC: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Jonathan Creekmore <jonathan.creekmore@gmail.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

---
Changed since v1:
  * rename the __start and __end symbols to better match
    the rest of the file
---
 xen/arch/arm/xen.lds.S | 4 ++++
 xen/arch/x86/xen.lds.S | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/xen/arch/arm/xen.lds.S b/xen/arch/arm/xen.lds.S
index 0488f37..f501a2f 100644
--- a/xen/arch/arm/xen.lds.S
+++ b/xen/arch/arm/xen.lds.S
@@ -57,6 +57,10 @@ SECTIONS
        . = ALIGN(PAGE_SIZE);
        *(.data.page_aligned)
        *(.data)
+       . = ALIGN(8);
+       __start_schedulers_array = .;
+       *(.data.schedulers)
+       __end_schedulers_array = .;
        *(.data.rel)
        *(.data.rel.*)
        CONSTRUCTORS
diff --git a/xen/arch/x86/xen.lds.S b/xen/arch/x86/xen.lds.S
index e18e08f..c1ce027 100644
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -80,6 +80,10 @@ SECTIONS
        __stop___pre_ex_table = .;
 
        *(.data.read_mostly)
+       . = ALIGN(8);
+       __start_schedulers_array = .;
+       *(.data.schedulers)
+       __end_schedulers_array = .;
        *(.data.rel.ro)
        *(.data.rel.ro.*)
   } :text
-- 
2.6.4

[PATCH v2 3/4] sched: Register the schedulers into the list

[Jonathan Creekmore]

Adds a simple macro to place a pointer to a scheduler into an array
section at compile time. Also, goes ahead and generates the array
entries with each of the schedulers.

CC: George Dunlap <george.dunlap@eu.citrix.com>
CC: Dario Faggioli <dario.faggioli@citrix.com>
CC: Josh Whitehead <josh.whitehead@dornerworks.com>
CC: Robert VanVossen <robert.vanvossen@dornerworks.com>
Signed-off-by: Jonathan Creekmore <jonathan.creekmore@gmail.com>
Acked-by: Dario Faggioli <dario.faggioli@citrix.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
 xen/common/sched_arinc653.c | 2 ++
 xen/common/sched_credit.c   | 2 ++
 xen/common/sched_credit2.c  | 2 ++
 xen/common/sched_rt.c       | 2 ++
 xen/include/xen/sched-if.h  | 2 ++
 5 files changed, 10 insertions(+)

diff --git a/xen/common/sched_arinc653.c b/xen/common/sched_arinc653.c
index dbe02ed..3b59514 100644
--- a/xen/common/sched_arinc653.c
+++ b/xen/common/sched_arinc653.c
@@ -767,6 +767,8 @@ const struct scheduler sched_arinc653_def = {
     .tick_resume    = NULL,
 };
 
+REGISTER_SCHEDULER(sched_arinc653_def);
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/common/sched_credit.c b/xen/common/sched_credit.c
index 0dce790..e586248 100644
--- a/xen/common/sched_credit.c
+++ b/xen/common/sched_credit.c
@@ -2027,3 +2027,5 @@ const struct scheduler sched_credit_def = {
     .tick_suspend   = csched_tick_suspend,
     .tick_resume    = csched_tick_resume,
 };
+
+REGISTER_SCHEDULER(sched_credit_def);
diff --git a/xen/common/sched_credit2.c b/xen/common/sched_credit2.c
index 3c49ffa..38b02d0 100644
--- a/xen/common/sched_credit2.c
+++ b/xen/common/sched_credit2.c
@@ -2228,3 +2228,5 @@ const struct scheduler sched_credit2_def = {
     .alloc_domdata  = csched2_alloc_domdata,
     .free_domdata   = csched2_free_domdata,
 };
+
+REGISTER_SCHEDULER(sched_credit2_def);
diff --git a/xen/common/sched_rt.c b/xen/common/sched_rt.c
index 3f1d047..7640cd0 100644
--- a/xen/common/sched_rt.c
+++ b/xen/common/sched_rt.c
@@ -1199,3 +1199,5 @@ const struct scheduler sched_rtds_def = {
     .wake           = rt_vcpu_wake,
     .context_saved  = rt_context_saved,
 };
+
+REGISTER_SCHEDULER(sched_rtds_def);
diff --git a/xen/include/xen/sched-if.h b/xen/include/xen/sched-if.h
index 493d43f..9c6e0f5 100644
--- a/xen/include/xen/sched-if.h
+++ b/xen/include/xen/sched-if.h
@@ -170,6 +170,8 @@ extern const struct scheduler sched_credit2_def;
 extern const struct scheduler sched_arinc653_def;
 extern const struct scheduler sched_rtds_def;
 
+#define REGISTER_SCHEDULER(x) static const struct scheduler *x##_entry \
+  __used_section(".data.schedulers") = &x;
 
 struct cpupool
 {
-- 
2.6.4

[PATCH v2 4/4] sched: Use the auto-generated list of schedulers

[Jonathan Creekmore]

Instead of having a manually-curated list of schedulers, use the array
that was auto-generated simply by compiling in the scheduler files as
the sole source of truth of the available schedulers.

CC: George Dunlap <george.dunlap@eu.citrix.com>
CC: Dario Faggioli <dario.faggioli@citrix.com>
Signed-off-by: Jonathan Creekmore <jonathan.creekmore@gmail.com>
Acked-by: Dario Faggioli <dario.faggioli@citrix.com>

---
Changed since v1:
  * Simplify the calculation of the number of schedulers
  * Make the scheduler ops structures static to their files
---
 xen/common/sched_arinc653.c |  2 +-
 xen/common/sched_credit.c   |  2 +-
 xen/common/sched_credit2.c  |  2 +-
 xen/common/sched_rt.c       |  2 +-
 xen/common/schedule.c       | 24 +++++++-----------------
 xen/include/xen/sched-if.h  |  5 -----
 6 files changed, 11 insertions(+), 26 deletions(-)

diff --git a/xen/common/sched_arinc653.c b/xen/common/sched_arinc653.c
index 3b59514..0606988 100644
--- a/xen/common/sched_arinc653.c
+++ b/xen/common/sched_arinc653.c
@@ -724,7 +724,7 @@ a653sched_adjust_global(const struct scheduler *ops,
  * callback functions.
  * The symbol must be visible to the rest of Xen at link time.
  */
-const struct scheduler sched_arinc653_def = {
+static const struct scheduler sched_arinc653_def = {
     .name           = "ARINC 653 Scheduler",
     .opt_name       = "arinc653",
     .sched_id       = XEN_SCHEDULER_ARINC653,
diff --git a/xen/common/sched_credit.c b/xen/common/sched_credit.c
index e586248..028e41b 100644
--- a/xen/common/sched_credit.c
+++ b/xen/common/sched_credit.c
@@ -1991,7 +1991,7 @@ static void csched_tick_resume(const struct scheduler *ops, unsigned int cpu)
 
 static struct csched_private _csched_priv;
 
-const struct scheduler sched_credit_def = {
+static const struct scheduler sched_credit_def = {
     .name           = "SMP Credit Scheduler",
     .opt_name       = "credit",
     .sched_id       = XEN_SCHEDULER_CREDIT,
diff --git a/xen/common/sched_credit2.c b/xen/common/sched_credit2.c
index 38b02d0..78220a7 100644
--- a/xen/common/sched_credit2.c
+++ b/xen/common/sched_credit2.c
@@ -2194,7 +2194,7 @@ csched2_deinit(const struct scheduler *ops)
 
 static struct csched2_private _csched2_priv;
 
-const struct scheduler sched_credit2_def = {
+static const struct scheduler sched_credit2_def = {
     .name           = "SMP Credit Scheduler rev2",
     .opt_name       = "credit2",
     .sched_id       = XEN_SCHEDULER_CREDIT2,
diff --git a/xen/common/sched_rt.c b/xen/common/sched_rt.c
index 7640cd0..2e5430f 100644
--- a/xen/common/sched_rt.c
+++ b/xen/common/sched_rt.c
@@ -1170,7 +1170,7 @@ rt_dom_cntl(
 
 static struct rt_private _rt_priv;
 
-const struct scheduler sched_rtds_def = {
+static const struct scheduler sched_rtds_def = {
     .name           = "SMP RTDS Scheduler",
     .opt_name       = "rtds",
     .sched_id       = XEN_SCHEDULER_RTDS,
diff --git a/xen/common/schedule.c b/xen/common/schedule.c
index 2f98a48..91e53c1 100644
--- a/xen/common/schedule.c
+++ b/xen/common/schedule.c
@@ -64,20 +64,10 @@ static void poll_timer_fn(void *data);
 DEFINE_PER_CPU(struct schedule_data, schedule_data);
 DEFINE_PER_CPU(struct scheduler *, scheduler);
 
-static const struct scheduler *schedulers[] = {
-#ifdef CONFIG_SCHED_CREDIT
-    &sched_credit_def,
-#endif
-#ifdef CONFIG_SCHED_CREDIT2
-    &sched_credit2_def,
-#endif
-#ifdef CONFIG_SCHED_ARINC653
-    &sched_arinc653_def,
-#endif
-#ifdef CONFIG_SCHED_RTDS
-    &sched_rtds_def,
-#endif
-};
+extern const struct scheduler *__start_schedulers_array[], *__end_schedulers_array[];
+extern const size_t schedulers_array_size;
+#define NUM_SCHEDULERS (__end_schedulers_array - __start_schedulers_array)
+static const struct scheduler **schedulers = __start_schedulers_array;
 
 static struct scheduler __read_mostly ops;
 
@@ -1468,7 +1458,7 @@ void __init scheduler_init(void)
 
     open_softirq(SCHEDULE_SOFTIRQ, schedule);
 
-    for ( i = 0; i < ARRAY_SIZE(schedulers); i++ )
+    for ( i = 0; i < NUM_SCHEDULERS; i++)
     {
         if ( schedulers[i]->global_init && schedulers[i]->global_init() < 0 )
             schedulers[i] = NULL;
@@ -1479,7 +1469,7 @@ void __init scheduler_init(void)
     if ( !ops.name )
     {
         printk("Could not find scheduler: %s\n", opt_sched);
-        for ( i = 0; i < ARRAY_SIZE(schedulers); i++ )
+        for ( i = 0; i < NUM_SCHEDULERS; i++ )
             if ( schedulers[i] )
             {
                 ops = *schedulers[i];
@@ -1599,7 +1589,7 @@ struct scheduler *scheduler_alloc(unsigned int sched_id, int *perr)
     int i;
     struct scheduler *sched;
 
-    for ( i = 0; i < ARRAY_SIZE(schedulers); i++ )
+    for ( i = 0; i < NUM_SCHEDULERS; i++ )
         if ( schedulers[i] && schedulers[i]->sched_id == sched_id )
             goto found;
     *perr = -ENOENT;
diff --git a/xen/include/xen/sched-if.h b/xen/include/xen/sched-if.h
index 9c6e0f5..66dc9c8 100644
--- a/xen/include/xen/sched-if.h
+++ b/xen/include/xen/sched-if.h
@@ -165,11 +165,6 @@ struct scheduler {
     void         (*tick_resume)     (const struct scheduler *, unsigned int);
 };
 
-extern const struct scheduler sched_credit_def;
-extern const struct scheduler sched_credit2_def;
-extern const struct scheduler sched_arinc653_def;
-extern const struct scheduler sched_rtds_def;
-
 #define REGISTER_SCHEDULER(x) static const struct scheduler *x##_entry \
   __used_section(".data.schedulers") = &x;
 
-- 
2.6.4

Re: [PATCH v2 4/4] sched: Use the auto-generated list of schedulers

[Andrew Cooper]

On 18/12/2015 21:41, Jonathan Creekmore wrote:
> Instead of having a manually-curated list of schedulers, use the array
> that was auto-generated simply by compiling in the scheduler files as
> the sole source of truth of the available schedulers.
>
> CC: George Dunlap <george.dunlap@eu.citrix.com>
> CC: Dario Faggioli <dario.faggioli@citrix.com>
> Signed-off-by: Jonathan Creekmore <jonathan.creekmore@gmail.com>
> Acked-by: Dario Faggioli <dario.faggioli@citrix.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>