sched: Force the address order of each sched class descriptor

In order to make a micro optimization in pick_next_task(), the order of the
sched class descriptor address must be in the same order as their priority
to each other. That is:

 &idle_sched_class < &fair_sched_class < &rt_sched_class <
 &dl_sched_class < &stop_sched_class

In order to guarantee this order of the sched class descriptors, add each
one into their own data section and force the order in the linker script.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/157675913272.349305.8936736338884044103.stgit@localhost.localdomain

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index db600ef218d7d2e56b6fdf895b1958f80fa05462..2186d7b01af61e21395ef6a8df5832c02ea0838a 100644 (file)
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -108,6 +108,18 @@
 #define SBSS_MAIN .sbss
 #endif
 
+/*
+ * The order of the sched class addresses are important, as they are
+ * used to determine the order of the priority of each sched class in
+ * relation to each other.
+ */
+#define SCHED_DATA                             \
+       *(__idle_sched_class)                   \
+       *(__fair_sched_class)                   \
+       *(__rt_sched_class)                     \
+       *(__dl_sched_class)                     \
+       *(__stop_sched_class)
+
 /*
  * Align to a 32 byte boundary equal to the
  * alignment gcc 4.5 uses for a struct
@@ -388,6 +400,7 @@
        .rodata           : AT(ADDR(.rodata) - LOAD_OFFSET) {           \
                __start_rodata = .;                                     \
                *(.rodata) *(.rodata.*)                                 \
+               SCHED_DATA                                              \
                RO_AFTER_INIT_DATA      /* Read only after init */      \
                . = ALIGN(8);                                           \
                __start___tracepoints_ptrs = .;                         \

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index d4708e29008fb39c984927b9471c58f8ea44e139..d9e79462993bd3e177da59559e015125c9929e43 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2479,7 +2479,8 @@ static void prio_changed_dl(struct rq *rq, struct task_struct *p,
        }
 }
 
-const struct sched_class dl_sched_class = {
+const struct sched_class dl_sched_class
+       __attribute__((section("__dl_sched_class"))) = {
        .next                   = &rt_sched_class,
        .enqueue_task           = enqueue_task_dl,
        .dequeue_task           = dequeue_task_dl,

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 0424a0af5f87bb4378e9d15534b7bcc6bbca9f3d..3365f6b07c367dd04db2316b52f4b7d562b586fc 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -11122,7 +11122,8 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
 /*
  * All the scheduling class methods:
  */
-const struct sched_class fair_sched_class = {
+const struct sched_class fair_sched_class
+       __attribute__((section("__fair_sched_class"))) = {
        .next                   = &idle_sched_class,
        .enqueue_task           = enqueue_task_fair,
        .dequeue_task           = dequeue_task_fair,

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 8d75ca201484b79907577713af981f2f38d8c0b0..f5806295356bfb3113541aa837a026aa21d74346 100644 (file)
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -453,7 +453,8 @@ static void update_curr_idle(struct rq *rq)
 /*
  * Simple, special scheduling class for the per-CPU idle tasks:
  */
-const struct sched_class idle_sched_class = {
+const struct sched_class idle_sched_class
+       __attribute__((section("__idle_sched_class"))) = {
        /* .next is NULL */
        /* no enqueue/yield_task for idle tasks */
 

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index f395ddb75f3857451497877aad36583ced560de6..6543d4430331e628a85bd94e32690768c87796e8 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2429,7 +2429,8 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
                return 0;
 }
 
-const struct sched_class rt_sched_class = {
+const struct sched_class rt_sched_class
+       __attribute__((section("__rt_sched_class"))) = {
        .next                   = &fair_sched_class,
        .enqueue_task           = enqueue_task_rt,
        .dequeue_task           = dequeue_task_rt,

diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 3e50a6a8f1e5073e44ebde08a1f85d8a9ddc7bbd..f4bbd54caae0cf411306f81847b62d7d956b4846 100644 (file)
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -109,7 +109,8 @@ static void update_curr_stop(struct rq *rq)
 /*
  * Simple, special scheduling class for the per-CPU stop tasks:
  */
-const struct sched_class stop_sched_class = {
+const struct sched_class stop_sched_class
+       __attribute__((section("__stop_sched_class"))) = {
        .next                   = &dl_sched_class,
 
        .enqueue_task           = enqueue_task_stop,