rcu-tasks: Initialize callback lists at rcu_init() time

In order for RCU Tasks to reliably maintain per-CPU lists of exiting
tasks, those lists must be initialized before it is possible for tasks
to exit, especially given that the boot CPU is not necessarily CPU 0
(an example being, powerpc kexec() kernels).  And at the time that
rcu_init_tasks_generic() is called, a task could potentially exit,
unconventional though that sort of thing might be.

This commit therefore moves the calls to cblist_init_generic() from
functions called from rcu_init_tasks_generic() to a new function named
tasks_cblist_init_generic() that is invoked from rcu_init().

This constituted a bug in a commit that never went to mainline, so
there is no need for any backporting to -stable.

Reported-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index f94f65877f2b68055b4e5c7a057c22bf4fb96a18..ef63ea59c8b6732f75548cd07c1d0c99ae69c431 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -528,6 +528,12 @@ struct task_struct *get_rcu_tasks_gp_kthread(void);
 struct task_struct *get_rcu_tasks_rude_gp_kthread(void);
 #endif // # ifdef CONFIG_TASKS_RUDE_RCU
 
+#ifdef CONFIG_TASKS_RCU_GENERIC
+void tasks_cblist_init_generic(void);
+#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
+static inline void tasks_cblist_init_generic(void) { }
+#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
+
 #define RCU_SCHEDULER_INACTIVE 0
 #define RCU_SCHEDULER_INIT     1
 #define RCU_SCHEDULER_RUNNING  2

diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index b7d5f27570532e8c822c618dd3164ff8787f4080..6961a1b5b7835f5fe99da251b77d83343c28eeb0 100644 (file)
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -242,7 +242,6 @@ static const char *tasks_gp_state_getname(struct rcu_tasks *rtp)
 static void cblist_init_generic(struct rcu_tasks *rtp)
 {
        int cpu;
-       unsigned long flags;
        int lim;
        int shift;
 
@@ -268,10 +267,8 @@ static void cblist_init_generic(struct rcu_tasks *rtp)
                WARN_ON_ONCE(!rtpcp);
                if (cpu)
                        raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock));
-               local_irq_save(flags);  // serialize initialization
                if (rcu_segcblist_empty(&rtpcp->cblist))
                        rcu_segcblist_init(&rtpcp->cblist);
-               local_irq_restore(flags);
                INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);
                rtpcp->cpu = cpu;
                rtpcp->rtpp = rtp;
@@ -1120,7 +1117,6 @@ module_param(rcu_tasks_lazy_ms, int, 0444);
 
 static int __init rcu_spawn_tasks_kthread(void)
 {
-       cblist_init_generic(&rcu_tasks);
        rcu_tasks.gp_sleep = HZ / 10;
        rcu_tasks.init_fract = HZ / 10;
        if (rcu_tasks_lazy_ms >= 0)
@@ -1284,7 +1280,6 @@ module_param(rcu_tasks_rude_lazy_ms, int, 0444);
 
 static int __init rcu_spawn_tasks_rude_kthread(void)
 {
-       cblist_init_generic(&rcu_tasks_rude);
        rcu_tasks_rude.gp_sleep = HZ / 10;
        if (rcu_tasks_rude_lazy_ms >= 0)
                rcu_tasks_rude.lazy_jiffies = msecs_to_jiffies(rcu_tasks_rude_lazy_ms);
@@ -1916,7 +1911,6 @@ module_param(rcu_tasks_trace_lazy_ms, int, 0444);
 
 static int __init rcu_spawn_tasks_trace_kthread(void)
 {
-       cblist_init_generic(&rcu_tasks_trace);
        if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {
                rcu_tasks_trace.gp_sleep = HZ / 10;
                rcu_tasks_trace.init_fract = HZ / 10;
@@ -2088,6 +2082,24 @@ late_initcall(rcu_tasks_verify_schedule_work);
 static void rcu_tasks_initiate_self_tests(void) { }
 #endif /* #else #ifdef CONFIG_PROVE_RCU */
 
+void __init tasks_cblist_init_generic(void)
+{
+       lockdep_assert_irqs_disabled();
+       WARN_ON(num_online_cpus() > 1);
+
+#ifdef CONFIG_TASKS_RCU
+       cblist_init_generic(&rcu_tasks);
+#endif
+
+#ifdef CONFIG_TASKS_RUDE_RCU
+       cblist_init_generic(&rcu_tasks_rude);
+#endif
+
+#ifdef CONFIG_TASKS_TRACE_RCU
+       cblist_init_generic(&rcu_tasks_trace);
+#endif
+}
+
 void __init rcu_init_tasks_generic(void)
 {
 #ifdef CONFIG_TASKS_RCU

diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index fec804b7908032d4416ea9ff3531eae154bba8f9..705c0d16850aa28db4e0ec7d26d10204e1ed24ce 100644 (file)
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -261,4 +261,5 @@ void __init rcu_init(void)
 {
        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
        rcu_early_boot_tests();
+       tasks_cblist_init_generic();
 }

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index b2bccfd37c383d04692fb6a7a72eb71a1f62798b..ba9137f39d143ccef43bcceaf1b9cf2f2e425e84 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -5165,6 +5165,8 @@ void __init rcu_init(void)
        (void)start_poll_synchronize_rcu_expedited();
 
        rcu_test_sync_prims();
+
+       tasks_cblist_init_generic();
 }
 
 #include "tree_stall.h"