rcu/nocb: Invoke rcu_core() at the start of deoffloading

On PREEMPT_RT, if rcu_core() is preempted by the de-offloading process,
some work, such as callbacks acceleration and invocation, may be left
unattended due to the volatile checks on the offloaded state.

In the worst case this work is postponed until the next rcu_pending()
check that can take a jiffy to reach, which can be a problem in case
of callbacks flooding.

Solve that with invoking rcu_core() early in the de-offloading process.
This way any work dismissed by an ongoing rcu_core() call fooled by
a preempting deoffloading process will be caught up by a nearby future
recall to rcu_core(), this time fully aware of the de-offloading state.

Tested-by: Valentin Schneider <valentin.schneider@arm.com>
Tested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Valentin Schneider <valentin.schneider@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Neeraj Upadhyay <neeraju@codeaurora.org>
Cc: Uladzislau Rezki <urezki@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h
index 812961b1d0642b5c080ebfbd9c38ad643f44dd2b..659d13a7ddaaadf7ab41692af5caf684aef7523b 100644 (file)
--- a/include/linux/rcu_segcblist.h
+++ b/include/linux/rcu_segcblist.h
@@ -136,6 +136,20 @@ struct rcu_cblist {
  *  |--------------------------------------------------------------------------|
  *  |                           SEGCBLIST_RCU_CORE   |                         |
  *  |                           SEGCBLIST_LOCKING    |                         |
+ *  |                           SEGCBLIST_OFFLOADED  |                         |
+ *  |                           SEGCBLIST_KTHREAD_CB |                         |
+ *  |                           SEGCBLIST_KTHREAD_GP                           |
+ *  |                                                                          |
+ *  |   CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core()    |
+ *  |   handles callbacks concurrently. Bypass enqueue is enabled.             |
+ *  |   Invoke RCU core so we make sure not to preempt it in the middle with   |
+ *  |   leaving some urgent work unattended within a jiffy.                    |
+ *  ----------------------------------------------------------------------------
+ *                                      |
+ *                                      v
+ *  |--------------------------------------------------------------------------|
+ *  |                           SEGCBLIST_RCU_CORE   |                         |
+ *  |                           SEGCBLIST_LOCKING    |                         |
  *  |                           SEGCBLIST_KTHREAD_CB |                         |
  *  |                           SEGCBLIST_KTHREAD_GP                           |
  *  |                                                                          |

diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c
index c07aab6e39ef266f74a9b0eea12ab2f897c4b0da..81145c3ece25fab1f089a2e0eeaee80776d55c23 100644 (file)
--- a/kernel/rcu/rcu_segcblist.c
+++ b/kernel/rcu/rcu_segcblist.c
@@ -265,12 +265,10 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
  */
 void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload)
 {
-       if (offload) {
+       if (offload)
                rcu_segcblist_set_flags(rsclp, SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED);
-       } else {
-               rcu_segcblist_set_flags(rsclp, SEGCBLIST_RCU_CORE);
+       else
                rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED);
-       }
 }
 
 /*

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e905d7e4ddb9151fe63c6eba42ca5429b9e4ecaf..a329adfece86e233f7479c420fff900b27485457 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2707,6 +2707,23 @@ static __latent_entropy void rcu_core(void)
        unsigned long flags;
        struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
        struct rcu_node *rnp = rdp->mynode;
+       /*
+        * On RT rcu_core() can be preempted when IRQs aren't disabled.
+        * Therefore this function can race with concurrent NOCB (de-)offloading
+        * on this CPU and the below condition must be considered volatile.
+        * However if we race with:
+        *
+        * _ Offloading:   In the worst case we accelerate or process callbacks
+        *                 concurrently with NOCB kthreads. We are guaranteed to
+        *                 call rcu_nocb_lock() if that happens.
+        *
+        * _ Deoffloading: In the worst case we miss callbacks acceleration or
+        *                 processing. This is fine because the early stage
+        *                 of deoffloading invokes rcu_core() after setting
+        *                 SEGCBLIST_RCU_CORE. So we guarantee that we'll process
+        *                 what could have been dismissed without the need to wait
+        *                 for the next rcu_pending() check in the next jiffy.
+        */
        const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);
 
        if (cpu_is_offline(smp_processor_id()))

diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index b3e07d0bfbbf80b6367940c43b8960299151dbe4..2461fe8d0c23a8e873f85b728e6b7e2ec3356ba5 100644 (file)
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -990,6 +990,15 @@ static long rcu_nocb_rdp_deoffload(void *arg)
         * will refuse to put anything into the bypass.
         */
        WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
+       /*
+        * Start with invoking rcu_core() early. This way if the current thread
+        * happens to preempt an ongoing call to rcu_core() in the middle,
+        * leaving some work dismissed because rcu_core() still thinks the rdp is
+        * completely offloaded, we are guaranteed a nearby future instance of
+        * rcu_core() to catch up.
+        */
+       rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE);
+       invoke_rcu_core();
        ret = rdp_offload_toggle(rdp, false, flags);
        swait_event_exclusive(rdp->nocb_state_wq,
                              !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB |