rcu: Make RCU IRQ enter/exit functions rely on in_nmi()

The rcu_nmi_enter_common() and rcu_nmi_exit_common() functions take an
"irq" parameter that indicates whether these functions have been invoked from
an irq handler (irq==true) or an NMI handler (irq==false).

However, recent changes have applied notrace to a few critical functions
such that rcu_nmi_enter_common() and rcu_nmi_exit_common() many now rely on
in_nmi().  Note that in_nmi() works no differently than before, but rather
that tracing is now prohibited in code regions where in_nmi() would
incorrectly report NMI state.

Therefore remove the "irq" parameter and inline rcu_nmi_enter_common() and
rcu_nmi_exit_common() into rcu_nmi_enter() and rcu_nmi_exit(),
respectively.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Link: https://lkml.kernel.org/r/20200505134101.617130349@linutronix.de

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 0713ef3fa560d1412892597bc8fb8d285fdb163b..945401674b9d6c1f54ea6ee319f60e1ea2404670 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -664,16 +664,18 @@ noinstr void rcu_user_enter(void)
 }
 #endif /* CONFIG_NO_HZ_FULL */
 
-/*
+/**
+ * rcu_nmi_exit - inform RCU of exit from NMI context
+ *
  * If we are returning from the outermost NMI handler that interrupted an
  * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
  * to let the RCU grace-period handling know that the CPU is back to
  * being RCU-idle.
  *
- * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
+ * If you add or remove a call to rcu_nmi_exit(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-static __always_inline void rcu_nmi_exit_common(bool irq)
+noinstr void rcu_nmi_exit(void)
 {
        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 
@@ -704,7 +706,7 @@ static __always_inline void rcu_nmi_exit_common(bool irq)
        trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
        WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
 
-       if (irq)
+       if (!in_nmi())
                rcu_prepare_for_idle();
        instrumentation_end();
 
@@ -712,21 +714,10 @@ static __always_inline void rcu_nmi_exit_common(bool irq)
        rcu_dynticks_eqs_enter();
        // ... but is no longer watching here.
 
-       if (irq)
+       if (!in_nmi())
                rcu_dynticks_task_enter();
 }
 
-/**
- * rcu_nmi_exit - inform RCU of exit from NMI context
- *
- * If you add or remove a call to rcu_nmi_exit(), be sure to test
- * with CONFIG_RCU_EQS_DEBUG=y.
- */
-void noinstr rcu_nmi_exit(void)
-{
-       rcu_nmi_exit_common(false);
-}
-
 /**
  * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
  *
@@ -749,7 +740,7 @@ void noinstr rcu_nmi_exit(void)
 void noinstr rcu_irq_exit(void)
 {
        lockdep_assert_irqs_disabled();
-       rcu_nmi_exit_common(true);
+       rcu_nmi_exit();
 }
 
 /*
@@ -838,7 +829,7 @@ void noinstr rcu_user_exit(void)
 #endif /* CONFIG_NO_HZ_FULL */
 
 /**
- * rcu_nmi_enter_common - inform RCU of entry to NMI context
+ * rcu_nmi_enter - inform RCU of entry to NMI context
  * @irq: Is this call from rcu_irq_enter?
  *
  * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
@@ -847,10 +838,10 @@ void noinstr rcu_user_exit(void)
  * long as the nesting level does not overflow an int.  (You will probably
  * run out of stack space first.)
  *
- * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
+ * If you add or remove a call to rcu_nmi_enter(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-static __always_inline void rcu_nmi_enter_common(bool irq)
+noinstr void rcu_nmi_enter(void)
 {
        long incby = 2;
        struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
@@ -868,18 +859,18 @@ static __always_inline void rcu_nmi_enter_common(bool irq)
         */
        if (rcu_dynticks_curr_cpu_in_eqs()) {
 
-               if (irq)
+               if (!in_nmi())
                        rcu_dynticks_task_exit();
 
                // RCU is not watching here ...
                rcu_dynticks_eqs_exit();
                // ... but is watching here.
 
-               if (irq)
+               if (!in_nmi())
                        rcu_cleanup_after_idle();
 
                incby = 1;
-       } else if (irq) {
+       } else if (!in_nmi()) {
                instrumentation_begin();
                if (tick_nohz_full_cpu(rdp->cpu) &&
                    rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
@@ -913,14 +904,6 @@ static __always_inline void rcu_nmi_enter_common(bool irq)
        barrier();
 }
 
-/**
- * rcu_nmi_enter - inform RCU of entry to NMI context
- */
-noinstr void rcu_nmi_enter(void)
-{
-       rcu_nmi_enter_common(false);
-}
-
 /**
  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  *
@@ -946,7 +929,7 @@ noinstr void rcu_nmi_enter(void)
 noinstr void rcu_irq_enter(void)
 {
        lockdep_assert_irqs_disabled();
-       rcu_nmi_enter_common(true);
+       rcu_nmi_enter();
 }
 
 /*