context_tracking: Take NMI eqs entrypoints over RCU

The RCU dynticks counter is going to be merged into the context tracking
subsystem. Prepare with moving the NMI extended quiescent states
entrypoints to context tracking. For now those are dumb redirection to
existing RCU calls.

Acked-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Nicolas Saenz Julienne <nsaenz@kernel.org>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Xiongfeng Wang <wangxiongfeng2@huawei.com>
Cc: Yu Liao <liaoyu15@huawei.com>
Cc: Phil Auld <pauld@redhat.com>
Cc: Paul Gortmaker<paul.gortmaker@windriver.com>
Cc: Alex Belits <abelits@marvell.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Reviewed-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>
Tested-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>

diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst
index 074810c739367742f7a83d246974f140879018a3..a0f8164c85135f76a259be05b61a1e45366ba3af 100644 (file)
--- a/Documentation/RCU/Design/Requirements/Requirements.rst
+++ b/Documentation/RCU/Design/Requirements/Requirements.rst
@@ -1847,7 +1847,7 @@ normal interrupts. One way that this can happen is for code that
 directly invokes ct_irq_enter() and ct_irq_exit() to be called
 from an NMI handler. This astonishing fact of life prompted the current
 code structure, which has ct_irq_enter() invoking
-rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit().
+ct_nmi_enter() and ct_irq_exit() invoking ct_nmi_exit().
 And yes, I also learned of this requirement the hard way.
 
 Loadable Modules

diff --git a/arch/Kconfig b/arch/Kconfig
index 342642be105fcbb43f80fab37e1bed4a3dfd6a5d..f56f7c0e924d88bb4e9d51b8aab8e3f17b0e6381 100644 (file)
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -797,7 +797,7 @@ config HAVE_CONTEXT_TRACKING_USER_OFFSTACK
 
          - Critical entry code isn't preemptible (or better yet:
            not interruptible).
-         - No use of RCU read side critical sections, unless rcu_nmi_enter()
+         - No use of RCU read side critical sections, unless ct_nmi_enter()
            got called.
          - No use of instrumentation, unless instrumentation_begin() got
            called.

diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 8dabe9ec10f1655b91bb1a1be44c5899f31a1020..c75ca36b4a49113091b804ebdcfd695350b9b487 100644 (file)
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -161,7 +161,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs)
        __nmi_enter();
        lockdep_hardirqs_off(CALLER_ADDR0);
        lockdep_hardirq_enter();
-       rcu_nmi_enter();
+       ct_nmi_enter();
 
        trace_hardirqs_off_finish();
        ftrace_nmi_enter();
@@ -182,7 +182,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs)
                lockdep_hardirqs_on_prepare();
        }
 
-       rcu_nmi_exit();
+       ct_nmi_exit();
        lockdep_hardirq_exit();
        if (restore)
                lockdep_hardirqs_on(CALLER_ADDR0);
@@ -199,7 +199,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
        regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
 
        lockdep_hardirqs_off(CALLER_ADDR0);
-       rcu_nmi_enter();
+       ct_nmi_enter();
 
        trace_hardirqs_off_finish();
 }
@@ -218,7 +218,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
                lockdep_hardirqs_on_prepare();
        }
 
-       rcu_nmi_exit();
+       ct_nmi_exit();
        if (restore)
                lockdep_hardirqs_on(CALLER_ADDR0);
 }

diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h
index 62f62bbd1a50d6feec1caa610f1e95eaf64124cd..c50b5670c4a52fd905da5a7b70b581d6a88959b2 100644 (file)
--- a/include/linux/context_tracking_irq.h
+++ b/include/linux/context_tracking_irq.h
@@ -7,11 +7,15 @@ void ct_irq_enter(void);
 void ct_irq_exit(void);
 void ct_irq_enter_irqson(void);
 void ct_irq_exit_irqson(void);
+void ct_nmi_enter(void);
+void ct_nmi_exit(void);
 #else
 static inline void ct_irq_enter(void) { }
 static inline void ct_irq_exit(void) { }
 static inline void ct_irq_enter_irqson(void) { }
 static inline void ct_irq_exit_irqson(void) { }
+static inline void ct_nmi_enter(void) { }
+static inline void ct_nmi_exit(void) { }
 #endif
 
 #endif

diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 76878b357ffa9cfde87f1e4bd751db789547c4df..345cdbe9c1b708f46db965640f50720edbf52990 100644 (file)
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -124,7 +124,7 @@ extern void rcu_nmi_exit(void);
        do {                                                    \
                __nmi_enter();                                  \
                lockdep_hardirq_enter();                        \
-               rcu_nmi_enter();                                \
+               ct_nmi_enter();                         \
                instrumentation_begin();                        \
                ftrace_nmi_enter();                             \
                instrumentation_end();                          \
@@ -143,7 +143,7 @@ extern void rcu_nmi_exit(void);
                instrumentation_begin();                        \
                ftrace_nmi_exit();                              \
                instrumentation_end();                          \
-               rcu_nmi_exit();                                 \
+               ct_nmi_exit();                                  \
                lockdep_hardirq_exit();                         \
                __nmi_exit();                                   \
        } while (0)

diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 72bd71a02c444c9c3f9ec33cd1b13a97ecb2cf1e..b8a731f20778c60eccfbd9d887bd31c1e4dd53cf 100644 (file)
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -55,6 +55,16 @@ void ct_irq_exit_irqson(void)
 {
        rcu_irq_exit_irqson();
 }
+
+noinstr void ct_nmi_enter(void)
+{
+       rcu_nmi_enter();
+}
+
+noinstr void ct_nmi_exit(void)
+{
+       rcu_nmi_exit();
+}
 #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
 
 #ifdef CONFIG_CONTEXT_TRACKING_USER

diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 667ba5d581ff76159d21c3fc802bcc99bdfd0472..063068a9ea9b35a54289d361a9429dd660f34b8e 100644 (file)
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -449,7 +449,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
        __nmi_enter();
        lockdep_hardirqs_off(CALLER_ADDR0);
        lockdep_hardirq_enter();
-       rcu_nmi_enter();
+       ct_nmi_enter();
 
        instrumentation_begin();
        trace_hardirqs_off_finish();
@@ -469,7 +469,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
        }
        instrumentation_end();
 
-       rcu_nmi_exit();
+       ct_nmi_exit();
        lockdep_hardirq_exit();
        if (irq_state.lockdep)
                lockdep_hardirqs_on(CALLER_ADDR0);

diff --git a/kernel/extable.c b/kernel/extable.c
index bda5e976154182c1253aedd4d371aae42328d5af..71f482581cab4c316ed2ddd2a4eedb6771b11c37 100644 (file)
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -114,7 +114,7 @@ int kernel_text_address(unsigned long addr)
 
        /* Treat this like an NMI as it can happen anywhere */
        if (no_rcu)
-               rcu_nmi_enter();
+               ct_nmi_enter();
 
        if (is_module_text_address(addr))
                goto out;
@@ -127,7 +127,7 @@ int kernel_text_address(unsigned long addr)
        ret = 0;
 out:
        if (no_rcu)
-               rcu_nmi_exit();
+               ct_nmi_exit();
 
        return ret;
 }

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index fe78a681812638f79e8cb36a163b71fe261ca25d..5fc7f17f5ec7b7dcaa2de4cf3d1ab0f277f20c00 100644 (file)
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -3105,7 +3105,7 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
        }
 
        /*
-        * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
+        * When an NMI triggers, RCU is enabled via ct_nmi_enter(),
         * but if the above rcu_is_watching() failed, then the NMI
         * triggered someplace critical, and ct_irq_enter() should
         * not be called from NMI.