context_tracking: Take idle eqs entrypoints over RCU

The RCU dynticks counter is going to be merged into the context tracking
subsystem. Start with moving the idle extended quiescent states
entrypoints to context tracking. For now those are dumb redirections to
existing RCU calls.

[ paulmck: Apply kernel test robot feedback. ]

Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Nicolas Saenz Julienne <nsaenz@kernel.org>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Xiongfeng Wang <wangxiongfeng2@huawei.com>
Cc: Yu Liao <liaoyu15@huawei.com>
Cc: Phil Auld <pauld@redhat.com>
Cc: Paul Gortmaker<paul.gortmaker@windriver.com>
Cc: Alex Belits <abelits@marvell.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Reviewed-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>
Tested-by: Nicolas Saenz Julienne <nsaenzju@redhat.com>

diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst
index 794837eb519b94949dff617259a72ab2c1f2d2b6..b95bda7755fa90846d4a8ba045748909851ae47a 100644 (file)
--- a/Documentation/RCU/stallwarn.rst
+++ b/Documentation/RCU/stallwarn.rst
@@ -97,8 +97,8 @@ warnings:
        which will include additional debugging information.
 
 -      A low-level kernel issue that either fails to invoke one of the
-       variants of rcu_user_enter(), rcu_user_exit(), rcu_idle_enter(),
-       rcu_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one
+       variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(),
+       ct_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one
        hand, or that invokes one of them too many times on the other.
        Historically, the most frequent issue has been an omission
        of either irq_enter() or irq_exit(), which in turn invoke

diff --git a/arch/arm/mach-imx/cpuidle-imx6q.c b/arch/arm/mach-imx/cpuidle-imx6q.c
index 094337dc1bc7e3bd77c03ebd73d5a59506a80892..d086cbae09c37ff178a594608c996ed86b6f9cdb 100644 (file)
--- a/arch/arm/mach-imx/cpuidle-imx6q.c
+++ b/arch/arm/mach-imx/cpuidle-imx6q.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2012 Freescale Semiconductor, Inc.
  */
 
+#include <linux/context_tracking.h>
 #include <linux/cpuidle.h>
 #include <linux/module.h>
 #include <asm/cpuidle.h>
@@ -24,9 +25,9 @@ static int imx6q_enter_wait(struct cpuidle_device *dev,
                imx6_set_lpm(WAIT_UNCLOCKED);
        raw_spin_unlock(&cpuidle_lock);
 
-       rcu_idle_enter();
+       ct_idle_enter();
        cpu_do_idle();
-       rcu_idle_exit();
+       ct_idle_exit();
 
        raw_spin_lock(&cpuidle_lock);
        if (num_idle_cpus-- == num_online_cpus())

diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 6a5572a1a80ccfba497c95ca96d00ecf6badf4a1..1401d193a2dfc70d5dc2fd3a178acc5a5b7f9e24 100644 (file)
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -23,6 +23,7 @@
 #include <linux/minmax.h>
 #include <linux/perf_event.h>
 #include <acpi/processor.h>
+#include <linux/context_tracking.h>
 
 /*
  * Include the apic definitions for x86 to have the APIC timer related defines
@@ -647,11 +648,11 @@ static int acpi_idle_enter_bm(struct cpuidle_driver *drv,
                raw_spin_unlock(&c3_lock);
        }
 
-       rcu_idle_enter();
+       ct_idle_enter();
 
        acpi_idle_do_entry(cx);
 
-       rcu_idle_exit();
+       ct_idle_exit();
 
        /* Re-enable bus master arbitration */
        if (dis_bm) {

diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index ef2ea1b12cd842082e2bfb3903e1bb5d8632ba13..62dd956025f333ed5a014125f83ed64049b7c1fd 100644 (file)
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -23,6 +23,7 @@
 #include <linux/suspend.h>
 #include <linux/tick.h>
 #include <linux/mmu_context.h>
+#include <linux/context_tracking.h>
 #include <trace/events/power.h>
 
 #include "cpuidle.h"
@@ -150,12 +151,12 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
         */
        stop_critical_timings();
        if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
-               rcu_idle_enter();
+               ct_idle_enter();
        target_state->enter_s2idle(dev, drv, index);
        if (WARN_ON_ONCE(!irqs_disabled()))
                local_irq_disable();
        if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
-               rcu_idle_exit();
+               ct_idle_exit();
        tick_unfreeze();
        start_critical_timings();
 
@@ -233,10 +234,10 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
 
        stop_critical_timings();
        if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
-               rcu_idle_enter();
+               ct_idle_enter();
        entered_state = target_state->enter(dev, drv, index);
        if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
-               rcu_idle_exit();
+               ct_idle_exit();
        start_critical_timings();
 
        sched_clock_idle_wakeup_event();

diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index e35ae66b4794ea16c56f0e4ffc659e56fa8f0403..01abadb2f9930034a853e4c59291ce952d10ab1f 100644 (file)
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -119,4 +119,12 @@ extern void context_tracking_init(void);
 static inline void context_tracking_init(void) { }
 #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */
 
+#ifdef CONFIG_CONTEXT_TRACKING_IDLE
+extern void ct_idle_enter(void);
+extern void ct_idle_exit(void);
+#else
+static inline void ct_idle_enter(void) { }
+static inline void ct_idle_exit(void) { }
+#endif /* !CONFIG_CONTEXT_TRACKING_IDLE */
+
 #endif

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 1a32036c918cd1e22e99bdbdd305769cfc74f429..6ebe754501c38a97e697c028b7a803bbfdab827d 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -128,7 +128,7 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { }
  * @a: Code that RCU needs to pay attention to.
  *
  * RCU read-side critical sections are forbidden in the inner idle loop,
- * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU
+ * that is, between the ct_idle_enter() and the ct_idle_exit() -- RCU
  * will happily ignore any such read-side critical sections.  However,
  * things like powertop need tracepoints in the inner idle loop.
  *

diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index f3dec1be2bf62a077a85aad58d3382c52e842dda..c0b3798d4e940274e1018c45e723b9eaf0336d16 100644 (file)
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -22,6 +22,21 @@
 #include <linux/export.h>
 #include <linux/kprobes.h>
 
+
+#ifdef CONFIG_CONTEXT_TRACKING_IDLE
+noinstr void ct_idle_enter(void)
+{
+       rcu_idle_enter();
+}
+EXPORT_SYMBOL_GPL(ct_idle_enter);
+
+void ct_idle_exit(void)
+{
+       rcu_idle_exit();
+}
+EXPORT_SYMBOL_GPL(ct_idle_exit);
+#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
+
 #ifdef CONFIG_CONTEXT_TRACKING_USER
 
 #define CREATE_TRACE_POINTS

diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index f06b91ca6482d36777d42e0af52c82f2b6ba6bb0..5ea690cb4f7af5af527d469e6c9ba513a3c7cf69 100644 (file)
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -6570,7 +6570,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
 
        /*
         * If a CPU is in the RCU-free window in idle (ie: in the section
-        * between rcu_idle_enter() and rcu_idle_exit(), then RCU
+        * between ct_idle_enter() and ct_idle_exit(), then RCU
         * considers that CPU to be in an "extended quiescent state",
         * which means that RCU will be completely ignoring that CPU.
         * Therefore, rcu_read_lock() and friends have absolutely no

diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig
index 1c630e573548df34a28a9061e87b961c2a879573..3fa24e63d6f9b1427955112b14a9855e20c46714 100644 (file)
--- a/kernel/rcu/Kconfig
+++ b/kernel/rcu/Kconfig
@@ -8,6 +8,8 @@ menu "RCU Subsystem"
 config TREE_RCU
        bool
        default y if SMP
+       # Dynticks-idle tracking
+       select CONTEXT_TRACKING_IDLE
        help
          This option selects the RCU implementation that is
          designed for very large SMP system with hundreds or

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 9a5edab5558c9dfc201b89a4f0403a4a85f2cc40..051fed0844b675c6fbe6dd7fc7918d7e477f7a6a 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -664,7 +664,6 @@ void noinstr rcu_idle_enter(void)
        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled());
        rcu_eqs_enter(false);
 }
-EXPORT_SYMBOL_GPL(rcu_idle_enter);
 
 #ifdef CONFIG_NO_HZ_FULL
 
@@ -904,7 +903,6 @@ void noinstr rcu_idle_exit(void)
        rcu_eqs_exit(false);
        raw_local_irq_restore(flags);
 }
-EXPORT_SYMBOL_GPL(rcu_idle_exit);
 
 #ifdef CONFIG_NO_HZ_FULL
 /**

diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index fc7fef57560646d5a8e64e757434e4d4ad988840..147214b2cd68ecd2495bf7f46be9012681a15763 100644 (file)
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -85,7 +85,7 @@ module_param(rcu_normal_after_boot, int, 0444);
  * and while lockdep is disabled.
  *
  * Note that if the CPU is in the idle loop from an RCU point of view (ie:
- * that we are in the section between rcu_idle_enter() and rcu_idle_exit())
+ * that we are in the section between ct_idle_enter() and ct_idle_exit())
  * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an
  * rcu_read_lock().  The reason for this is that RCU ignores CPUs that are
  * in such a section, considering these as in extended quiescent state,

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 328cccbee4441ab44b07dd60b78c7646d20d087f..f26ab2675f7d74da6b2ffb066a5612d61c7153a5 100644 (file)
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -53,14 +53,14 @@ static noinline int __cpuidle cpu_idle_poll(void)
 {
        trace_cpu_idle(0, smp_processor_id());
        stop_critical_timings();
-       rcu_idle_enter();
+       ct_idle_enter();
        local_irq_enable();
 
        while (!tif_need_resched() &&
               (cpu_idle_force_poll || tick_check_broadcast_expired()))
                cpu_relax();
 
-       rcu_idle_exit();
+       ct_idle_exit();
        start_critical_timings();
        trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
 
@@ -98,12 +98,12 @@ void __cpuidle default_idle_call(void)
                 *
                 * Trace IRQs enable here, then switch off RCU, and have
                 * arch_cpu_idle() use raw_local_irq_enable(). Note that
-                * rcu_idle_enter() relies on lockdep IRQ state, so switch that
+                * ct_idle_enter() relies on lockdep IRQ state, so switch that
                 * last -- this is very similar to the entry code.
                 */
                trace_hardirqs_on_prepare();
                lockdep_hardirqs_on_prepare();
-               rcu_idle_enter();
+               ct_idle_enter();
                lockdep_hardirqs_on(_THIS_IP_);
 
                arch_cpu_idle();
@@ -116,7 +116,7 @@ void __cpuidle default_idle_call(void)
                 */
                raw_local_irq_disable();
                lockdep_hardirqs_off(_THIS_IP_);
-               rcu_idle_exit();
+               ct_idle_exit();
                lockdep_hardirqs_on(_THIS_IP_);
                raw_local_irq_enable();
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 47b89a0fc6e558595c9f78346ac0424006b858bd..0cfe2d0af2947c5d80d6d9d943b57e0ad16d5993 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -27,6 +27,7 @@
 #include <linux/capability.h>
 #include <linux/cgroup_api.h>
 #include <linux/cgroup.h>
+#include <linux/context_tracking.h>
 #include <linux/cpufreq.h>
 #include <linux/cpumask_api.h>
 #include <linux/ctype.h>

diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 41f99bcfe9e668ca451a95a5c549b7c857ffa517..a41753be1a2bf266183071a9e3c3964ffeabbc81 100644 (file)
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -76,6 +76,12 @@ config TIME_KUNIT_TEST
 config CONTEXT_TRACKING
        bool
 
+config CONTEXT_TRACKING_IDLE
+       bool
+       select CONTEXT_TRACKING
+       help
+         Tracks idle state on behalf of RCU.
+
 if GENERIC_CLOCKEVENTS
 menu "Timers subsystem"