rcutorture: Suppress debugging grace period delays during flooding

Tree RCU supports grace-period delays using the rcutree.gp_cleanup_delay,
rcutree.gp_init_delay, and rcutree.gp_preinit_delay kernel boot
parameters.  These delays are strictly for debugging purposes, and have
proven quite effective at exposing bugs involving race with CPU-hotplug
operations.  However, these delays can result in false positives when
used in conjunction with callback flooding, for example, those generated
by the rcutorture.fwd_progress kernel boot parameter.

This commit therefore suppresses grace-period delays while callback
flooding is in progress.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 24b5f2c2de87b4c37b0d3c52c3189f26d669b455..7a221393fcdbdd135582989c8991cb83bc656642 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -523,6 +523,8 @@ static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { ret
 static inline void show_rcu_gp_kthreads(void) { }
 static inline int rcu_get_gp_kthreads_prio(void) { return 0; }
 static inline void rcu_fwd_progress_check(unsigned long j) { }
+static inline void rcu_gp_slow_register(atomic_t *rgssp) { }
+static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { }
 #else /* #ifdef CONFIG_TINY_RCU */
 bool rcu_dynticks_zero_in_eqs(int cpu, int *vp);
 unsigned long rcu_get_gp_seq(void);
@@ -535,6 +537,8 @@ void rcu_fwd_progress_check(unsigned long j);
 void rcu_force_quiescent_state(void);
 extern struct workqueue_struct *rcu_gp_wq;
 extern struct workqueue_struct *rcu_par_gp_wq;
+void rcu_gp_slow_register(atomic_t *rgssp);
+void rcu_gp_slow_unregister(atomic_t *rgssp);
 #endif /* #else #ifdef CONFIG_TINY_RCU */
 
 #ifdef CONFIG_RCU_NOCB_CPU

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 55d049c39608f581bc8157fc0fbb6264c2ba0bde..f37b7a01dcd002a4924e5aacc8a27a1057a4d04d 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -2916,10 +2916,12 @@ rcu_torture_cleanup(void)
                        pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
                        cur_ops->cb_barrier();
                }
+               rcu_gp_slow_unregister(NULL);
                return;
        }
        if (!cur_ops) {
                torture_cleanup_end();
+               rcu_gp_slow_unregister(NULL);
                return;
        }
 
@@ -3016,6 +3018,7 @@ rcu_torture_cleanup(void)
        else
                rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
        torture_cleanup_end();
+       rcu_gp_slow_unregister(&rcu_fwd_cb_nodelay);
 }
 
 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
@@ -3320,6 +3323,7 @@ rcu_torture_init(void)
        if (object_debug)
                rcu_test_debug_objects();
        torture_init_end();
+       rcu_gp_slow_register(&rcu_fwd_cb_nodelay);
        return 0;
 
 unwind:

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index a4b8189455d5eca9694101d779346a269597994b..db67dae8ed882b25573b449b72fcd13e15595910 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1705,11 +1705,37 @@ static void note_gp_changes(struct rcu_data *rdp)
                rcu_gp_kthread_wake();
 }
 
+static atomic_t *rcu_gp_slow_suppress;
+
+/* Register a counter to suppress debugging grace-period delays. */
+void rcu_gp_slow_register(atomic_t *rgssp)
+{
+       WARN_ON_ONCE(rcu_gp_slow_suppress);
+
+       WRITE_ONCE(rcu_gp_slow_suppress, rgssp);
+}
+EXPORT_SYMBOL_GPL(rcu_gp_slow_register);
+
+/* Unregister a counter, with NULL for not caring which. */
+void rcu_gp_slow_unregister(atomic_t *rgssp)
+{
+       WARN_ON_ONCE(rgssp && rgssp != rcu_gp_slow_suppress);
+
+       WRITE_ONCE(rcu_gp_slow_suppress, NULL);
+}
+EXPORT_SYMBOL_GPL(rcu_gp_slow_unregister);
+
+static bool rcu_gp_slow_is_suppressed(void)
+{
+       atomic_t *rgssp = READ_ONCE(rcu_gp_slow_suppress);
+
+       return rgssp && atomic_read(rgssp);
+}
+
 static void rcu_gp_slow(int delay)
 {
-       if (delay > 0 &&
-           !(rcu_seq_ctr(rcu_state.gp_seq) %
-             (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
+       if (!rcu_gp_slow_is_suppressed() && delay > 0 &&
+           !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
                schedule_timeout_idle(delay);
 }