rcu: Add full-sized polling for cond_sync_exp_full()

The cond_synchronize_rcu_expedited() API compresses the combined expedited and
normal grace-period states into a single unsigned long, which conserves
storage, but can miss grace periods in certain cases involving overlapping
normal and expedited grace periods.  Missing the occasional grace period
is usually not a problem, but there are use cases that care about each
and every grace period.

This commit therefore adds yet another member of the full-state RCU
grace-period polling API, which is the cond_synchronize_rcu_exp_full()
function.  This uses up to three times the storage (rcu_gp_oldstate
structure instead of unsigned long), but is guaranteed not to miss
grace periods.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 3bee97f76bf43825a010778a1fd9c495b13bd2b2..4405e9112cee869967238cb3c6cab547eb08b07a 100644 (file)
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -64,6 +64,11 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate)
        cond_synchronize_rcu(oldstate);
 }
 
+static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+       cond_synchronize_rcu_expedited(rgosp->rgos_norm);
+}
+
 extern void rcu_barrier(void);
 
 static inline void synchronize_rcu_expedited(void)

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 1b44288c027da0564313497d411a42499d7c6755..755b082f4ec6298ed7b36c11ba1d72d7dba66866 100644 (file)
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -50,6 +50,7 @@ struct rcu_gp_oldstate {
 unsigned long start_poll_synchronize_rcu_expedited(void);
 void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
 void cond_synchronize_rcu_expedited(unsigned long oldstate);
+void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
 unsigned long get_state_synchronize_rcu(void);
 void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 unsigned long start_poll_synchronize_rcu(void);

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 9d22161bf7700904e93b4ca32f0a5d5eb8784bee..8995429c6f1c22e5c1241127fcaf2db29a771fa5 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -85,6 +85,8 @@ torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind ne
 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");
 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives");
+torture_param(bool, gp_cond_exp_full, false,
+                   "Use conditional/async full-stateexpedited GP wait primitives");
 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");
 torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
@@ -199,20 +201,22 @@ static int rcu_torture_writer_state;
 #define RTWS_COND_GET          5
 #define RTWS_COND_GET_FULL     6
 #define RTWS_COND_GET_EXP      7
-#define RTWS_COND_SYNC         8
-#define RTWS_COND_SYNC_FULL    9
-#define RTWS_COND_SYNC_EXP     10
-#define RTWS_POLL_GET          11
-#define RTWS_POLL_GET_FULL     12
-#define RTWS_POLL_GET_EXP      13
-#define RTWS_POLL_GET_EXP_FULL 14
-#define RTWS_POLL_WAIT         15
-#define RTWS_POLL_WAIT_FULL    16
-#define RTWS_POLL_WAIT_EXP     17
-#define RTWS_POLL_WAIT_EXP_FULL        18
-#define RTWS_SYNC              19
-#define RTWS_STUTTER           20
-#define RTWS_STOPPING          21
+#define RTWS_COND_GET_EXP_FULL 8
+#define RTWS_COND_SYNC         9
+#define RTWS_COND_SYNC_FULL    10
+#define RTWS_COND_SYNC_EXP     11
+#define RTWS_COND_SYNC_EXP_FULL        12
+#define RTWS_POLL_GET          13
+#define RTWS_POLL_GET_FULL     14
+#define RTWS_POLL_GET_EXP      15
+#define RTWS_POLL_GET_EXP_FULL 16
+#define RTWS_POLL_WAIT         17
+#define RTWS_POLL_WAIT_FULL    18
+#define RTWS_POLL_WAIT_EXP     19
+#define RTWS_POLL_WAIT_EXP_FULL        20
+#define RTWS_SYNC              21
+#define RTWS_STUTTER           22
+#define RTWS_STOPPING          23
 static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_FIXED_DELAY",
        "RTWS_DELAY",
@@ -222,9 +226,11 @@ static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_COND_GET",
        "RTWS_COND_GET_FULL",
        "RTWS_COND_GET_EXP",
+       "RTWS_COND_GET_EXP_FULL",
        "RTWS_COND_SYNC",
        "RTWS_COND_SYNC_FULL",
        "RTWS_COND_SYNC_EXP",
+       "RTWS_COND_SYNC_EXP_FULL",
        "RTWS_POLL_GET",
        "RTWS_POLL_GET_FULL",
        "RTWS_POLL_GET_EXP",
@@ -350,6 +356,7 @@ struct rcu_torture_ops {
        void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);
        bool (*poll_gp_state_exp)(unsigned long oldstate);
        void (*cond_sync_exp)(unsigned long oldstate);
+       void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);
        unsigned long (*get_gp_state)(void);
        void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
        unsigned long (*get_gp_completed)(void);
@@ -1183,16 +1190,17 @@ static int nsynctypes;
 static void rcu_torture_write_types(void)
 {
        bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full;
-       bool gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp, gp_poll_exp_full1 = gp_poll_exp_full;
-       bool gp_normal1 = gp_normal, gp_poll1 = gp_poll, gp_poll_full1 = gp_poll_full;
-       bool gp_sync1 = gp_sync;
+       bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp;
+       bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
+       bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;
 
        /* Initialize synctype[] array.  If none set, take default. */
-       if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_exp1 && !gp_poll_exp &&
-           !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1)
-               gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_exp1 = gp_poll_exp1 =
-                          gp_poll_exp_full1 = gp_normal1 = gp_poll1 = gp_poll_full1 =
-                          gp_sync1 = true;
+       if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_cond_exp_full1 && !gp_exp1 &&
+           !gp_poll_exp && !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 &&
+           !gp_sync1)
+               gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_cond_exp_full1 = gp_exp1 =
+                          gp_poll_exp1 = gp_poll_exp_full1 = gp_normal1 = gp_poll1 =
+                          gp_poll_full1 = gp_sync1 = true;
        if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
                synctype[nsynctypes++] = RTWS_COND_GET;
                pr_info("%s: Testing conditional GPs.\n", __func__);
@@ -1211,6 +1219,13 @@ static void rcu_torture_write_types(void)
        } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
                pr_alert("%s: gp_cond_full without primitives.\n", __func__);
        }
+       if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
+               synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL;
+               pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__);
+       } else if (gp_cond_exp_full &&
+                  (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
+               pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__);
+       }
        if (gp_exp1 && cur_ops->exp_sync) {
                synctype[nsynctypes++] = RTWS_EXP_SYNC;
                pr_info("%s: Testing expedited GPs.\n", __func__);
@@ -1418,6 +1433,14 @@ rcu_torture_writer(void *arg)
                                cur_ops->cond_sync_full(&gp_snap_full);
                                rcu_torture_pipe_update(old_rp);
                                break;
+                       case RTWS_COND_GET_EXP_FULL:
+                               rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL;
+                               cur_ops->get_gp_state_full(&gp_snap_full);
+                               torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+                               rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL;
+                               cur_ops->cond_sync_exp_full(&gp_snap_full);
+                               rcu_torture_pipe_update(old_rp);
+                               break;
                        case RTWS_POLL_GET:
                                rcu_torture_writer_state = RTWS_POLL_GET;
                                gp_snap = cur_ops->start_gp_poll();
@@ -1567,6 +1590,11 @@ rcu_torture_fakewriter(void *arg)
                                torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
                                cur_ops->cond_sync_full(&gp_snap_full);
                                break;
+                       case RTWS_COND_GET_EXP_FULL:
+                               cur_ops->get_gp_state_full(&gp_snap_full);
+                               torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+                               cur_ops->cond_sync_exp_full(&gp_snap_full);
+                               break;
                        case RTWS_POLL_GET:
                                gp_snap = cur_ops->start_gp_poll();
                                while (!cur_ops->poll_gp_state(gp_snap)) {

diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 18128ee0d36c0bfafdbb6317e27d81a4023346a7..9c0ae834ef076af2e53e5274d81605cc9fa1299a 100644 (file)
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -1071,3 +1071,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate)
                synchronize_rcu_expedited();
 }
 EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited);
+
+/**
+ * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period
+ * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full()
+ *
+ * If a full RCU grace period has elapsed since the call to
+ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
+ * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was
+ * obtained, just return.  Otherwise, invoke synchronize_rcu_expedited()
+ * to wait for a full grace period.
+ *
+ * Yes, this function does not take counter wrap into account.
+ * But counter wrap is harmless.  If the counter wraps, we have waited for
+ * more than 2 billion grace periods (and way more on a 64-bit system!),
+ * so waiting for a couple of additional grace periods should be just fine.
+ *
+ * This function provides the same memory-ordering guarantees that
+ * would be provided by a synchronize_rcu() that was invoked at the call
+ * to the function that provided @rgosp and that returned at the end of
+ * this function.
+ */
+void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+       if (!poll_state_synchronize_rcu_full(rgosp))
+               synchronize_rcu_expedited();
+}
+EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full);