srcu: Use invalid initial value for srcu_node GP sequence numbers

Currently, tree SRCU relies on the srcu_node structures being initialized
at the same time that the srcu_struct itself is initialized, and thus
use the initial grace-period sequence number as the initial value for
the srcu_node structure's ->srcu_have_cbs[] and ->srcu_gp_seq_needed_exp
fields.  Although this has a high probability of also working when the
srcu_node array is allocated and initialized at some random later time,
it would be better to avoid leaving such things to chance.

This commit therefore initializes these fields with 0x2, which is a
recognizable invalid value.  It then adds the required checks for this
invalid value in order to avoid confusion on long-running kernels
(especially those on 32-bit systems) that allocate and initialize
srcu_node arrays late in life.

Co-developed-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index fec608f69962ccbc9c6235d97506d249142474eb..155c430c6a7365e7117b38dde3a414c569f4ab4f 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -107,6 +107,18 @@ static void init_srcu_struct_data(struct srcu_struct *ssp)
        }
 }
 
+/* Invalid seq state, used during snp node initialization */
+#define SRCU_SNP_INIT_SEQ              0x2
+
+/*
+ * Check whether sequence number corresponding to snp node,
+ * is invalid.
+ */
+static inline bool srcu_invl_snp_seq(unsigned long s)
+{
+       return rcu_seq_state(s) == SRCU_SNP_INIT_SEQ;
+}
+
 /*
  * Allocated and initialize SRCU combining tree.  Returns @true if
  * allocation succeeded and @false otherwise.
@@ -139,10 +151,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp)
                WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=
                             ARRAY_SIZE(snp->srcu_data_have_cbs));
                for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {
-                       snp->srcu_have_cbs[i] = 0;
+                       snp->srcu_have_cbs[i] = SRCU_SNP_INIT_SEQ;
                        snp->srcu_data_have_cbs[i] = 0;
                }
-               snp->srcu_gp_seq_needed_exp = 0;
+               snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;
                snp->grplo = -1;
                snp->grphi = -1;
                if (snp == &ssp->node[0]) {
@@ -386,8 +398,7 @@ static bool srcu_readers_active(struct srcu_struct *ssp)
  */
 static unsigned long srcu_get_delay(struct srcu_struct *ssp)
 {
-       if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq),
-                        READ_ONCE(ssp->srcu_gp_seq_needed_exp)))
+       if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp)))
                return 0;
        return SRCU_INTERVAL;
 }
@@ -561,6 +572,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
        int idx;
        unsigned long mask;
        struct srcu_data *sdp;
+       unsigned long sgsne;
        struct srcu_node *snp;
        int ss_state;
 
@@ -594,7 +606,8 @@ static void srcu_gp_end(struct srcu_struct *ssp)
                                cbs = snp->srcu_have_cbs[idx] == gpseq;
                        snp->srcu_have_cbs[idx] = gpseq;
                        rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
-                       if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))
+                       sgsne = snp->srcu_gp_seq_needed_exp;
+                       if (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, gpseq))
                                WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
                        mask = snp->srcu_data_have_cbs[idx];
                        snp->srcu_data_have_cbs[idx] = 0;
@@ -652,14 +665,17 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp
                                  unsigned long s)
 {
        unsigned long flags;
+       unsigned long sgsne;
 
        if (snp)
                for (; snp != NULL; snp = snp->srcu_parent) {
+                       sgsne = READ_ONCE(snp->srcu_gp_seq_needed_exp);
                        if (rcu_seq_done(&ssp->srcu_gp_seq, s) ||
-                           ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))
+                           (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)))
                                return;
                        spin_lock_irqsave_rcu_node(snp, flags);
-                       if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {
+                       sgsne = snp->srcu_gp_seq_needed_exp;
+                       if (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)) {
                                spin_unlock_irqrestore_rcu_node(snp, flags);
                                return;
                        }
@@ -687,6 +703,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
 {
        unsigned long flags;
        int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);
+       unsigned long sgsne;
        struct srcu_node *snp;
        struct srcu_node *snp_leaf = smp_load_acquire(&sdp->mynode);
        unsigned long snp_seq;
@@ -698,7 +715,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
                                return; /* GP already done and CBs recorded. */
                        spin_lock_irqsave_rcu_node(snp, flags);
                        snp_seq = snp->srcu_have_cbs[idx];
-                       if (ULONG_CMP_GE(snp_seq, s)) {
+                       if (!srcu_invl_snp_seq(snp_seq) && ULONG_CMP_GE(snp_seq, s)) {
                                if (snp == snp_leaf && snp_seq == s)
                                        snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
                                spin_unlock_irqrestore_rcu_node(snp, flags);
@@ -713,7 +730,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
                        snp->srcu_have_cbs[idx] = s;
                        if (snp == snp_leaf)
                                snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
-                       if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
+                       sgsne = snp->srcu_gp_seq_needed_exp;
+                       if (!do_norm && (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, s)))
                                WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
                        spin_unlock_irqrestore_rcu_node(snp, flags);
                }