srcu: Move ->level from srcu_struct to srcu_usage

This commit moves the ->level[] array from the srcu_struct structure to
the srcu_usage structure to reduce the size of the former in order to
improve cache locality.

Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 276f325f1296048cd573f48743f84fdc0aaf2f98..c7373fe5c14be7654fd62362491804ebb1cd2fa7 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -62,14 +62,14 @@ struct srcu_node {
  */
 struct srcu_usage {
        struct srcu_node *node;                 /* Combining tree. */
+       struct srcu_node *level[RCU_NUM_LVLS + 1];
+                                               /* First node at each level. */
 };
 
 /*
  * Per-SRCU-domain structure, similar in function to rcu_state.
  */
 struct srcu_struct {
-       struct srcu_node *level[RCU_NUM_LVLS + 1];
-                                               /* First node at each level. */
        int srcu_size_state;                    /* Small-to-big transition state. */
        struct mutex srcu_cb_mutex;             /* Serialize CB preparation. */
        spinlock_t __private lock;              /* Protect counters and size state. */

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 049e20dbec7650439d4012729c3957b137d6983f..acb0862faafa2e035d5feb4da9d82b211c414e54 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -178,9 +178,9 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
                return false;
 
        /* Work out the overall tree geometry. */
-       ssp->level[0] = &ssp->srcu_sup->node[0];
+       ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];
        for (i = 1; i < rcu_num_lvls; i++)
-               ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];
+               ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];
        rcu_init_levelspread(levelspread, num_rcu_lvl);
 
        /* Each pass through this loop initializes one srcu_node structure. */
@@ -202,10 +202,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
                }
 
                /* Non-root node. */
-               if (snp == ssp->level[level + 1])
+               if (snp == ssp->srcu_sup->level[level + 1])
                        level++;
-               snp->srcu_parent = ssp->level[level - 1] +
-                                  (snp - ssp->level[level]) /
+               snp->srcu_parent = ssp->srcu_sup->level[level - 1] +
+                                  (snp - ssp->srcu_sup->level[level]) /
                                   levelspread[level - 1];
        }
 
@@ -214,7 +214,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
         * leaves of the srcu_node tree.
         */
        level = rcu_num_lvls - 1;
-       snp_first = ssp->level[level];
+       snp_first = ssp->srcu_sup->level[level];
        for_each_possible_cpu(cpu) {
                sdp = per_cpu_ptr(ssp->sda, cpu);
                sdp->mynode = &snp_first[cpu / levelspread[level]];
@@ -889,7 +889,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
                srcu_for_each_node_breadth_first(ssp, snp) {
                        spin_lock_irq_rcu_node(snp);
                        cbs = false;
-                       last_lvl = snp >= ssp->level[rcu_num_lvls - 1];
+                       last_lvl = snp >= ssp->srcu_sup->level[rcu_num_lvls - 1];
                        if (last_lvl)
                                cbs = ss_state < SRCU_SIZE_BIG || snp->srcu_have_cbs[idx] == gpseq;
                        snp->srcu_have_cbs[idx] = gpseq;