srcu: Move ->srcu_cb_mutex from srcu_struct to srcu_usage

This commit moves the ->srcu_cb_mutex field from the srcu_struct structure
to the srcu_usage structure to reduce the size of the former in order
to improve cache locality.

Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 443d27a214ef9c7c4da4450b49b7f51aa9697c98..231de66ceb158ef1de69b6cc75972e543d33efe8 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -65,13 +65,13 @@ struct srcu_usage {
        struct srcu_node *level[RCU_NUM_LVLS + 1];
                                                /* First node at each level. */
        int srcu_size_state;                    /* Small-to-big transition state. */
+       struct mutex srcu_cb_mutex;             /* Serialize CB preparation. */
 };
 
 /*
  * Per-SRCU-domain structure, similar in function to rcu_state.
  */
 struct srcu_struct {
-       struct mutex srcu_cb_mutex;             /* Serialize CB preparation. */
        spinlock_t __private lock;              /* Protect counters and size state. */
        struct mutex srcu_gp_mutex;             /* Serialize GP work. */
        unsigned int srcu_idx;                  /* Current rdr array element. */

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 8428a184d506b4f4b33c04bd2d3a97822c87404f..1814f3bfc219d1862c076a39fd54a0e4713693e0 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -242,7 +242,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
                return -ENOMEM;
        ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;
        ssp->srcu_sup->node = NULL;
-       mutex_init(&ssp->srcu_cb_mutex);
+       mutex_init(&ssp->srcu_sup->srcu_cb_mutex);
        mutex_init(&ssp->srcu_gp_mutex);
        ssp->srcu_idx = 0;
        ssp->srcu_gp_seq = 0;
@@ -861,7 +861,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
        int ss_state;
 
        /* Prevent more than one additional grace period. */
-       mutex_lock(&ssp->srcu_cb_mutex);
+       mutex_lock(&ssp->srcu_sup->srcu_cb_mutex);
 
        /* End the current grace period. */
        spin_lock_irq_rcu_node(ssp);
@@ -921,7 +921,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
                }
 
        /* Callback initiation done, allow grace periods after next. */
-       mutex_unlock(&ssp->srcu_cb_mutex);
+       mutex_unlock(&ssp->srcu_sup->srcu_cb_mutex);
 
        /* Start a new grace period if needed. */
        spin_lock_irq_rcu_node(ssp);