srcu: Move ->sda_is_static from srcu_struct to srcu_usage

This commit moves the ->sda_is_static field from the srcu_struct structure
to the srcu_usage structure to reduce the size of the former in order
to improve cache locality.

Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 3023492d8d8931b03b3e826229aee9c127f809ae..d3534ecb806efe3baf1b747b84b8a8e4e8ca28b7 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -76,6 +76,7 @@ struct srcu_usage {
        unsigned long srcu_size_jiffies;        /* Current contention-measurement interval. */
        unsigned long srcu_n_lock_retries;      /* Contention events in current interval. */
        unsigned long srcu_n_exp_nodelay;       /* # expedited no-delays in current GP phase. */
+       bool sda_is_static;                     /* May ->sda be passed to free_percpu()? */
 };
 
 /*
@@ -84,7 +85,6 @@ struct srcu_usage {
 struct srcu_struct {
        unsigned int srcu_idx;                  /* Current rdr array element. */
        struct srcu_data __percpu *sda;         /* Per-CPU srcu_data array. */
-       bool sda_is_static;                     /* May ->sda be passed to free_percpu()? */
        unsigned long srcu_barrier_seq;         /* srcu_barrier seq #. */
        struct mutex srcu_barrier_mutex;        /* Serialize barrier ops. */
        struct completion srcu_barrier_completion;

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 291fb520bce054b0d31154449f1560209c9ff91e..20f2373f7e250e5881da2fdfc0d7342051f788d3 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -252,7 +252,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
        mutex_init(&ssp->srcu_barrier_mutex);
        atomic_set(&ssp->srcu_barrier_cpu_cnt, 0);
        INIT_DELAYED_WORK(&ssp->work, process_srcu);
-       ssp->sda_is_static = is_static;
+       ssp->srcu_sup->sda_is_static = is_static;
        if (!is_static)
                ssp->sda = alloc_percpu(struct srcu_data);
        if (!ssp->sda) {
@@ -265,7 +265,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
        ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();
        if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
                if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {
-                       if (!ssp->sda_is_static) {
+                       if (!ssp->srcu_sup->sda_is_static) {
                                free_percpu(ssp->sda);
                                ssp->sda = NULL;
                                kfree(ssp->srcu_sup);
@@ -667,7 +667,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
        kfree(ssp->srcu_sup->node);
        ssp->srcu_sup->node = NULL;
        ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;
-       if (!ssp->sda_is_static) {
+       if (!ssp->srcu_sup->sda_is_static) {
                free_percpu(ssp->sda);
                ssp->sda = NULL;
                kfree(ssp->srcu_sup);
@@ -1906,7 +1906,7 @@ static void srcu_module_going(struct module *mod)
        for (i = 0; i < mod->num_srcu_structs; i++) {
                ssp = *(sspp++);
                if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&
-                   !WARN_ON_ONCE(!ssp->sda_is_static))
+                   !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))
                        cleanup_srcu_struct(ssp);
                free_percpu(ssp->sda);
        }