workqueue: RCU protect wq->dfl_pwq and implement accessors for it

wq->cpu_pwq is RCU protected but wq->dfl_pwq isn't. This is okay because
currently wq->dfl_pwq is used only accessed to install it into wq->cpu_pwq
which doesn't require RCU access. However, we want to be able to access
wq->dfl_pwq under RCU in the future to access its __pod_cpumask and the code
can be made easier to read by making the two pwq fields behave in the same
way.

- Make wq->dfl_pwq RCU protected.

- Add unbound_pwq_slot() and unbound_pwq() which can access both ->dfl_pwq
  and ->cpu_pwq. The former returns the double pointer that can be used
  access and update the pwqs. The latter performs locking check and
  dereferences the double pointer.

- pwq accesses and updates are converted to use unbound_pwq[_slot]().

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Lai Jiangshan <jiangshanlai@gmail.com>

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 489f70846ac9d9090941580e56147b7aa3fa7f61..1579b8c9a5791c7a3f7c708024a8a43af519a906 100644 (file)
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -308,7 +308,7 @@ struct workqueue_struct {
        int                     saved_max_active; /* WQ: saved max_active */
 
        struct workqueue_attrs  *unbound_attrs; /* PW: only for unbound wqs */
-       struct pool_workqueue   *dfl_pwq;       /* PW: only for unbound wqs */
+       struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */
 
 #ifdef CONFIG_SYSFS
        struct wq_device        *wq_dev;        /* I: for sysfs interface */
@@ -639,6 +639,23 @@ static int worker_pool_assign_id(struct worker_pool *pool)
        return ret;
 }
 
+static struct pool_workqueue __rcu **
+unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
+{
+       if (cpu >= 0)
+               return per_cpu_ptr(wq->cpu_pwq, cpu);
+       else
+               return &wq->dfl_pwq;
+}
+
+/* @cpu < 0 for dfl_pwq */
+static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
+{
+       return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
+                                    lockdep_is_held(&wq_pool_mutex) ||
+                                    lockdep_is_held(&wq->mutex));
+}
+
 static unsigned int work_color_to_flags(int color)
 {
        return color << WORK_STRUCT_COLOR_SHIFT;
@@ -4328,10 +4345,11 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu,
                                "possible intersect\n");
 }
 
-/* install @pwq into @wq's cpu_pwq and return the old pwq */
+/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
                                        int cpu, struct pool_workqueue *pwq)
 {
+       struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
        struct pool_workqueue *old_pwq;
 
        lockdep_assert_held(&wq_pool_mutex);
@@ -4340,8 +4358,8 @@ static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
        /* link_pwq() can handle duplicate calls */
        link_pwq(pwq);
 
-       old_pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
-       rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
+       old_pwq = rcu_access_pointer(*slot);
+       rcu_assign_pointer(*slot, pwq);
        return old_pwq;
 }
 
@@ -4441,14 +4459,11 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
 
        copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
 
-       /* save the previous pwq and install the new one */
+       /* save the previous pwqs and install the new ones */
        for_each_possible_cpu(cpu)
                ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
                                                        ctx->pwq_tbl[cpu]);
-
-       /* @dfl_pwq might not have been used, ensure it's linked */
-       link_pwq(ctx->dfl_pwq);
-       swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
+       ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
 
        mutex_unlock(&ctx->wq->mutex);
 }
@@ -4558,9 +4573,7 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu,
 
        /* nothing to do if the target cpumask matches the current pwq */
        wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
-       pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),
-                                       lockdep_is_held(&wq_pool_mutex));
-       if (wqattrs_equal(target_attrs, pwq->pool->attrs))
+       if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
                return;
 
        /* create a new pwq */
@@ -4578,10 +4591,11 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu,
 
 use_dfl_pwq:
        mutex_lock(&wq->mutex);
-       raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
-       get_pwq(wq->dfl_pwq);
-       raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
-       old_pwq = install_unbound_pwq(wq, cpu, wq->dfl_pwq);
+       pwq = unbound_pwq(wq, -1);
+       raw_spin_lock_irq(&pwq->pool->lock);
+       get_pwq(pwq);
+       raw_spin_unlock_irq(&pwq->pool->lock);
+       old_pwq = install_unbound_pwq(wq, cpu, pwq);
 out_unlock:
        mutex_unlock(&wq->mutex);
        put_pwq_unlocked(old_pwq);
@@ -4619,10 +4633,13 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 
        cpus_read_lock();
        if (wq->flags & __WQ_ORDERED) {
+               struct pool_workqueue *dfl_pwq;
+
                ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
                /* there should only be single pwq for ordering guarantee */
-               WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
-                             wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
+               dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
+               WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
+                             wq->pwqs.prev != &dfl_pwq->pwqs_node),
                     "ordering guarantee broken for workqueue %s\n", wq->name);
        } else {
                ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
@@ -4856,7 +4873,7 @@ static bool pwq_busy(struct pool_workqueue *pwq)
                if (pwq->nr_in_flight[i])
                        return true;
 
-       if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
+       if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
                return true;
        if (!pwq_is_empty(pwq))
                return true;
@@ -4940,13 +4957,12 @@ void destroy_workqueue(struct workqueue_struct *wq)
        rcu_read_lock();
 
        for_each_possible_cpu(cpu) {
-               pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
-               RCU_INIT_POINTER(*per_cpu_ptr(wq->cpu_pwq, cpu), NULL);
-               put_pwq_unlocked(pwq);
+               put_pwq_unlocked(unbound_pwq(wq, cpu));
+               RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
        }
 
-       put_pwq_unlocked(wq->dfl_pwq);
-       wq->dfl_pwq = NULL;
+       put_pwq_unlocked(unbound_pwq(wq, -1));
+       RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
 
        rcu_read_unlock();
 }