workqueue: Move pwq_dec_nr_in_flight() to the end of work item handling

The planned shared nr_active handling for unbound workqueues will make
pwq_dec_nr_active() sometimes drop the pool lock temporarily to acquire
other pool locks, which is necessary as retirement of an nr_active count
from one pool may need kick off an inactive work item in another pool.

This patch moves pwq_dec_nr_in_flight() call in try_to_grab_pending() to the
end of work item handling so that work item state changes stay atomic.
process_one_work() which is the other user of pwq_dec_nr_in_flight() already
calls it at the end of work item handling. Comments are added to both call
sites and pwq_dec_nr_in_flight().

This shouldn't cause any behavior changes.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Lai Jiangshan <jiangshanlai@gmail.com>

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1579b8c9a5791c7a3f7c708024a8a43af519a906..b5aba0e5a699045f522b1e461b4f8feb60a1b94c 100644 (file)
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1587,6 +1587,11 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
  * A work either has completed or is removed from pending queue,
  * decrement nr_in_flight of its pwq and handle workqueue flushing.
  *
+ * NOTE:
+ * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
+ * and thus should be called after all other state updates for the in-flight
+ * work item is complete.
+ *
  * CONTEXT:
  * raw_spin_lock_irq(pool->lock).
  */
@@ -1711,11 +1716,13 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
                pwq_activate_work(pwq, work);
 
                list_del_init(&work->entry);
-               pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
 
                /* work->data points to pwq iff queued, point to pool */
                set_work_pool_and_keep_pending(work, pool->id);
 
+               /* must be the last step, see the function comment */
+               pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
+
                raw_spin_unlock(&pool->lock);
                rcu_read_unlock();
                return 1;
@@ -2780,6 +2787,8 @@ __acquires(&pool->lock)
        worker->current_func = NULL;
        worker->current_pwq = NULL;
        worker->current_color = INT_MAX;
+
+       /* must be the last step, see the function comment */
        pwq_dec_nr_in_flight(pwq, work_data);
 }