bfq: Split shared queues on move between cgroups

commit 3bc5e683c67d94bd839a1da2e796c15847b51b69 upstream.

When bfqq is shared by multiple processes it can happen that one of the
processes gets moved to a different cgroup (or just starts submitting IO
for different cgroup). In case that happens we need to split the merged
bfqq as otherwise we will have IO for multiple cgroups in one bfqq and
we will just account IO time to wrong entities etc.

Similarly if the bfqq is scheduled to merge with another bfqq but the
merge didn't happen yet, cancel the merge as it need not be valid
anymore.

CC: stable@vger.kernel.org
Fixes: e21b7a0b9887 ("block, bfq: add full hierarchical scheduling and cgroups support")
Tested-by: "yukuai (C)" <yukuai3@huawei.com>
Signed-off-by: Jan Kara <jack@suse.cz>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20220401102752.8599-3-jack@suse.cz
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index e37af3f8a733d139385bcbc9c91fae4344abfa2d..7f7243aa6677be3e1d0b000422b9249a1e6a100e 100644 (file)
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -733,9 +733,39 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
        }
 
        if (sync_bfqq) {
-               entity = &sync_bfqq->entity;
-               if (entity->sched_data != &bfqg->sched_data)
-                       bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
+               if (!sync_bfqq->new_bfqq && !bfq_bfqq_coop(sync_bfqq)) {
+                       /* We are the only user of this bfqq, just move it */
+                       if (sync_bfqq->entity.sched_data != &bfqg->sched_data)
+                               bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
+               } else {
+                       struct bfq_queue *bfqq;
+
+                       /*
+                        * The queue was merged to a different queue. Check
+                        * that the merge chain still belongs to the same
+                        * cgroup.
+                        */
+                       for (bfqq = sync_bfqq; bfqq; bfqq = bfqq->new_bfqq)
+                               if (bfqq->entity.sched_data !=
+                                   &bfqg->sched_data)
+                                       break;
+                       if (bfqq) {
+                               /*
+                                * Some queue changed cgroup so the merge is
+                                * not valid anymore. We cannot easily just
+                                * cancel the merge (by clearing new_bfqq) as
+                                * there may be other processes using this
+                                * queue and holding refs to all queues below
+                                * sync_bfqq->new_bfqq. Similarly if the merge
+                                * already happened, we need to detach from
+                                * bfqq now so that we cannot merge bio to a
+                                * request from the old cgroup.
+                                */
+                               bfq_put_cooperator(sync_bfqq);
+                               bfq_release_process_ref(bfqd, sync_bfqq);
+                               bic_set_bfqq(bic, NULL, 1);
+                       }
+               }
        }
 
        return bfqg;

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index bdc0298c72f17c5fea53213d96449abc26380682..d32d4efba9684d7c66b14a043c2006c9d32b3b08 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -5193,7 +5193,7 @@ static void bfq_put_stable_ref(struct bfq_queue *bfqq)
        bfq_put_queue(bfqq);
 }
 
-static void bfq_put_cooperator(struct bfq_queue *bfqq)
+void bfq_put_cooperator(struct bfq_queue *bfqq)
 {
        struct bfq_queue *__bfqq, *next;
 

diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index 9dc87d3c40c3c1c9340af4a7f2015b5bb78c7b6d..79d003cca90e7417c93b8972b67f79f8b91c41b7 100644 (file)
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -977,6 +977,7 @@ void bfq_weights_tree_remove(struct bfq_data *bfqd,
 void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
                     bool compensate, enum bfqq_expiration reason);
 void bfq_put_queue(struct bfq_queue *bfqq);
+void bfq_put_cooperator(struct bfq_queue *bfqq);
 void bfq_end_wr_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);
 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq);
 void bfq_schedule_dispatch(struct bfq_data *bfqd);