dm bufio: Support IO priority

Some IO will dispatch from kworker with different io_context settings
than the submitting task, we may need to specify a priority to avoid
losing priority.

Add dm_bufio_read_with_ioprio() and dm_bufio_prefetch_with_ioprio()
for use by bufio users to pass an ioprio other than IOPRIO_DEFAULT.

Co-developed-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Hongyu Jin <hongyu.jin@unisoc.com>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Mikulas Patocka <mpatocka@redhat.com>
[snitzer: introduced _with_ioprio() wrappers to reduce churn]
Signed-off-by: Mike Snitzer <snitzer@kernel.org>

diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index f5541b8f6320c9a5db6bb5c22a3d0d59d1f3ec3c..dad75b8d0b55cc351bc6d95ff740221670b25400 100644 (file)
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -1292,7 +1292,8 @@ static void dmio_complete(unsigned long error, void *context)
 }
 
 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
-                    unsigned int n_sectors, unsigned int offset)
+                    unsigned int n_sectors, unsigned int offset,
+                    unsigned short ioprio)
 {
        int r;
        struct dm_io_request io_req = {
@@ -1315,7 +1316,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
                io_req.mem.ptr.vma = (char *)b->data + offset;
        }
 
-       r = dm_io(&io_req, 1, &region, NULL, IOPRIO_DEFAULT);
+       r = dm_io(&io_req, 1, &region, NULL, ioprio);
        if (unlikely(r))
                b->end_io(b, errno_to_blk_status(r));
 }
@@ -1331,7 +1332,8 @@ static void bio_complete(struct bio *bio)
 }
 
 static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
-                   unsigned int n_sectors, unsigned int offset)
+                   unsigned int n_sectors, unsigned int offset,
+                   unsigned short ioprio)
 {
        struct bio *bio;
        char *ptr;
@@ -1339,13 +1341,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
 
        bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
        if (!bio) {
-               use_dmio(b, op, sector, n_sectors, offset);
+               use_dmio(b, op, sector, n_sectors, offset, ioprio);
                return;
        }
        bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op);
        bio->bi_iter.bi_sector = sector;
        bio->bi_end_io = bio_complete;
        bio->bi_private = b;
+       bio->bi_ioprio = ioprio;
 
        ptr = (char *)b->data + offset;
        len = n_sectors << SECTOR_SHIFT;
@@ -1368,7 +1371,7 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block
        return sector;
 }
 
-static void submit_io(struct dm_buffer *b, enum req_op op,
+static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio,
                      void (*end_io)(struct dm_buffer *, blk_status_t))
 {
        unsigned int n_sectors;
@@ -1398,9 +1401,9 @@ static void submit_io(struct dm_buffer *b, enum req_op op,
        }
 
        if (b->data_mode != DATA_MODE_VMALLOC)
-               use_bio(b, op, sector, n_sectors, offset);
+               use_bio(b, op, sector, n_sectors, offset, ioprio);
        else
-               use_dmio(b, op, sector, n_sectors, offset);
+               use_dmio(b, op, sector, n_sectors, offset, ioprio);
 }
 
 /*
@@ -1456,7 +1459,7 @@ static void __write_dirty_buffer(struct dm_buffer *b,
        b->write_end = b->dirty_end;
 
        if (!write_list)
-               submit_io(b, REQ_OP_WRITE, write_endio);
+               submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio);
        else
                list_add_tail(&b->write_list, write_list);
 }
@@ -1470,7 +1473,7 @@ static void __flush_write_list(struct list_head *write_list)
                struct dm_buffer *b =
                        list_entry(write_list->next, struct dm_buffer, write_list);
                list_del(&b->write_list);
-               submit_io(b, REQ_OP_WRITE, write_endio);
+               submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio);
                cond_resched();
        }
        blk_finish_plug(&plug);
@@ -1852,7 +1855,8 @@ static void read_endio(struct dm_buffer *b, blk_status_t status)
  * and uses dm_bufio_mark_buffer_dirty to write new data back).
  */
 static void *new_read(struct dm_bufio_client *c, sector_t block,
-                     enum new_flag nf, struct dm_buffer **bp)
+                     enum new_flag nf, struct dm_buffer **bp,
+                     unsigned short ioprio)
 {
        int need_submit = 0;
        struct dm_buffer *b;
@@ -1905,7 +1909,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block,
                return NULL;
 
        if (need_submit)
-               submit_io(b, REQ_OP_READ, read_endio);
+               submit_io(b, REQ_OP_READ, ioprio, read_endio);
 
        if (nf != NF_GET)       /* we already tested this condition above */
                wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
@@ -1926,32 +1930,46 @@ static void *new_read(struct dm_bufio_client *c, sector_t block,
 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
                   struct dm_buffer **bp)
 {
-       return new_read(c, block, NF_GET, bp);
+       return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_get);
 
-void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
-                   struct dm_buffer **bp)
+static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block,
+                       struct dm_buffer **bp, unsigned short ioprio)
 {
        if (WARN_ON_ONCE(dm_bufio_in_request()))
                return ERR_PTR(-EINVAL);
 
-       return new_read(c, block, NF_READ, bp);
+       return new_read(c, block, NF_READ, bp, ioprio);
+}
+
+void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
+                   struct dm_buffer **bp)
+{
+       return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_read);
 
+void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block,
+                               struct dm_buffer **bp, unsigned short ioprio)
+{
+       return __dm_bufio_read(c, block, bp, ioprio);
+}
+EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio);
+
 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
                   struct dm_buffer **bp)
 {
        if (WARN_ON_ONCE(dm_bufio_in_request()))
                return ERR_PTR(-EINVAL);
 
-       return new_read(c, block, NF_FRESH, bp);
+       return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_new);
 
-void dm_bufio_prefetch(struct dm_bufio_client *c,
-                      sector_t block, unsigned int n_blocks)
+static void __dm_bufio_prefetch(struct dm_bufio_client *c,
+                       sector_t block, unsigned int n_blocks,
+                       unsigned short ioprio)
 {
        struct blk_plug plug;
 
@@ -1987,7 +2005,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
                        dm_bufio_unlock(c);
 
                        if (need_submit)
-                               submit_io(b, REQ_OP_READ, read_endio);
+                               submit_io(b, REQ_OP_READ, ioprio, read_endio);
                        dm_bufio_release(b);
 
                        cond_resched();
@@ -2002,8 +2020,20 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
 flush_plug:
        blk_finish_plug(&plug);
 }
+
+void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks)
+{
+       return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT);
+}
 EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
 
+void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block,
+                               unsigned int n_blocks, unsigned short ioprio)
+{
+       return __dm_bufio_prefetch(c, block, n_blocks, ioprio);
+}
+EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio);
+
 void dm_bufio_release(struct dm_buffer *b)
 {
        struct dm_bufio_client *c = b->c;

diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h
index 75e7d8cbb5327e4fa0350733bf58ede57066323d..d1503b815a7890d4dba44f6401d6921d8a20e56a 100644 (file)
--- a/include/linux/dm-bufio.h
+++ b/include/linux/dm-bufio.h
@@ -64,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start);
 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
                    struct dm_buffer **bp);
 
+void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block,
+                               struct dm_buffer **bp, unsigned short ioprio);
+
 /*
  * Like dm_bufio_read, but return buffer from cache, don't read
  * it. If the buffer is not in the cache, return NULL.
@@ -86,6 +89,10 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
 void dm_bufio_prefetch(struct dm_bufio_client *c,
                       sector_t block, unsigned int n_blocks);
 
+void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c,
+                               sector_t block, unsigned int n_blocks,
+                               unsigned short ioprio);
+
 /*
  * Release a reference obtained with dm_bufio_{read,get,new}. The data
  * pointer and dm_buffer pointer is no longer valid after this call.