From 025471f9f50fede6527c70336484becbcb2aff28 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 23 Dec 2019 10:49:01 +0100 Subject: [PATCH] md/raid1: use bucket based mechanism for IO serialization Since raid1 had already used bucket based mechanism to reduce the conflict between write IO and resync IO, it is possible to speed up performance for io serialization with refer to the same mechanism. To align with the barrier bucket mechanism, we created arrays (with the same number of BARRIER_BUCKETS_NR) for spinlock, rb tree and waitqueue. Then we can reduce lock competition with multiple spinlocks, boost search performance with multiple rb trees and also reduce thundering herd problem with multiple waitqueues. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 18 +++++++++++++----- drivers/md/raid1.c | 9 ++++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 9c4e61c988acf..4824d50526fab 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -130,7 +130,7 @@ static void rdev_uninit_serial(struct md_rdev *rdev) if (!test_and_clear_bit(CollisionCheck, &rdev->flags)) return; - kfree(rdev->serial); + kvfree(rdev->serial); rdev->serial = NULL; } @@ -144,18 +144,26 @@ static void rdevs_uninit_serial(struct mddev *mddev) static int rdev_init_serial(struct md_rdev *rdev) { + /* serial_nums equals with BARRIER_BUCKETS_NR */ + int i, serial_nums = 1 << ((PAGE_SHIFT - ilog2(sizeof(atomic_t)))); struct serial_in_rdev *serial = NULL; if (test_bit(CollisionCheck, &rdev->flags)) return 0; - serial = kmalloc(sizeof(struct serial_in_rdev), GFP_KERNEL); + serial = kvmalloc(sizeof(struct serial_in_rdev) * serial_nums, + GFP_KERNEL); if (!serial) return -ENOMEM; - spin_lock_init(&serial->serial_lock); - serial->serial_rb = RB_ROOT_CACHED; - init_waitqueue_head(&serial->serial_io_wait); + for (i = 0; i < serial_nums; i++) { + struct serial_in_rdev *serial_tmp = &serial[i]; + + spin_lock_init(&serial_tmp->serial_lock); + serial_tmp->serial_rb = RB_ROOT_CACHED; + init_waitqueue_head(&serial_tmp->serial_io_wait); + } + rdev->serial = serial; set_bit(CollisionCheck, &rdev->flags); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 5c6a037474480..48d553d7989a4 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -62,7 +62,8 @@ static int check_and_add_serial(struct md_rdev *rdev, sector_t lo, sector_t hi) unsigned long flags; int ret = 0; struct mddev *mddev = rdev->mddev; - struct serial_in_rdev *serial = rdev->serial; + int idx = sector_to_idx(lo); + struct serial_in_rdev *serial = &rdev->serial[idx]; si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO); @@ -87,7 +88,8 @@ static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi) unsigned long flags; int found = 0; struct mddev *mddev = rdev->mddev; - struct serial_in_rdev *serial = rdev->serial; + int idx = sector_to_idx(lo); + struct serial_in_rdev *serial = &rdev->serial[idx]; spin_lock_irqsave(&serial->serial_lock, flags); for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); @@ -1486,7 +1488,8 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, for (i = 0; i < disks; i++) { struct bio *mbio = NULL; struct md_rdev *rdev = conf->mirrors[i].rdev; - struct serial_in_rdev *serial = rdev->serial; + int idx = sector_to_idx(lo); + struct serial_in_rdev *serial = &rdev->serial[idx]; if (!r1_bio->bios[i]) continue; -- 2.30.2