From f09313c57a17683cbcb305989daf1d94b49fd32c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 18 Oct 2021 11:39:45 -0600 Subject: [PATCH] block: cache inode size in bdev Reading the inode size brings in a new cacheline for IO submit, and it's in the hot path being checked for every single IO. When doing millions of IOs per core per second, this is noticeable overhead. Cache the nr_sectors in the bdev itself. Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/genhd.c | 1 + block/partitions/core.c | 1 + include/linux/blk_types.h | 1 + include/linux/genhd.h | 8 ++++---- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/block/genhd.c b/block/genhd.c index 759bc06810f80..53495e3391e3a 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -58,6 +58,7 @@ void set_capacity(struct gendisk *disk, sector_t sectors) spin_lock(&bdev->bd_size_lock); i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); + bdev->bd_nr_sectors = sectors; spin_unlock(&bdev->bd_size_lock); } EXPORT_SYMBOL(set_capacity); diff --git a/block/partitions/core.c b/block/partitions/core.c index 9dbddc355b403..66ef9bc6d6a11 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -91,6 +91,7 @@ static void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) { spin_lock(&bdev->bd_size_lock); i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); + bdev->bd_nr_sectors = sectors; spin_unlock(&bdev->bd_size_lock); } diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 472e55e0e94fd..fe065c394fff6 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -39,6 +39,7 @@ struct bio_crypt_ctx; struct block_device { sector_t bd_start_sect; + sector_t bd_nr_sectors; struct disk_stats __percpu *bd_stats; unsigned long bd_stamp; bool bd_read_only; /* read-only policy */ diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 7b0326661a1ed..900325aa5d316 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -236,14 +236,14 @@ static inline sector_t get_start_sect(struct block_device *bdev) return bdev->bd_start_sect; } -static inline loff_t bdev_nr_bytes(struct block_device *bdev) +static inline sector_t bdev_nr_sectors(struct block_device *bdev) { - return i_size_read(bdev->bd_inode); + return bdev->bd_nr_sectors; } -static inline sector_t bdev_nr_sectors(struct block_device *bdev) +static inline loff_t bdev_nr_bytes(struct block_device *bdev) { - return bdev_nr_bytes(bdev) >> SECTOR_SHIFT; + return bdev_nr_sectors(bdev) << SECTOR_SHIFT; } static inline sector_t get_capacity(struct gendisk *disk) -- 2.30.2