block: cache inode size in bdev

Reading the inode size brings in a new cacheline for IO submit, and
it's in the hot path being checked for every single IO. When doing
millions of IOs per core per second, this is noticeable overhead.

Cache the nr_sectors in the bdev itself.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/genhd.c b/block/genhd.c
index 759bc06810f801c933dc0435485f0324eff0aece..53495e3391e3a7bcce7057c8a76159c9f12e33a5 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -58,6 +58,7 @@ void set_capacity(struct gendisk *disk, sector_t sectors)
 
        spin_lock(&bdev->bd_size_lock);
        i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
+       bdev->bd_nr_sectors = sectors;
        spin_unlock(&bdev->bd_size_lock);
 }
 EXPORT_SYMBOL(set_capacity);

diff --git a/block/partitions/core.c b/block/partitions/core.c
index 9dbddc355b403cf14f2dabd740c52851dd6468b3..66ef9bc6d6a11085018e2bffd8fc7da3cf33ee37 100644 (file)
--- a/block/partitions/core.c
+++ b/block/partitions/core.c
@@ -91,6 +91,7 @@ static void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors)
 {
        spin_lock(&bdev->bd_size_lock);
        i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
+       bdev->bd_nr_sectors = sectors;
        spin_unlock(&bdev->bd_size_lock);
 }
 

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 472e55e0e94fd906c7946beac65a69a09bd17840..fe065c394fff66f3c72a5ceb88ba802fc157b737 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -39,6 +39,7 @@ struct bio_crypt_ctx;
 
 struct block_device {
        sector_t                bd_start_sect;
+       sector_t                bd_nr_sectors;
        struct disk_stats __percpu *bd_stats;
        unsigned long           bd_stamp;
        bool                    bd_read_only;   /* read-only policy */

diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 7b0326661a1edffd9b7a206c5fdc348f842fd758..900325aa5d3167a8f096a59906212f5665610b1b 100644 (file)
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -236,14 +236,14 @@ static inline sector_t get_start_sect(struct block_device *bdev)
        return bdev->bd_start_sect;
 }
 
-static inline loff_t bdev_nr_bytes(struct block_device *bdev)
+static inline sector_t bdev_nr_sectors(struct block_device *bdev)
 {
-       return i_size_read(bdev->bd_inode);
+       return bdev->bd_nr_sectors;
 }
 
-static inline sector_t bdev_nr_sectors(struct block_device *bdev)
+static inline loff_t bdev_nr_bytes(struct block_device *bdev)
 {
-       return bdev_nr_bytes(bdev) >> SECTOR_SHIFT;
+       return bdev_nr_sectors(bdev) << SECTOR_SHIFT;
 }
 
 static inline sector_t get_capacity(struct gendisk *disk)